Ejemplo n.º 1
0
 static GlobalMetaData mergeInto(
     FileMetaData toMerge, GlobalMetaData mergedMetadata, boolean strict) {
   MessageType schema = null;
   Map<String, Set<String>> newKeyValues = new HashMap<String, Set<String>>();
   Set<String> createdBy = new HashSet<String>();
   if (mergedMetadata != null) {
     schema = mergedMetadata.getSchema();
     newKeyValues.putAll(mergedMetadata.getKeyValueMetaData());
     createdBy.addAll(mergedMetadata.getCreatedBy());
   }
   if ((schema == null && toMerge.getSchema() != null)
       || (schema != null && !schema.equals(toMerge.getSchema()))) {
     schema = mergeInto(toMerge.getSchema(), schema, strict);
   }
   for (Entry<String, String> entry : toMerge.getKeyValueMetaData().entrySet()) {
     Set<String> values = newKeyValues.get(entry.getKey());
     if (values == null) {
       values = new HashSet<String>();
       newKeyValues.put(entry.getKey(), values);
     }
     values.add(entry.getValue());
   }
   createdBy.add(toMerge.getCreatedBy());
   return new GlobalMetaData(schema, newKeyValues, createdBy);
 }
Ejemplo n.º 2
0
 static ParquetMetadata mergeFooters(Path root, List<Footer> footers) {
   String rootPath = root.toUri().getPath();
   GlobalMetaData fileMetaData = null;
   List<BlockMetaData> blocks = new ArrayList<BlockMetaData>();
   for (Footer footer : footers) {
     String footerPath = footer.getFile().toUri().getPath();
     if (!footerPath.startsWith(rootPath)) {
       throw new ParquetEncodingException(
           footerPath + " invalid: all the files must be contained in the root " + root);
     }
     footerPath = footerPath.substring(rootPath.length());
     while (footerPath.startsWith("/")) {
       footerPath = footerPath.substring(1);
     }
     fileMetaData = mergeInto(footer.getParquetMetadata().getFileMetaData(), fileMetaData);
     for (BlockMetaData block : footer.getParquetMetadata().getBlocks()) {
       block.setPath(footerPath);
       blocks.add(block);
     }
   }
   return new ParquetMetadata(fileMetaData.merge(), blocks);
 }