public FileSystemViewKeyInputFormat(FileSystemDataset<E> dataset, Configuration conf) {
    this.dataset = dataset;
    LOG.debug("Dataset: {}", dataset);

    Format format = dataset.getDescriptor().getFormat();

    boolean isSpecific = SpecificRecord.class.isAssignableFrom(dataset.getType());

    if (Formats.AVRO.equals(format)) {
      setModel.invoke(conf, DataModelUtil.getDataModelForType(dataset.getType()).getClass());

      // Use the reader's schema type if provided.
      if (isSpecific) {

        conf.set(AVRO_SCHEMA_INPUT_KEY, SpecificData.get().getSchema(dataset.getType()).toString());
      }
    } else if (Formats.PARQUET.equals(format)) {

      // Use the reader's schema type if provided.
      if (isSpecific) {

        AvroReadSupport.setAvroReadSchema(conf, SpecificData.get().getSchema(dataset.getType()));
      }
    }
  }
Beispiel #2
0
 @Override
 public void initialize() {
   if (key == null) {
     // restore transient objects from serializable versions
     PartitionStrategy strategy = PartitionStrategyParser.parse(strategyString);
     Schema schema = new Schema.Parser().parse(schemaString);
     this.key = new AvroStorageKey(strategy, schema);
     this.accessor = DataModelUtil.accessor(type, schema);
     if (constraints != null) {
       this.provided =
           Constraints.fromQueryMap(schema, strategy, constraints).getProvidedValues();
     }
   }
 }