Ejemplo n.º 1
0
 MapConverter(List<TProtocol> parentEvents, GroupType parquetSchema, ThriftField field) {
   this.parentEvents = parentEvents;
   if (parquetSchema.getFieldCount() != 1) {
     throw new IllegalArgumentException(
         "maps have only one field. "
             + parquetSchema
             + " size = "
             + parquetSchema.getFieldCount());
   }
   Type nestedType = parquetSchema.getType(0);
   final ThriftField key = ((MapType) field.getType()).getKey();
   keyType = key.getType().getType().getThriftType();
   final ThriftField value = ((MapType) field.getType()).getValue();
   valueType = value.getType().getType().getThriftType();
   child = new GroupCounter(new MapKeyValueConverter(mapEvents, nestedType, key, value));
 }
Ejemplo n.º 2
0
 public FieldEnumConverter(List<TProtocol> events, ThriftField field) {
   this.events = events;
   final List<EnumValue> values = ((EnumType) field.getType()).getValues();
   for (EnumValue enumValue : values) {
     enumLookup.put(Binary.fromString(enumValue.getName()), enumValue.getId());
   }
 }
Ejemplo n.º 3
0
 public PrimitiveFieldHandler(
     PrimitiveConverter delegate, final ThriftField field, List<TProtocol> events) {
   this.delegate = delegate;
   this.events = events;
   final byte thriftType =
       field.getType().getType() == ThriftTypeID.ENUM
           ? ThriftTypeID.I32.getThriftType()
           : // enums are serialized as I32
           field.getType().getType().getThriftType();
   this.readFieldBegin =
       new ParquetProtocol("readFieldBegin()") {
         @Override
         public TField readFieldBegin() throws TException {
           return new TField(field.getName(), thriftType, field.getFieldId());
         }
       };
 }
Ejemplo n.º 4
0
 private Converter newConverter(List<TProtocol> events, Type type, ThriftField field) {
   switch (field.getType().getType()) {
     case LIST:
       return new ListConverter(events, type.asGroupType(), field);
     case SET:
       return new SetConverter(events, type.asGroupType(), field);
     case MAP:
       return new MapConverter(events, type.asGroupType(), field);
     case STRUCT:
       return new StructConverter(events, type.asGroupType(), field);
     case STRING:
       return new FieldStringConverter(events, field);
     case ENUM:
       return new FieldEnumConverter(events, field);
     default:
       return new FieldPrimitiveConverter(events, field);
   }
 }
Ejemplo n.º 5
0
 private StructConverter(List<TProtocol> events, GroupType parquetSchema, ThriftField field) {
   this.events = events;
   this.name = field.getName();
   this.tStruct = new TStruct(name);
   this.thriftType = (StructType) field.getType();
   this.schemaSize = parquetSchema.getFieldCount();
   this.converters = new Converter[this.schemaSize];
   List<ThriftField> thriftChildren = thriftType.getChildren();
   for (int i = 0; i < schemaSize; i++) {
     Type schemaType = parquetSchema.getType(i);
     String fieldName = schemaType.getName();
     ThriftField matchingThrift = null;
     for (ThriftField childField : thriftChildren) {
       String thriftChildName = childField.getName();
       if (thriftChildName != null && thriftChildName.equalsIgnoreCase(fieldName)) {
         matchingThrift = childField;
         break;
       }
     }
     if (matchingThrift == null) {
       // this means the file did not contain that field
       // it will never be populated in this instance
       // other files might populate it
       continue;
     }
     if (schemaType.isPrimitive()) {
       converters[i] =
           new PrimitiveFieldHandler(
               newConverter(events, schemaType, matchingThrift).asPrimitiveConverter(),
               matchingThrift,
               events);
     } else {
       converters[i] =
           new GroupFieldhandler(
               newConverter(events, schemaType, matchingThrift).asGroupConverter(),
               matchingThrift,
               events);
     }
   }
 }
Ejemplo n.º 6
0
 CollectionConverter(List<TProtocol> parentEvents, GroupType parquetSchema, ThriftField values) {
   this.parentEvents = parentEvents;
   if (parquetSchema.getFieldCount() != 1) {
     throw new IllegalArgumentException(
         "lists have only one field. "
             + parquetSchema
             + " size = "
             + parquetSchema.getFieldCount());
   }
   nestedType = parquetSchema.getType(0);
   valuesType = values.getType().getType();
   if (nestedType.isPrimitive()) {
     PrimitiveCounter counter =
         new PrimitiveCounter(
             newConverter(listEvents, nestedType, values).asPrimitiveConverter());
     child = counter;
     childCounter = counter;
   } else {
     GroupCounter counter =
         new GroupCounter(newConverter(listEvents, nestedType, values).asGroupConverter());
     child = counter;
     childCounter = counter;
   }
 }
Ejemplo n.º 7
0
 ListConverter(List<TProtocol> parentEvents, GroupType parquetSchema, ThriftField field) {
   super(parentEvents, parquetSchema, ((ListType) field.getType()).getValues());
   this.parentEvents = parentEvents;
 }
Ejemplo n.º 8
0
 public FieldPrimitiveConverter(List<TProtocol> events, ThriftField field) {
   this.events = events;
   this.type = field.getType().getType();
 }