public FieldEnumConverter(List<TProtocol> events, ThriftField field) { this.events = events; final List<EnumValue> values = ((EnumType) field.getType()).getValues(); for (EnumValue enumValue : values) { enumLookup.put(Binary.fromString(enumValue.getName()), enumValue.getId()); } }
public PrimitiveFieldHandler( PrimitiveConverter delegate, final ThriftField field, List<TProtocol> events) { this.delegate = delegate; this.events = events; final byte thriftType = field.getType().getType() == ThriftTypeID.ENUM ? ThriftTypeID.I32.getThriftType() : // enums are serialized as I32 field.getType().getType().getThriftType(); this.readFieldBegin = new ParquetProtocol("readFieldBegin()") { @Override public TField readFieldBegin() throws TException { return new TField(field.getName(), thriftType, field.getFieldId()); } }; }
private StructConverter(List<TProtocol> events, GroupType parquetSchema, ThriftField field) { this.events = events; this.name = field.getName(); this.tStruct = new TStruct(name); this.thriftType = (StructType) field.getType(); this.schemaSize = parquetSchema.getFieldCount(); this.converters = new Converter[this.schemaSize]; List<ThriftField> thriftChildren = thriftType.getChildren(); for (int i = 0; i < schemaSize; i++) { Type schemaType = parquetSchema.getType(i); String fieldName = schemaType.getName(); ThriftField matchingThrift = null; for (ThriftField childField : thriftChildren) { String thriftChildName = childField.getName(); if (thriftChildName != null && thriftChildName.equalsIgnoreCase(fieldName)) { matchingThrift = childField; break; } } if (matchingThrift == null) { // this means the file did not contain that field // it will never be populated in this instance // other files might populate it continue; } if (schemaType.isPrimitive()) { converters[i] = new PrimitiveFieldHandler( newConverter(events, schemaType, matchingThrift).asPrimitiveConverter(), matchingThrift, events); } else { converters[i] = new GroupFieldhandler( newConverter(events, schemaType, matchingThrift).asGroupConverter(), matchingThrift, events); } } }
private Converter newConverter(List<TProtocol> events, Type type, ThriftField field) { switch (field.getType().getType()) { case LIST: return new ListConverter(events, type.asGroupType(), field); case SET: return new SetConverter(events, type.asGroupType(), field); case MAP: return new MapConverter(events, type.asGroupType(), field); case STRUCT: return new StructConverter(events, type.asGroupType(), field); case STRING: return new FieldStringConverter(events, field); case ENUM: return new FieldEnumConverter(events, field); default: return new FieldPrimitiveConverter(events, field); } }
MapConverter(List<TProtocol> parentEvents, GroupType parquetSchema, ThriftField field) { this.parentEvents = parentEvents; if (parquetSchema.getFieldCount() != 1) { throw new IllegalArgumentException( "maps have only one field. " + parquetSchema + " size = " + parquetSchema.getFieldCount()); } Type nestedType = parquetSchema.getType(0); final ThriftField key = ((MapType) field.getType()).getKey(); keyType = key.getType().getType().getThriftType(); final ThriftField value = ((MapType) field.getType()).getValue(); valueType = value.getType().getType().getThriftType(); child = new GroupCounter(new MapKeyValueConverter(mapEvents, nestedType, key, value)); }
CollectionConverter(List<TProtocol> parentEvents, GroupType parquetSchema, ThriftField values) { this.parentEvents = parentEvents; if (parquetSchema.getFieldCount() != 1) { throw new IllegalArgumentException( "lists have only one field. " + parquetSchema + " size = " + parquetSchema.getFieldCount()); } nestedType = parquetSchema.getType(0); valuesType = values.getType().getType(); if (nestedType.isPrimitive()) { PrimitiveCounter counter = new PrimitiveCounter( newConverter(listEvents, nestedType, values).asPrimitiveConverter()); child = counter; childCounter = counter; } else { GroupCounter counter = new GroupCounter(newConverter(listEvents, nestedType, values).asGroupConverter()); child = counter; childCounter = counter; } }
ListConverter(List<TProtocol> parentEvents, GroupType parquetSchema, ThriftField field) { super(parentEvents, parquetSchema, ((ListType) field.getType()).getValues()); this.parentEvents = parentEvents; }
public FieldPrimitiveConverter(List<TProtocol> events, ThriftField field) { this.events = events; this.type = field.getType().getType(); }