MapConverter(List<TProtocol> parentEvents, GroupType parquetSchema, ThriftField field) {
this.parentEvents = parentEvents;
if (parquetSchema.getFieldCount() != 1) {
throw new IllegalArgumentException(
"maps have only one field. "
+ parquetSchema
+ " size = "
+ parquetSchema.getFieldCount());
}
Type nestedType = parquetSchema.getType(0);
final ThriftField key = ((MapType) field.getType()).getKey();
keyType = key.getType().getType().getThriftType();
final ThriftField value = ((MapType) field.getType()).getValue();
valueType = value.getType().getType().getThriftType();
child = new GroupCounter(new MapKeyValueConverter(mapEvents, nestedType, key, value));
}
public FieldEnumConverter(List<TProtocol> events, ThriftField field) {
this.events = events;
final List<EnumValue> values = ((EnumType) field.getType()).getValues();
for (EnumValue enumValue : values) {
enumLookup.put(Binary.fromString(enumValue.getName()), enumValue.getId());
}
}
public PrimitiveFieldHandler(
PrimitiveConverter delegate, final ThriftField field, List<TProtocol> events) {
this.delegate = delegate;
this.events = events;
final byte thriftType =
field.getType().getType() == ThriftTypeID.ENUM
? ThriftTypeID.I32.getThriftType()
: // enums are serialized as I32
field.getType().getType().getThriftType();
this.readFieldBegin =
new ParquetProtocol("readFieldBegin()") {
@Override
public TField readFieldBegin() throws TException {
return new TField(field.getName(), thriftType, field.getFieldId());
}
};
}
private Converter newConverter(List<TProtocol> events, Type type, ThriftField field) {
switch (field.getType().getType()) {
case LIST:
return new ListConverter(events, type.asGroupType(), field);
case SET:
return new SetConverter(events, type.asGroupType(), field);
case MAP:
return new MapConverter(events, type.asGroupType(), field);
case STRUCT:
return new StructConverter(events, type.asGroupType(), field);
case STRING:
return new FieldStringConverter(events, field);
case ENUM:
return new FieldEnumConverter(events, field);
default:
return new FieldPrimitiveConverter(events, field);
}
}
private StructConverter(List<TProtocol> events, GroupType parquetSchema, ThriftField field) {
this.events = events;
this.name = field.getName();
this.tStruct = new TStruct(name);
this.thriftType = (StructType) field.getType();
this.schemaSize = parquetSchema.getFieldCount();
this.converters = new Converter[this.schemaSize];
List<ThriftField> thriftChildren = thriftType.getChildren();
for (int i = 0; i < schemaSize; i++) {
Type schemaType = parquetSchema.getType(i);
String fieldName = schemaType.getName();
ThriftField matchingThrift = null;
for (ThriftField childField : thriftChildren) {
String thriftChildName = childField.getName();
if (thriftChildName != null && thriftChildName.equalsIgnoreCase(fieldName)) {
matchingThrift = childField;
break;
}
}
if (matchingThrift == null) {
// this means the file did not contain that field
// it will never be populated in this instance
// other files might populate it
continue;
}
if (schemaType.isPrimitive()) {
converters[i] =
new PrimitiveFieldHandler(
newConverter(events, schemaType, matchingThrift).asPrimitiveConverter(),
matchingThrift,
events);
} else {
converters[i] =
new GroupFieldhandler(
newConverter(events, schemaType, matchingThrift).asGroupConverter(),
matchingThrift,
events);
}
}
}
CollectionConverter(List<TProtocol> parentEvents, GroupType parquetSchema, ThriftField values) {
this.parentEvents = parentEvents;
if (parquetSchema.getFieldCount() != 1) {
throw new IllegalArgumentException(
"lists have only one field. "
+ parquetSchema
+ " size = "
+ parquetSchema.getFieldCount());
}
nestedType = parquetSchema.getType(0);
valuesType = values.getType().getType();
if (nestedType.isPrimitive()) {
PrimitiveCounter counter =
new PrimitiveCounter(
newConverter(listEvents, nestedType, values).asPrimitiveConverter());
child = counter;
childCounter = counter;
} else {
GroupCounter counter =
new GroupCounter(newConverter(listEvents, nestedType, values).asGroupConverter());
child = counter;
childCounter = counter;
}
}
ListConverter(List<TProtocol> parentEvents, GroupType parquetSchema, ThriftField field) {
super(parentEvents, parquetSchema, ((ListType) field.getType()).getValues());
this.parentEvents = parentEvents;
}
public FieldPrimitiveConverter(List<TProtocol> events, ThriftField field) {
this.events = events;
this.type = field.getType().getType();
}