/**
* It writes the primitive value to the Parquet RecordConsumer.
*
* @param value The writable object that contains the primitive value.
*/
private void writePrimitive(final Writable value) {
if (value == null) {
return;
}
if (value instanceof DoubleWritable) {
recordConsumer.addDouble(((DoubleWritable) value).get());
} else if (value instanceof BooleanWritable) {
recordConsumer.addBoolean(((BooleanWritable) value).get());
} else if (value instanceof FloatWritable) {
recordConsumer.addFloat(((FloatWritable) value).get());
} else if (value instanceof IntWritable) {
recordConsumer.addInteger(((IntWritable) value).get());
} else if (value instanceof LongWritable) {
recordConsumer.addLong(((LongWritable) value).get());
} else if (value instanceof ShortWritable) {
recordConsumer.addInteger(((ShortWritable) value).get());
} else if (value instanceof ByteWritable) {
recordConsumer.addInteger(((ByteWritable) value).get());
} else if (value instanceof HiveDecimalWritable) {
throw new UnsupportedOperationException("HiveDecimalWritable writing not implemented");
} else if (value instanceof BytesWritable) {
recordConsumer.addBinary((Binary.fromByteArray(((BytesWritable) value).getBytes())));
} else if (value instanceof TimestampWritable) {
Timestamp ts = ((TimestampWritable) value).getTimestamp();
NanoTime nt = NanoTimeUtils.getNanoTime(ts);
nt.writeValue(recordConsumer);
} else {
throw new IllegalArgumentException("Unknown value type: " + value + " " + value.getClass());
}
}
public FieldEnumConverter(List<TProtocol> events, ThriftField field) {
this.events = events;
final List<EnumValue> values = ((EnumType) field.getType()).getValues();
for (EnumValue enumValue : values) {
enumLookup.put(Binary.fromString(enumValue.getName()), enumValue.getId());
}
}
/**
* will add an entry to the dictionary if the value is new
*
* @param v the value to dictionary encode
*/
private void writeBytesUsingDict(Binary v) {
Integer id = dict.get(v);
if (id == null) {
id = dict.size();
dict.put(v, id);
// length as int (2 bytes) + actual bytes
dictionaryByteSize += 2 + v.length();
}
out.add(id);
}
@Override
public final void addBinary(Binary value) {
try {
ProtobufDatumFactory factory = ProtobufDatumFactory.get(dataType.getCode());
Message.Builder builder = factory.newBuilder();
builder.mergeFrom(value.getBytes());
parent.add(factory.createDatum(builder));
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
}
@Override
public DictionaryPage createDictionaryPage() {
if (lastUsedDictionarySize > 0) {
// return a dictionary only if we actually used it
try {
CapacityByteArrayOutputStream dictBuf =
new CapacityByteArrayOutputStream(lastUsedDictionaryByteSize);
LittleEndianDataOutputStream dictOut = new LittleEndianDataOutputStream(dictBuf);
Iterator<Binary> entryIterator = dict.keySet().iterator();
// write only the part of the dict that we used
for (int i = 0; i < lastUsedDictionarySize; i++) {
Binary entry = entryIterator.next();
dictOut.writeInt(entry.length());
entry.writeTo(dictOut);
}
return new DictionaryPage(
BytesInput.from(dictBuf), lastUsedDictionarySize, PLAIN_DICTIONARY);
} catch (IOException e) {
throw new ParquetEncodingException("Could not generate dictionary Page", e);
}
}
return plainValuesWriter.createDictionaryPage();
}
private Writable createPrimitive(final Object obj, final PrimitiveObjectInspector inspector)
throws SerDeException {
if (obj == null) {
return null;
}
switch (inspector.getPrimitiveCategory()) {
case VOID:
return null;
case BOOLEAN:
return new BooleanWritable(
((BooleanObjectInspector) inspector).get(obj) ? Boolean.TRUE : Boolean.FALSE);
case BYTE:
return new ByteWritable(((ByteObjectInspector) inspector).get(obj));
case DOUBLE:
return new DoubleWritable(((DoubleObjectInspector) inspector).get(obj));
case FLOAT:
return new FloatWritable(((FloatObjectInspector) inspector).get(obj));
case INT:
return new IntWritable(((IntObjectInspector) inspector).get(obj));
case LONG:
return new LongWritable(((LongObjectInspector) inspector).get(obj));
case SHORT:
return new ShortWritable(((ShortObjectInspector) inspector).get(obj));
case STRING:
String v = ((StringObjectInspector) inspector).getPrimitiveJavaObject(obj);
try {
return new BytesWritable(v.getBytes("UTF-8"));
} catch (UnsupportedEncodingException e) {
throw new SerDeException("Failed to encode string in UTF-8", e);
}
case DECIMAL:
HiveDecimal hd = (HiveDecimal) inspector.getPrimitiveJavaObject(obj);
DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) inspector.getTypeInfo();
int prec = decTypeInfo.precision();
int scale = decTypeInfo.scale();
byte[] src = hd.setScale(scale).unscaledValue().toByteArray();
// Estimated number of bytes needed.
int bytes = PRECISION_TO_BYTE_COUNT[prec - 1];
if (bytes == src.length) {
// No padding needed.
return new BytesWritable(src);
}
byte[] tgt = new byte[bytes];
if (hd.signum() == -1) {
// For negative number, initializing bits to 1
for (int i = 0; i < bytes; i++) {
tgt[i] |= 0xFF;
}
}
System.arraycopy(
src, 0, tgt, bytes - src.length, src.length); // Padding leading zeroes/ones.
return new BytesWritable(tgt);
case TIMESTAMP:
return new TimestampWritable(
((TimestampObjectInspector) inspector).getPrimitiveJavaObject(obj));
case CHAR:
String strippedValue =
((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(obj).getStrippedValue();
return new BytesWritable(Binary.fromString(strippedValue).getBytes());
case VARCHAR:
String value =
((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(obj).getValue();
return new BytesWritable(Binary.fromString(value).getBytes());
case BINARY:
return new BytesWritable(((BinaryObjectInspector) inspector).getPrimitiveJavaObject(obj));
default:
throw new SerDeException("Unknown primitive : " + inspector.getPrimitiveCategory());
}
}
private void addString(String value) {
inOrder.verify(mockRecordConsumer).addBinary(Binary.fromString(value));
}
@Override
public void addBinary(Binary value) {
nulls[fieldIndex] = false;
slices[fieldIndex] = Slices.wrappedBuffer(value.getBytes());
}
@Override
public void addBinary(Binary value) {
VARBINARY.writeSlice(builder, Slices.wrappedBuffer(value.getBytes()));
wroteValue = true;
}
@Override
public final void addBinary(Binary value) {
parent.add(new BlobDatum(ByteBuffer.wrap(value.getBytes())));
}
@Override
public final void addBinary(Binary value) {
parent.add(DatumFactory.createText(value.getBytes()));
}