/**
  * It writes the primitive value to the Parquet RecordConsumer.
  *
  * @param value The writable object that contains the primitive value.
  */
 private void writePrimitive(final Writable value) {
   if (value == null) {
     return;
   }
   if (value instanceof DoubleWritable) {
     recordConsumer.addDouble(((DoubleWritable) value).get());
   } else if (value instanceof BooleanWritable) {
     recordConsumer.addBoolean(((BooleanWritable) value).get());
   } else if (value instanceof FloatWritable) {
     recordConsumer.addFloat(((FloatWritable) value).get());
   } else if (value instanceof IntWritable) {
     recordConsumer.addInteger(((IntWritable) value).get());
   } else if (value instanceof LongWritable) {
     recordConsumer.addLong(((LongWritable) value).get());
   } else if (value instanceof ShortWritable) {
     recordConsumer.addInteger(((ShortWritable) value).get());
   } else if (value instanceof ByteWritable) {
     recordConsumer.addInteger(((ByteWritable) value).get());
   } else if (value instanceof HiveDecimalWritable) {
     throw new UnsupportedOperationException("HiveDecimalWritable writing not implemented");
   } else if (value instanceof BytesWritable) {
     recordConsumer.addBinary((Binary.fromByteArray(((BytesWritable) value).getBytes())));
   } else if (value instanceof TimestampWritable) {
     Timestamp ts = ((TimestampWritable) value).getTimestamp();
     NanoTime nt = NanoTimeUtils.getNanoTime(ts);
     nt.writeValue(recordConsumer);
   } else {
     throw new IllegalArgumentException("Unknown value type: " + value + " " + value.getClass());
   }
 }
 public FieldEnumConverter(List<TProtocol> events, ThriftField field) {
   this.events = events;
   final List<EnumValue> values = ((EnumType) field.getType()).getValues();
   for (EnumValue enumValue : values) {
     enumLookup.put(Binary.fromString(enumValue.getName()), enumValue.getId());
   }
 }
 /**
  * will add an entry to the dictionary if the value is new
  *
  * @param v the value to dictionary encode
  */
 private void writeBytesUsingDict(Binary v) {
   Integer id = dict.get(v);
   if (id == null) {
     id = dict.size();
     dict.put(v, id);
     // length as int (2 bytes) + actual bytes
     dictionaryByteSize += 2 + v.length();
   }
   out.add(id);
 }
示例#4
0
 @Override
 public final void addBinary(Binary value) {
   try {
     ProtobufDatumFactory factory = ProtobufDatumFactory.get(dataType.getCode());
     Message.Builder builder = factory.newBuilder();
     builder.mergeFrom(value.getBytes());
     parent.add(factory.createDatum(builder));
   } catch (InvalidProtocolBufferException e) {
     throw new RuntimeException(e);
   }
 }
 @Override
 public DictionaryPage createDictionaryPage() {
   if (lastUsedDictionarySize > 0) {
     // return a dictionary only if we actually used it
     try {
       CapacityByteArrayOutputStream dictBuf =
           new CapacityByteArrayOutputStream(lastUsedDictionaryByteSize);
       LittleEndianDataOutputStream dictOut = new LittleEndianDataOutputStream(dictBuf);
       Iterator<Binary> entryIterator = dict.keySet().iterator();
       // write only the part of the dict that we used
       for (int i = 0; i < lastUsedDictionarySize; i++) {
         Binary entry = entryIterator.next();
         dictOut.writeInt(entry.length());
         entry.writeTo(dictOut);
       }
       return new DictionaryPage(
           BytesInput.from(dictBuf), lastUsedDictionarySize, PLAIN_DICTIONARY);
     } catch (IOException e) {
       throw new ParquetEncodingException("Could not generate dictionary Page", e);
     }
   }
   return plainValuesWriter.createDictionaryPage();
 }
 private Writable createPrimitive(final Object obj, final PrimitiveObjectInspector inspector)
     throws SerDeException {
   if (obj == null) {
     return null;
   }
   switch (inspector.getPrimitiveCategory()) {
     case VOID:
       return null;
     case BOOLEAN:
       return new BooleanWritable(
           ((BooleanObjectInspector) inspector).get(obj) ? Boolean.TRUE : Boolean.FALSE);
     case BYTE:
       return new ByteWritable(((ByteObjectInspector) inspector).get(obj));
     case DOUBLE:
       return new DoubleWritable(((DoubleObjectInspector) inspector).get(obj));
     case FLOAT:
       return new FloatWritable(((FloatObjectInspector) inspector).get(obj));
     case INT:
       return new IntWritable(((IntObjectInspector) inspector).get(obj));
     case LONG:
       return new LongWritable(((LongObjectInspector) inspector).get(obj));
     case SHORT:
       return new ShortWritable(((ShortObjectInspector) inspector).get(obj));
     case STRING:
       String v = ((StringObjectInspector) inspector).getPrimitiveJavaObject(obj);
       try {
         return new BytesWritable(v.getBytes("UTF-8"));
       } catch (UnsupportedEncodingException e) {
         throw new SerDeException("Failed to encode string in UTF-8", e);
       }
     case DECIMAL:
       HiveDecimal hd = (HiveDecimal) inspector.getPrimitiveJavaObject(obj);
       DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) inspector.getTypeInfo();
       int prec = decTypeInfo.precision();
       int scale = decTypeInfo.scale();
       byte[] src = hd.setScale(scale).unscaledValue().toByteArray();
       // Estimated number of bytes needed.
       int bytes = PRECISION_TO_BYTE_COUNT[prec - 1];
       if (bytes == src.length) {
         // No padding needed.
         return new BytesWritable(src);
       }
       byte[] tgt = new byte[bytes];
       if (hd.signum() == -1) {
         // For negative number, initializing bits to 1
         for (int i = 0; i < bytes; i++) {
           tgt[i] |= 0xFF;
         }
       }
       System.arraycopy(
           src, 0, tgt, bytes - src.length, src.length); // Padding leading zeroes/ones.
       return new BytesWritable(tgt);
     case TIMESTAMP:
       return new TimestampWritable(
           ((TimestampObjectInspector) inspector).getPrimitiveJavaObject(obj));
     case CHAR:
       String strippedValue =
           ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(obj).getStrippedValue();
       return new BytesWritable(Binary.fromString(strippedValue).getBytes());
     case VARCHAR:
       String value =
           ((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(obj).getValue();
       return new BytesWritable(Binary.fromString(value).getBytes());
     case BINARY:
       return new BytesWritable(((BinaryObjectInspector) inspector).getPrimitiveJavaObject(obj));
     default:
       throw new SerDeException("Unknown primitive : " + inspector.getPrimitiveCategory());
   }
 }
 private void addString(String value) {
   inOrder.verify(mockRecordConsumer).addBinary(Binary.fromString(value));
 }
 @Override
 public void addBinary(Binary value) {
   nulls[fieldIndex] = false;
   slices[fieldIndex] = Slices.wrappedBuffer(value.getBytes());
 }
 @Override
 public void addBinary(Binary value) {
   VARBINARY.writeSlice(builder, Slices.wrappedBuffer(value.getBytes()));
   wroteValue = true;
 }
示例#10
0
 @Override
 public final void addBinary(Binary value) {
   parent.add(new BlobDatum(ByteBuffer.wrap(value.getBytes())));
 }
示例#11
0
 @Override
 public final void addBinary(Binary value) {
   parent.add(DatumFactory.createText(value.getBytes()));
 }