private void parseColumn(int column) { Type type = types[column]; if (BOOLEAN.equals(type)) { parseBooleanColumn(column); } else if (BIGINT.equals(type)) { parseLongColumn(column); } else if (INTEGER.equals(type)) { parseLongColumn(column); } else if (SMALLINT.equals(type)) { parseLongColumn(column); } else if (TINYINT.equals(type)) { parseLongColumn(column); } else if (DOUBLE.equals(type)) { parseDoubleColumn(column); } else if (isVarcharType(type) || VARBINARY.equals(type)) { parseStringColumn(column); } else if (isStructuralType(hiveTypes[column])) { parseObjectColumn(column); } else if (DATE.equals(type)) { parseLongColumn(column); } else if (TIMESTAMP.equals(type)) { parseLongColumn(column); } else if (type instanceof DecimalType) { parseDecimalColumn(column); } else { throw new UnsupportedOperationException("Unsupported column type: " + type); } }
public GenericHiveRecordCursor( RecordReader<K, V> recordReader, long totalBytes, Properties splitSchema, List<HivePartitionKey> partitionKeys, List<HiveColumnHandle> columns, DateTimeZone hiveStorageTimeZone, TypeManager typeManager) { requireNonNull(recordReader, "recordReader is null"); checkArgument(totalBytes >= 0, "totalBytes is negative"); requireNonNull(splitSchema, "splitSchema is null"); requireNonNull(partitionKeys, "partitionKeys is null"); requireNonNull(columns, "columns is null"); requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null"); this.recordReader = recordReader; this.totalBytes = totalBytes; this.key = recordReader.createKey(); this.value = recordReader.createValue(); this.hiveStorageTimeZone = hiveStorageTimeZone; this.deserializer = getDeserializer(splitSchema); this.rowInspector = getTableObjectInspector(deserializer); int size = columns.size(); String[] names = new String[size]; this.types = new Type[size]; this.hiveTypes = new HiveType[size]; this.structFields = new StructField[size]; this.fieldInspectors = new ObjectInspector[size]; this.isPartitionColumn = new boolean[size]; this.loaded = new boolean[size]; this.booleans = new boolean[size]; this.longs = new long[size]; this.doubles = new double[size]; this.slices = new Slice[size]; this.objects = new Object[size]; this.nulls = new boolean[size]; // initialize data columns for (int i = 0; i < columns.size(); i++) { HiveColumnHandle column = columns.get(i); names[i] = column.getName(); types[i] = typeManager.getType(column.getTypeSignature()); hiveTypes[i] = column.getHiveType(); if (!column.isPartitionKey()) { StructField field = rowInspector.getStructFieldRef(column.getName()); structFields[i] = field; fieldInspectors[i] = field.getFieldObjectInspector(); } isPartitionColumn[i] = column.isPartitionKey(); } // parse requested partition columns Map<String, HivePartitionKey> partitionKeysByName = uniqueIndex(partitionKeys, HivePartitionKey::getName); for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) { HiveColumnHandle column = columns.get(columnIndex); if (column.isPartitionKey()) { HivePartitionKey partitionKey = partitionKeysByName.get(column.getName()); checkArgument(partitionKey != null, "Unknown partition key %s", column.getName()); byte[] bytes = partitionKey.getValue().getBytes(UTF_8); String name = names[columnIndex]; Type type = types[columnIndex]; if (HiveUtil.isHiveNull(bytes)) { nulls[columnIndex] = true; } else if (BOOLEAN.equals(type)) { booleans[columnIndex] = booleanPartitionKey(partitionKey.getValue(), name); } else if (BIGINT.equals(type)) { longs[columnIndex] = bigintPartitionKey(partitionKey.getValue(), name); } else if (INTEGER.equals(type)) { longs[columnIndex] = integerPartitionKey(partitionKey.getValue(), name); } else if (SMALLINT.equals(type)) { longs[columnIndex] = smallintPartitionKey(partitionKey.getValue(), name); } else if (TINYINT.equals(type)) { longs[columnIndex] = tinyintPartitionKey(partitionKey.getValue(), name); } else if (DOUBLE.equals(type)) { doubles[columnIndex] = doublePartitionKey(partitionKey.getValue(), name); } else if (isVarcharType(type)) { slices[columnIndex] = varcharPartitionKey(partitionKey.getValue(), name, type); } else if (DATE.equals(type)) { longs[columnIndex] = datePartitionKey(partitionKey.getValue(), name); } else if (TIMESTAMP.equals(type)) { longs[columnIndex] = timestampPartitionKey(partitionKey.getValue(), hiveStorageTimeZone, name); } else if (isShortDecimal(type)) { longs[columnIndex] = shortDecimalPartitionKey(partitionKey.getValue(), (DecimalType) type, name); } else if (isLongDecimal(type)) { slices[columnIndex] = longDecimalPartitionKey(partitionKey.getValue(), (DecimalType) type, name); } else { throw new PrestoException( NOT_SUPPORTED, format( "Unsupported column type %s for partition key: %s", type.getDisplayName(), name)); } } } }
public static NullableValue parsePartitionValue( String partitionName, String value, Type type, DateTimeZone timeZone) { boolean isNull = HIVE_DEFAULT_DYNAMIC_PARTITION.equals(value); if (type instanceof DecimalType) { DecimalType decimalType = (DecimalType) type; if (isNull) { return NullableValue.asNull(decimalType); } if (decimalType.isShort()) { if (value.isEmpty()) { return NullableValue.of(decimalType, 0L); } return NullableValue.of( decimalType, shortDecimalPartitionKey(value, decimalType, partitionName)); } else { if (value.isEmpty()) { return NullableValue.of(decimalType, Decimals.encodeUnscaledValue(BigInteger.ZERO)); } return NullableValue.of( decimalType, longDecimalPartitionKey(value, decimalType, partitionName)); } } if (BOOLEAN.equals(type)) { if (isNull) { return NullableValue.asNull(BOOLEAN); } if (value.isEmpty()) { return NullableValue.of(BOOLEAN, false); } return NullableValue.of(BOOLEAN, booleanPartitionKey(value, partitionName)); } if (TINYINT.equals(type)) { if (isNull) { return NullableValue.asNull(TINYINT); } if (value.isEmpty()) { return NullableValue.of(TINYINT, 0L); } return NullableValue.of(TINYINT, tinyintPartitionKey(value, partitionName)); } if (SMALLINT.equals(type)) { if (isNull) { return NullableValue.asNull(SMALLINT); } if (value.isEmpty()) { return NullableValue.of(SMALLINT, 0L); } return NullableValue.of(SMALLINT, smallintPartitionKey(value, partitionName)); } if (INTEGER.equals(type)) { if (isNull) { return NullableValue.asNull(INTEGER); } if (value.isEmpty()) { return NullableValue.of(INTEGER, 0L); } return NullableValue.of(INTEGER, integerPartitionKey(value, partitionName)); } if (BIGINT.equals(type)) { if (isNull) { return NullableValue.asNull(BIGINT); } if (value.isEmpty()) { return NullableValue.of(BIGINT, 0L); } return NullableValue.of(BIGINT, bigintPartitionKey(value, partitionName)); } if (DATE.equals(type)) { if (isNull) { return NullableValue.asNull(DATE); } return NullableValue.of(DATE, datePartitionKey(value, partitionName)); } if (TIMESTAMP.equals(type)) { if (isNull) { return NullableValue.asNull(TIMESTAMP); } return NullableValue.of(TIMESTAMP, timestampPartitionKey(value, timeZone, partitionName)); } if (REAL.equals(type)) { if (isNull) { return NullableValue.asNull(REAL); } if (value.isEmpty()) { return NullableValue.of(REAL, (long) floatToRawIntBits(0.0f)); } return NullableValue.of(REAL, floatPartitionKey(value, partitionName)); } if (DOUBLE.equals(type)) { if (isNull) { return NullableValue.asNull(DOUBLE); } if (value.isEmpty()) { return NullableValue.of(DOUBLE, 0.0); } return NullableValue.of(DOUBLE, doublePartitionKey(value, partitionName)); } if (type instanceof VarcharType) { if (isNull) { return NullableValue.asNull(type); } return NullableValue.of(type, varcharPartitionKey(value, partitionName, type)); } if (isCharType(type)) { if (isNull) { return NullableValue.asNull(type); } return NullableValue.of(type, charPartitionKey(value, partitionName, type)); } throw new PrestoException( NOT_SUPPORTED, format("Unsupported Type [%s] for partition: %s", type, partitionName)); }