private String nodeToString(Object node) {
if (BOOLEAN.equals(getNodeQName())) {
return (Boolean) node ? "1" : "0";
}
return node.toString().trim();
}
private static void assertReadFields(RecordCursor cursor, List<ColumnMetadata> schema) {
for (int columnIndex = 0; columnIndex < schema.size(); columnIndex++) {
ColumnMetadata column = schema.get(columnIndex);
if (!cursor.isNull(columnIndex)) {
Type type = column.getType();
if (BOOLEAN.equals(type)) {
cursor.getBoolean(columnIndex);
} else if (BIGINT.equals(type)) {
cursor.getLong(columnIndex);
} else if (TIMESTAMP.equals(type)) {
cursor.getLong(columnIndex);
} else if (DOUBLE.equals(type)) {
cursor.getDouble(columnIndex);
} else if (VARCHAR.equals(type) || VARBINARY.equals(type)) {
try {
cursor.getSlice(columnIndex);
} catch (RuntimeException e) {
throw new RuntimeException("column " + column, e);
}
} else {
fail("Unknown primitive type " + columnIndex);
}
}
}
}
private void parseColumn(int column) {
Type type = types[column];
if (BOOLEAN.equals(type)) {
parseBooleanColumn(column);
} else if (BIGINT.equals(type)) {
parseLongColumn(column);
} else if (INTEGER.equals(type)) {
parseLongColumn(column);
} else if (SMALLINT.equals(type)) {
parseLongColumn(column);
} else if (TINYINT.equals(type)) {
parseLongColumn(column);
} else if (DOUBLE.equals(type)) {
parseDoubleColumn(column);
} else if (isVarcharType(type) || VARBINARY.equals(type)) {
parseStringColumn(column);
} else if (isStructuralType(hiveTypes[column])) {
parseObjectColumn(column);
} else if (DATE.equals(type)) {
parseLongColumn(column);
} else if (TIMESTAMP.equals(type)) {
parseLongColumn(column);
} else if (type instanceof DecimalType) {
parseDecimalColumn(column);
} else {
throw new UnsupportedOperationException("Unsupported column type: " + type);
}
}
protected void checkCursor(RecordCursor cursor, List<TestColumn> testColumns, int numRows)
throws IOException {
for (int row = 0; row < numRows; row++) {
assertTrue(cursor.advanceNextPosition());
for (int i = 0, testColumnsSize = testColumns.size(); i < testColumnsSize; i++) {
TestColumn testColumn = testColumns.get(i);
Object fieldFromCursor;
Type type =
HiveType.valueOf(testColumn.getObjectInspector().getTypeName()).getType(TYPE_MANAGER);
if (cursor.isNull(i)) {
fieldFromCursor = null;
} else if (BOOLEAN.equals(type)) {
fieldFromCursor = cursor.getBoolean(i);
} else if (BIGINT.equals(type)) {
fieldFromCursor = cursor.getLong(i);
} else if (DOUBLE.equals(type)) {
fieldFromCursor = cursor.getDouble(i);
} else if (VARCHAR.equals(type)) {
fieldFromCursor = cursor.getSlice(i);
} else if (VARBINARY.equals(type)) {
fieldFromCursor = cursor.getSlice(i);
} else if (DateType.DATE.equals(type)) {
fieldFromCursor = cursor.getLong(i);
} else if (TimestampType.TIMESTAMP.equals(type)) {
fieldFromCursor = cursor.getLong(i);
} else if (isStructuralType(type)) {
fieldFromCursor = cursor.getObject(i);
} else {
throw new RuntimeException("unknown type");
}
if (fieldFromCursor == null) {
assertEquals(
null,
testColumn.getExpectedValue(),
String.format("Expected null for column %s", testColumn.getName()));
} else if (testColumn.getObjectInspector().getTypeName().equals("float")
|| testColumn.getObjectInspector().getTypeName().equals("double")) {
assertEquals((double) fieldFromCursor, (double) testColumn.getExpectedValue(), EPSILON);
} else if (testColumn.getObjectInspector().getCategory() == Category.PRIMITIVE) {
assertEquals(
fieldFromCursor,
testColumn.getExpectedValue(),
String.format("Wrong value for column %s", testColumn.getName()));
} else {
Block expected = (Block) testColumn.getExpectedValue();
Block actual = (Block) fieldFromCursor;
assertBlockEquals(
actual, expected, String.format("Wrong value for column %s", testColumn.getName()));
}
}
}
}
@Override
public void addInput(Page page) {
requireNonNull(page, "page is null");
checkState(!finishing, "Operator is finishing");
checkState(channelSet != null, "Set has not been built yet");
checkState(outputPage == null, "Operator still has pending output");
// create the block builder for the new boolean column
// we know the exact size required for the block
BlockBuilder blockBuilder = BOOLEAN.createFixedSizeBlockBuilder(page.getPositionCount());
Page probeJoinPage = new Page(page.getBlock(probeJoinChannel));
// update hashing strategy to use probe cursor
for (int position = 0; position < page.getPositionCount(); position++) {
if (probeJoinPage.getBlock(0).isNull(position)) {
throw new PrestoException(
NOT_SUPPORTED,
"NULL values are not allowed on the probe side of SemiJoin operator. See the query plan for details.");
} else {
boolean contains = channelSet.contains(position, probeJoinPage);
if (!contains && channelSet.containsNull()) {
blockBuilder.appendNull();
} else {
BOOLEAN.writeBoolean(blockBuilder, contains);
}
}
}
// add the new boolean column to the page
Block[] sourceBlocks = page.getBlocks();
Block[] outputBlocks =
new Block[sourceBlocks.length + 1]; // +1 for the single boolean output channel
System.arraycopy(sourceBlocks, 0, outputBlocks, 0, sourceBlocks.length);
outputBlocks[sourceBlocks.length] = blockBuilder.build();
outputPage = new Page(outputBlocks);
}
private static void insertRows(
ConnectorTableMetadata tableMetadata, Handle handle, RecordSet data) {
List<ColumnMetadata> columns =
ImmutableList.copyOf(
Iterables.filter(
tableMetadata.getColumns(),
new Predicate<ColumnMetadata>() {
@Override
public boolean apply(ColumnMetadata columnMetadata) {
return !columnMetadata.isHidden();
}
}));
String vars = Joiner.on(',').join(nCopies(columns.size(), "?"));
String sql =
format("INSERT INTO %s VALUES (%s)", tableMetadata.getTable().getTableName(), vars);
RecordCursor cursor = data.cursor();
while (true) {
// insert 1000 rows at a time
PreparedBatch batch = handle.prepareBatch(sql);
for (int row = 0; row < 1000; row++) {
if (!cursor.advanceNextPosition()) {
batch.execute();
return;
}
PreparedBatchPart part = batch.add();
for (int column = 0; column < columns.size(); column++) {
Type type = columns.get(column).getType();
if (BOOLEAN.equals(type)) {
part.bind(column, cursor.getBoolean(column));
} else if (BIGINT.equals(type)) {
part.bind(column, cursor.getLong(column));
} else if (DOUBLE.equals(type)) {
part.bind(column, cursor.getDouble(column));
} else if (VARCHAR.equals(type)) {
part.bind(column, cursor.getSlice(column).toStringUtf8());
} else if (DATE.equals(type)) {
long millisUtc = TimeUnit.DAYS.toMillis(cursor.getLong(column));
// H2 expects dates in to be millis at midnight in the JVM timezone
long localMillis =
DateTimeZone.UTC.getMillisKeepLocal(DateTimeZone.getDefault(), millisUtc);
part.bind(column, new Date(localMillis));
} else {
throw new IllegalArgumentException("Unsupported type " + type);
}
}
}
batch.execute();
}
}
public static PropertyDataTypes getTypeByTypeNum(int typeNum) {
if (STRING.getTypeNum() == typeNum) return STRING;
else if (BINARY.getTypeNum() == typeNum) return BINARY;
else if (LONG.getTypeNum() == typeNum) return LONG;
else if (DOUBLE.getTypeNum() == typeNum) return DOUBLE;
else if (DATE.getTypeNum() == typeNum) return DATE;
else if (BOOLEAN.getTypeNum() == typeNum) return BOOLEAN;
else if (NAME.getTypeNum() == typeNum) return NAME;
else if (PATH.getTypeNum() == typeNum) return PATH;
else if (REFERENCE.getTypeNum() == typeNum) return REFERENCE;
else if (WEAKREFERENCE.getTypeNum() == typeNum) return WEAKREFERENCE;
else if (URI.getTypeNum() == typeNum) return URI;
else if (DECIMAL.getTypeNum() == typeNum) return DECIMAL;
else return UNDEFINED;
}
private void parseColumn(int column) {
Type type = types[column];
if (BOOLEAN.equals(type)) {
parseBooleanColumn(column);
} else if (BIGINT.equals(type)) {
parseLongColumn(column);
} else if (DOUBLE.equals(type)) {
parseDoubleColumn(column);
} else if (VARCHAR.equals(type) || VARBINARY.equals(type)) {
parseStringColumn(column);
} else if (TIMESTAMP.equals(type)) {
parseLongColumn(column);
} else {
throw new UnsupportedOperationException("Unsupported column type: " + type);
}
}
@Override
public boolean readNextValueInto(BlockBuilder builder, boolean skipNull) throws IOException {
if (presentStream != null && !presentStream.nextBit()) {
if (!skipNull) {
builder.appendNull();
return true;
}
return false;
}
if (dataStream == null) {
throw new OrcCorruptionException("Value is not null but data stream is not present");
}
BOOLEAN.writeBoolean(builder, dataStream.nextBit());
return true;
}
@Override
public Object convert(Object value) {
if (!(value instanceof String)) {
return value;
}
try {
return BOOLEAN.convert(value);
} catch (IllegalArgumentException e) {
}
try {
return INTEGER.convert(value);
} catch (IllegalArgumentException e) {
}
try {
return FLOAT.convert(value);
} catch (IllegalArgumentException e) {
}
return value;
}
private String insertOptionalNonStandardCharacterFrom(
final Character[] nonStandardChars, final String generated) {
String resultString = generated;
final boolean includeNonStandardChars = BOOLEAN.next();
if (includeNonStandardChars) {
final int positionToInsert = RANDOM.nextInt(generated.length());
String charToInsert = valueOf(values(nonStandardChars).next());
if (NONSTANDARD_CHARACTERS_TO_HAVE_BACKSLASH.contains(charToInsert)) {
charToInsert = format("\\%s", charToInsert);
}
resultString =
format(
"\"%s%s%s\"",
generated.substring(0, positionToInsert),
charToInsert,
generated.substring(positionToInsert, generated.length()));
}
return resultString;
}
public GenericHiveRecordCursor(
RecordReader<K, V> recordReader,
long totalBytes,
Properties splitSchema,
List<HivePartitionKey> partitionKeys,
List<HiveColumnHandle> columns,
DateTimeZone hiveStorageTimeZone,
DateTimeZone sessionTimeZone) {
checkNotNull(recordReader, "recordReader is null");
checkArgument(totalBytes >= 0, "totalBytes is negative");
checkNotNull(splitSchema, "splitSchema is null");
checkNotNull(partitionKeys, "partitionKeys is null");
checkNotNull(columns, "columns is null");
checkArgument(!columns.isEmpty(), "columns is empty");
checkNotNull(hiveStorageTimeZone, "hiveStorageTimeZone is null");
checkNotNull(sessionTimeZone, "sessionTimeZone is null");
this.recordReader = recordReader;
this.totalBytes = totalBytes;
this.key = recordReader.createKey();
this.value = recordReader.createValue();
this.hiveStorageTimeZone = hiveStorageTimeZone;
this.sessionTimeZone = sessionTimeZone;
this.deserializer = getDeserializer(splitSchema);
this.rowInspector = getTableObjectInspector(deserializer);
int size = columns.size();
String[] names = new String[size];
this.types = new Type[size];
this.hiveTypes = new HiveType[size];
this.structFields = new StructField[size];
this.fieldInspectors = new ObjectInspector[size];
this.isPartitionColumn = new boolean[size];
this.loaded = new boolean[size];
this.booleans = new boolean[size];
this.longs = new long[size];
this.doubles = new double[size];
this.slices = new Slice[size];
this.nulls = new boolean[size];
// initialize data columns
for (int i = 0; i < columns.size(); i++) {
HiveColumnHandle column = columns.get(i);
names[i] = column.getName();
types[i] = column.getType();
hiveTypes[i] = column.getHiveType();
if (!column.isPartitionKey()) {
StructField field = rowInspector.getStructFieldRef(column.getName());
structFields[i] = field;
fieldInspectors[i] = field.getFieldObjectInspector();
}
isPartitionColumn[i] = column.isPartitionKey();
}
// parse requested partition columns
Map<String, HivePartitionKey> partitionKeysByName =
uniqueIndex(partitionKeys, HivePartitionKey.nameGetter());
for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) {
HiveColumnHandle column = columns.get(columnIndex);
if (column.isPartitionKey()) {
HivePartitionKey partitionKey = partitionKeysByName.get(column.getName());
checkArgument(partitionKey != null, "Unknown partition key %s", column.getName());
byte[] bytes = partitionKey.getValue().getBytes(Charsets.UTF_8);
Type type = types[columnIndex];
if (BOOLEAN.equals(type)) {
if (isTrue(bytes, 0, bytes.length)) {
booleans[columnIndex] = true;
} else if (isFalse(bytes, 0, bytes.length)) {
booleans[columnIndex] = false;
} else {
String valueString = new String(bytes, Charsets.UTF_8);
throw new IllegalArgumentException(
String.format(
"Invalid partition value '%s' for BOOLEAN partition key %s",
valueString, names[columnIndex]));
}
} else if (BIGINT.equals(type)) {
if (bytes.length == 0) {
throw new IllegalArgumentException(
String.format(
"Invalid partition value '' for BIGINT partition key %s", names[columnIndex]));
}
longs[columnIndex] = parseLong(bytes, 0, bytes.length);
} else if (DOUBLE.equals(type)) {
if (bytes.length == 0) {
throw new IllegalArgumentException(
String.format(
"Invalid partition value '' for DOUBLE partition key %s", names[columnIndex]));
}
doubles[columnIndex] = parseDouble(bytes, 0, bytes.length);
} else if (VARCHAR.equals(type)) {
slices[columnIndex] = Slices.wrappedBuffer(Arrays.copyOf(bytes, bytes.length));
} else {
throw new UnsupportedOperationException("Unsupported column type: " + type);
}
}
}
}
@Test
public void testWriter() throws Exception {
List<Long> columnIds = ImmutableList.of(1L, 2L, 4L, 6L, 7L, 8L, 9L, 10L);
ArrayType arrayType = new ArrayType(BIGINT);
ArrayType arrayOfArrayType = new ArrayType(arrayType);
MapType mapType = new MapType(createVarcharType(10), BOOLEAN);
List<Type> columnTypes =
ImmutableList.of(
BIGINT,
createVarcharType(10),
VARBINARY,
DOUBLE,
BOOLEAN,
arrayType,
mapType,
arrayOfArrayType);
File file = new File(directory, System.nanoTime() + ".orc");
byte[] bytes1 = octets(0x00, 0xFE, 0xFF);
byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80);
RowPagesBuilder rowPagesBuilder =
RowPagesBuilder.rowPagesBuilder(columnTypes)
.row(
123L,
"hello",
wrappedBuffer(bytes1),
123.456,
true,
arrayBlockOf(BIGINT, 1, 2),
mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true),
arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)))
.row(
null,
"world",
null,
Double.POSITIVE_INFINITY,
null,
arrayBlockOf(BIGINT, 3, null),
mapBlockOf(createVarcharType(5), BOOLEAN, "k2", null),
arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7)))
.row(
456L,
"bye \u2603",
wrappedBuffer(bytes3),
Double.NaN,
false,
arrayBlockOf(BIGINT),
mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false),
arrayBlockOf(arrayType, arrayBlockOf(BIGINT)));
try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader());
OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) {
writer.appendPages(rowPagesBuilder.build());
}
try (OrcDataSource dataSource = fileOrcDataSource(file)) {
OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes);
assertEquals(reader.getReaderRowCount(), 3);
assertEquals(reader.getReaderPosition(), 0);
assertEquals(reader.getFileRowCount(), reader.getReaderRowCount());
assertEquals(reader.getFilePosition(), reader.getFilePosition());
assertEquals(reader.nextBatch(), 3);
assertEquals(reader.getReaderPosition(), 0);
assertEquals(reader.getFilePosition(), reader.getFilePosition());
Block column0 = reader.readBlock(BIGINT, 0);
assertEquals(column0.isNull(0), false);
assertEquals(column0.isNull(1), true);
assertEquals(column0.isNull(2), false);
assertEquals(BIGINT.getLong(column0, 0), 123L);
assertEquals(BIGINT.getLong(column0, 2), 456L);
Block column1 = reader.readBlock(createVarcharType(10), 1);
assertEquals(createVarcharType(10).getSlice(column1, 0), utf8Slice("hello"));
assertEquals(createVarcharType(10).getSlice(column1, 1), utf8Slice("world"));
assertEquals(createVarcharType(10).getSlice(column1, 2), utf8Slice("bye \u2603"));
Block column2 = reader.readBlock(VARBINARY, 2);
assertEquals(VARBINARY.getSlice(column2, 0), wrappedBuffer(bytes1));
assertEquals(column2.isNull(1), true);
assertEquals(VARBINARY.getSlice(column2, 2), wrappedBuffer(bytes3));
Block column3 = reader.readBlock(DOUBLE, 3);
assertEquals(column3.isNull(0), false);
assertEquals(column3.isNull(1), false);
assertEquals(column3.isNull(2), false);
assertEquals(DOUBLE.getDouble(column3, 0), 123.456);
assertEquals(DOUBLE.getDouble(column3, 1), Double.POSITIVE_INFINITY);
assertEquals(DOUBLE.getDouble(column3, 2), Double.NaN);
Block column4 = reader.readBlock(BOOLEAN, 4);
assertEquals(column4.isNull(0), false);
assertEquals(column4.isNull(1), true);
assertEquals(column4.isNull(2), false);
assertEquals(BOOLEAN.getBoolean(column4, 0), true);
assertEquals(BOOLEAN.getBoolean(column4, 2), false);
Block column5 = reader.readBlock(arrayType, 5);
assertEquals(column5.getPositionCount(), 3);
assertTrue(
arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 0), arrayBlockOf(BIGINT, 1, 2)));
assertTrue(
arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 1), arrayBlockOf(BIGINT, 3, null)));
assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 2), arrayBlockOf(BIGINT)));
Block column6 = reader.readBlock(mapType, 6);
assertEquals(column6.getPositionCount(), 3);
assertTrue(
mapBlocksEqual(
createVarcharType(5),
BOOLEAN,
arrayType.getObject(column6, 0),
mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true)));
assertTrue(
mapBlocksEqual(
createVarcharType(5),
BOOLEAN,
arrayType.getObject(column6, 1),
mapBlockOf(createVarcharType(5), BOOLEAN, "k2", null)));
assertTrue(
mapBlocksEqual(
createVarcharType(5),
BOOLEAN,
arrayType.getObject(column6, 2),
mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false)));
Block column7 = reader.readBlock(arrayOfArrayType, 7);
assertEquals(column7.getPositionCount(), 3);
assertTrue(
arrayBlocksEqual(
arrayType,
arrayOfArrayType.getObject(column7, 0),
arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5))));
assertTrue(
arrayBlocksEqual(
arrayType,
arrayOfArrayType.getObject(column7, 1),
arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7))));
assertTrue(
arrayBlocksEqual(
arrayType,
arrayOfArrayType.getObject(column7, 2),
arrayBlockOf(arrayType, arrayBlockOf(BIGINT))));
assertEquals(reader.nextBatch(), -1);
assertEquals(reader.getReaderPosition(), 3);
assertEquals(reader.getFilePosition(), reader.getFilePosition());
OrcFileMetadata orcFileMetadata =
METADATA_CODEC.fromJson(reader.getUserMetadata().get(OrcFileMetadata.KEY).getBytes());
assertEquals(
orcFileMetadata,
new OrcFileMetadata(
ImmutableMap.<Long, TypeSignature>builder()
.put(1L, BIGINT.getTypeSignature())
.put(2L, createVarcharType(10).getTypeSignature())
.put(4L, VARBINARY.getTypeSignature())
.put(6L, DOUBLE.getTypeSignature())
.put(7L, BOOLEAN.getTypeSignature())
.put(8L, arrayType.getTypeSignature())
.put(9L, mapType.getTypeSignature())
.put(10L, arrayOfArrayType.getTypeSignature())
.build()));
}
File crcFile = new File(file.getParentFile(), "." + file.getName() + ".crc");
assertFalse(crcFile.exists());
}
@Override
public void addBoolean(boolean value) {
BOOLEAN.writeBoolean(builder, value);
wroteValue = true;
}
@Override
public void readFields(DataInput in) throws IOException {
/*
* extract pkt len.
*
* GPSQL-1107:
* The DataInput might already be empty (EOF), but we can't check it beforehand.
* If that's the case, pktlen is updated to -1, to mark that the object is still empty.
* (can be checked with isEmpty()).
*/
pktlen = readPktLen(in);
if (isEmpty()) {
return;
}
/* extract the version and col cnt */
int version = in.readShort();
int curOffset = 4 + 2;
int colCnt;
/* !!! Check VERSION !!! */
if (version != GPDBWritable.VERSION && version != GPDBWritable.PREV_VERSION) {
throw new IOException(
"Current GPDBWritable version("
+ GPDBWritable.VERSION
+ ") does not match input version("
+ version
+ ")");
}
if (version == GPDBWritable.VERSION) {
errorFlag = in.readByte();
curOffset += 1;
}
colCnt = in.readShort();
curOffset += 2;
/* Extract Column Type */
colType = new int[colCnt];
DBType[] coldbtype = new DBType[colCnt];
for (int i = 0; i < colCnt; i++) {
int enumType = (in.readByte());
curOffset += 1;
if (enumType == DBType.BIGINT.ordinal()) {
colType[i] = BIGINT.getOID();
coldbtype[i] = DBType.BIGINT;
} else if (enumType == DBType.BOOLEAN.ordinal()) {
colType[i] = BOOLEAN.getOID();
coldbtype[i] = DBType.BOOLEAN;
} else if (enumType == DBType.FLOAT8.ordinal()) {
colType[i] = FLOAT8.getOID();
coldbtype[i] = DBType.FLOAT8;
} else if (enumType == DBType.INTEGER.ordinal()) {
colType[i] = INTEGER.getOID();
coldbtype[i] = DBType.INTEGER;
} else if (enumType == DBType.REAL.ordinal()) {
colType[i] = REAL.getOID();
coldbtype[i] = DBType.REAL;
} else if (enumType == DBType.SMALLINT.ordinal()) {
colType[i] = SMALLINT.getOID();
coldbtype[i] = DBType.SMALLINT;
} else if (enumType == DBType.BYTEA.ordinal()) {
colType[i] = BYTEA.getOID();
coldbtype[i] = DBType.BYTEA;
} else if (enumType == DBType.TEXT.ordinal()) {
colType[i] = TEXT.getOID();
coldbtype[i] = DBType.TEXT;
} else {
throw new IOException("Unknown GPDBWritable.DBType ordinal value");
}
}
/* Extract null bit array */
byte[] nullbytes = new byte[getNullByteArraySize(colCnt)];
in.readFully(nullbytes);
curOffset += nullbytes.length;
boolean[] colIsNull = byteArrayToBooleanArray(nullbytes, colCnt);
/* extract column value */
colValue = new Object[colCnt];
for (int i = 0; i < colCnt; i++) {
if (!colIsNull[i]) {
/* Skip the alignment padding */
int skipbytes = roundUpAlignment(curOffset, coldbtype[i].getAlignment()) - curOffset;
for (int j = 0; j < skipbytes; j++) {
in.readByte();
}
curOffset += skipbytes;
/* For fixed length type, increment the offset according to type type length here.
* For var length type (BYTEA, TEXT), we'll read 4 byte length header and the
* actual payload.
*/
int varcollen = -1;
if (coldbtype[i].isVarLength()) {
varcollen = in.readInt();
curOffset += 4 + varcollen;
} else {
curOffset += coldbtype[i].getTypeLength();
}
switch (DataType.get(colType[i])) {
case BIGINT:
{
colValue[i] = in.readLong();
break;
}
case BOOLEAN:
{
colValue[i] = in.readBoolean();
break;
}
case FLOAT8:
{
colValue[i] = in.readDouble();
break;
}
case INTEGER:
{
colValue[i] = in.readInt();
break;
}
case REAL:
{
colValue[i] = in.readFloat();
break;
}
case SMALLINT:
{
colValue[i] = in.readShort();
break;
}
/* For BYTEA column, it has a 4 byte var length header. */
case BYTEA:
{
colValue[i] = new byte[varcollen];
in.readFully((byte[]) colValue[i]);
break;
}
/* For text formatted column, it has a 4 byte var length header
* and it's always null terminated string.
* So, we can remove the last "\0" when constructing the string.
*/
case TEXT:
{
byte[] data = new byte[varcollen];
in.readFully(data, 0, varcollen);
colValue[i] = new String(data, 0, varcollen - 1, CHARSET);
break;
}
default:
throw new IOException("Unknown GPDBWritable ColType");
}
}
}
/* Skip the ending alignment padding */
int skipbytes = roundUpAlignment(curOffset, 8) - curOffset;
for (int j = 0; j < skipbytes; j++) {
in.readByte();
}
curOffset += skipbytes;
if (errorFlag != 0) {
throw new IOException("Received error value " + errorFlag + " from format");
}
}
public GenericHiveRecordCursor(
RecordReader<K, V> recordReader,
long totalBytes,
Properties splitSchema,
List<HivePartitionKey> partitionKeys,
List<HiveColumnHandle> columns,
DateTimeZone hiveStorageTimeZone,
TypeManager typeManager) {
requireNonNull(recordReader, "recordReader is null");
checkArgument(totalBytes >= 0, "totalBytes is negative");
requireNonNull(splitSchema, "splitSchema is null");
requireNonNull(partitionKeys, "partitionKeys is null");
requireNonNull(columns, "columns is null");
requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null");
this.recordReader = recordReader;
this.totalBytes = totalBytes;
this.key = recordReader.createKey();
this.value = recordReader.createValue();
this.hiveStorageTimeZone = hiveStorageTimeZone;
this.deserializer = getDeserializer(splitSchema);
this.rowInspector = getTableObjectInspector(deserializer);
int size = columns.size();
String[] names = new String[size];
this.types = new Type[size];
this.hiveTypes = new HiveType[size];
this.structFields = new StructField[size];
this.fieldInspectors = new ObjectInspector[size];
this.isPartitionColumn = new boolean[size];
this.loaded = new boolean[size];
this.booleans = new boolean[size];
this.longs = new long[size];
this.doubles = new double[size];
this.slices = new Slice[size];
this.objects = new Object[size];
this.nulls = new boolean[size];
// initialize data columns
for (int i = 0; i < columns.size(); i++) {
HiveColumnHandle column = columns.get(i);
names[i] = column.getName();
types[i] = typeManager.getType(column.getTypeSignature());
hiveTypes[i] = column.getHiveType();
if (!column.isPartitionKey()) {
StructField field = rowInspector.getStructFieldRef(column.getName());
structFields[i] = field;
fieldInspectors[i] = field.getFieldObjectInspector();
}
isPartitionColumn[i] = column.isPartitionKey();
}
// parse requested partition columns
Map<String, HivePartitionKey> partitionKeysByName =
uniqueIndex(partitionKeys, HivePartitionKey::getName);
for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) {
HiveColumnHandle column = columns.get(columnIndex);
if (column.isPartitionKey()) {
HivePartitionKey partitionKey = partitionKeysByName.get(column.getName());
checkArgument(partitionKey != null, "Unknown partition key %s", column.getName());
byte[] bytes = partitionKey.getValue().getBytes(UTF_8);
String name = names[columnIndex];
Type type = types[columnIndex];
if (HiveUtil.isHiveNull(bytes)) {
nulls[columnIndex] = true;
} else if (BOOLEAN.equals(type)) {
booleans[columnIndex] = booleanPartitionKey(partitionKey.getValue(), name);
} else if (BIGINT.equals(type)) {
longs[columnIndex] = bigintPartitionKey(partitionKey.getValue(), name);
} else if (INTEGER.equals(type)) {
longs[columnIndex] = integerPartitionKey(partitionKey.getValue(), name);
} else if (SMALLINT.equals(type)) {
longs[columnIndex] = smallintPartitionKey(partitionKey.getValue(), name);
} else if (TINYINT.equals(type)) {
longs[columnIndex] = tinyintPartitionKey(partitionKey.getValue(), name);
} else if (DOUBLE.equals(type)) {
doubles[columnIndex] = doublePartitionKey(partitionKey.getValue(), name);
} else if (isVarcharType(type)) {
slices[columnIndex] = varcharPartitionKey(partitionKey.getValue(), name, type);
} else if (DATE.equals(type)) {
longs[columnIndex] = datePartitionKey(partitionKey.getValue(), name);
} else if (TIMESTAMP.equals(type)) {
longs[columnIndex] =
timestampPartitionKey(partitionKey.getValue(), hiveStorageTimeZone, name);
} else if (isShortDecimal(type)) {
longs[columnIndex] =
shortDecimalPartitionKey(partitionKey.getValue(), (DecimalType) type, name);
} else if (isLongDecimal(type)) {
slices[columnIndex] =
longDecimalPartitionKey(partitionKey.getValue(), (DecimalType) type, name);
} else {
throw new PrestoException(
NOT_SUPPORTED,
format(
"Unsupported column type %s for partition key: %s", type.getDisplayName(), name));
}
}
}
}
/**
* Returns a Server variable type requested by name
*
* @param type of server variable
* @return Server Variable type
*/
public static ServerVariableTypeEnum getEnum(String type) {
if (type.equals(INT.name())) return INT;
if (type.equals(STRING.name())) return STRING;
if (type.equals(BOOLEAN.name())) return BOOLEAN;
return null;
}
public static NullableValue parsePartitionValue(
String partitionName, String value, Type type, DateTimeZone timeZone) {
boolean isNull = HIVE_DEFAULT_DYNAMIC_PARTITION.equals(value);
if (type instanceof DecimalType) {
DecimalType decimalType = (DecimalType) type;
if (isNull) {
return NullableValue.asNull(decimalType);
}
if (decimalType.isShort()) {
if (value.isEmpty()) {
return NullableValue.of(decimalType, 0L);
}
return NullableValue.of(
decimalType, shortDecimalPartitionKey(value, decimalType, partitionName));
} else {
if (value.isEmpty()) {
return NullableValue.of(decimalType, Decimals.encodeUnscaledValue(BigInteger.ZERO));
}
return NullableValue.of(
decimalType, longDecimalPartitionKey(value, decimalType, partitionName));
}
}
if (BOOLEAN.equals(type)) {
if (isNull) {
return NullableValue.asNull(BOOLEAN);
}
if (value.isEmpty()) {
return NullableValue.of(BOOLEAN, false);
}
return NullableValue.of(BOOLEAN, booleanPartitionKey(value, partitionName));
}
if (TINYINT.equals(type)) {
if (isNull) {
return NullableValue.asNull(TINYINT);
}
if (value.isEmpty()) {
return NullableValue.of(TINYINT, 0L);
}
return NullableValue.of(TINYINT, tinyintPartitionKey(value, partitionName));
}
if (SMALLINT.equals(type)) {
if (isNull) {
return NullableValue.asNull(SMALLINT);
}
if (value.isEmpty()) {
return NullableValue.of(SMALLINT, 0L);
}
return NullableValue.of(SMALLINT, smallintPartitionKey(value, partitionName));
}
if (INTEGER.equals(type)) {
if (isNull) {
return NullableValue.asNull(INTEGER);
}
if (value.isEmpty()) {
return NullableValue.of(INTEGER, 0L);
}
return NullableValue.of(INTEGER, integerPartitionKey(value, partitionName));
}
if (BIGINT.equals(type)) {
if (isNull) {
return NullableValue.asNull(BIGINT);
}
if (value.isEmpty()) {
return NullableValue.of(BIGINT, 0L);
}
return NullableValue.of(BIGINT, bigintPartitionKey(value, partitionName));
}
if (DATE.equals(type)) {
if (isNull) {
return NullableValue.asNull(DATE);
}
return NullableValue.of(DATE, datePartitionKey(value, partitionName));
}
if (TIMESTAMP.equals(type)) {
if (isNull) {
return NullableValue.asNull(TIMESTAMP);
}
return NullableValue.of(TIMESTAMP, timestampPartitionKey(value, timeZone, partitionName));
}
if (REAL.equals(type)) {
if (isNull) {
return NullableValue.asNull(REAL);
}
if (value.isEmpty()) {
return NullableValue.of(REAL, (long) floatToRawIntBits(0.0f));
}
return NullableValue.of(REAL, floatPartitionKey(value, partitionName));
}
if (DOUBLE.equals(type)) {
if (isNull) {
return NullableValue.asNull(DOUBLE);
}
if (value.isEmpty()) {
return NullableValue.of(DOUBLE, 0.0);
}
return NullableValue.of(DOUBLE, doublePartitionKey(value, partitionName));
}
if (type instanceof VarcharType) {
if (isNull) {
return NullableValue.asNull(type);
}
return NullableValue.of(type, varcharPartitionKey(value, partitionName, type));
}
if (isCharType(type)) {
if (isNull) {
return NullableValue.asNull(type);
}
return NullableValue.of(type, charPartitionKey(value, partitionName, type));
}
throw new PrestoException(
NOT_SUPPORTED, format("Unsupported Type [%s] for partition: %s", type, partitionName));
}
