private static Block serializeList(
Type type, BlockBuilder builder, Object object, ListObjectInspector inspector) {
List<?> list = inspector.getList(object);
if (list == null) {
requireNonNull(builder, "parent builder is null").appendNull();
return null;
}
List<Type> typeParameters = type.getTypeParameters();
checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter");
Type elementType = typeParameters.get(0);
ObjectInspector elementInspector = inspector.getListElementObjectInspector();
BlockBuilder currentBuilder;
if (builder != null) {
currentBuilder = builder.beginBlockEntry();
} else {
currentBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), list.size());
}
for (Object element : list) {
serializeObject(elementType, currentBuilder, element, elementInspector);
}
if (builder != null) {
builder.closeEntry();
return null;
} else {
Block resultBlock = currentBuilder.build();
return resultBlock;
}
}
public static Block arrayBlockOf(Type elementType, Object... values) {
BlockBuilder blockBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), 1024);
for (Object value : values) {
appendToBlockBuilder(elementType, value, blockBuilder);
}
return blockBuilder.build();
}
private static Block createVarWidthValueBlock(int positionCount, int mapSize) {
Type valueType = createUnboundedVarcharType();
BlockBuilder valueBlockBuilder =
valueType.createBlockBuilder(new BlockBuilderStatus(), positionCount * mapSize);
for (int i = 0; i < positionCount * mapSize; i++) {
int wordLength = ThreadLocalRandom.current().nextInt(5, 10);
valueType.writeSlice(valueBlockBuilder, utf8Slice(randomString(wordLength)));
}
return valueBlockBuilder.build();
}
protected void checkPageSource(
ConnectorPageSource pageSource, List<TestColumn> testColumns, List<Type> types)
throws IOException {
try {
MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, types);
for (MaterializedRow row : result) {
for (int i = 0, testColumnsSize = testColumns.size(); i < testColumnsSize; i++) {
TestColumn testColumn = testColumns.get(i);
Type type = types.get(i);
Object actualValue = row.getField(i);
Object expectedValue = testColumn.getExpectedValue();
if (actualValue == null) {
assertEquals(null, expectedValue, String.format("Expected non-null for column %d", i));
} else if (testColumn.getObjectInspector().getTypeName().equals("float")
|| testColumn.getObjectInspector().getTypeName().equals("double")) {
assertEquals((double) actualValue, (double) expectedValue, EPSILON);
} else if (testColumn.getObjectInspector().getTypeName().equals("date")) {
SqlDate expectedDate = new SqlDate(((Long) expectedValue).intValue());
assertEquals(actualValue, expectedDate);
} else if (testColumn.getObjectInspector().getTypeName().equals("timestamp")) {
SqlTimestamp expectedTimestamp =
new SqlTimestamp((Long) expectedValue, SESSION.getTimeZoneKey());
assertEquals(actualValue, expectedTimestamp);
} else if (testColumn.getObjectInspector().getCategory() == Category.PRIMITIVE) {
if (expectedValue instanceof Slice) {
expectedValue = ((Slice) expectedValue).toStringUtf8();
}
if (actualValue instanceof Slice) {
actualValue = ((Slice) actualValue).toStringUtf8();
}
if (actualValue instanceof SqlVarbinary) {
actualValue = new String(((SqlVarbinary) actualValue).getBytes(), UTF_8);
}
assertEquals(actualValue, expectedValue, String.format("Wrong value for column %d", i));
} else {
BlockBuilder builder = type.createBlockBuilder(new BlockBuilderStatus(), 1);
type.writeObject(builder, expectedValue);
expectedValue = type.getObjectValue(SESSION, builder.build(), 0);
assertEquals(
actualValue,
expectedValue,
String.format("Wrong value for column %s", testColumn.getName()));
}
}
}
} finally {
pageSource.close();
}
}
private Block createZeroBlock(Type type, int rowsCount, Slice constantSlice) {
checkArgument(isSupportedType(type), "Unsupported type [%s]", type);
Slice slice;
// do not exceed varchar limit
if (isVarcharType(type)) {
slice =
constantSlice.slice(
0, Math.min(((VarcharType) type).getLength(), constantSlice.length()));
} else if (isLongDecimal(type)) {
slice = encodeScaledValue(ZERO);
} else {
slice = constantSlice;
}
BlockBuilder builder;
if (type instanceof FixedWidthType) {
builder = type.createBlockBuilder(new BlockBuilderStatus(), rowsCount);
} else {
builder = type.createBlockBuilder(new BlockBuilderStatus(), rowsCount, slice.length());
}
for (int i = 0; i < rowsCount; i++) {
Class<?> javaType = type.getJavaType();
if (javaType == boolean.class) {
type.writeBoolean(builder, false);
} else if (javaType == long.class) {
type.writeLong(builder, 0);
} else if (javaType == double.class) {
type.writeDouble(builder, 0.0);
} else if (javaType == Slice.class) {
requireNonNull(slice, "slice is null");
type.writeSlice(builder, slice, 0, slice.length());
} else {
throw new UnsupportedOperationException("Unknown javaType: " + javaType.getName());
}
}
return builder.build();
}
public ParquetPageSource(
ParquetReader parquetReader,
ParquetDataSource dataSource,
MessageType fileSchema,
MessageType requestedSchema,
long totalBytes,
Properties splitSchema,
List<HiveColumnHandle> columns,
TupleDomain<HiveColumnHandle> effectivePredicate,
TypeManager typeManager,
boolean useParquetColumnNames) {
checkArgument(totalBytes >= 0, "totalBytes is negative");
requireNonNull(splitSchema, "splitSchema is null");
requireNonNull(columns, "columns is null");
requireNonNull(effectivePredicate, "effectivePredicate is null");
this.parquetReader = parquetReader;
this.dataSource = dataSource;
this.requestedSchema = requestedSchema;
this.totalBytes = totalBytes;
int size = columns.size();
this.constantBlocks = new Block[size];
this.hiveColumnIndexes = new int[size];
ImmutableList.Builder<String> namesBuilder = ImmutableList.builder();
ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder();
for (int columnIndex = 0; columnIndex < size; columnIndex++) {
HiveColumnHandle column = columns.get(columnIndex);
checkState(column.getColumnType() == REGULAR, "column type must be regular");
String name = column.getName();
Type type = typeManager.getType(column.getTypeSignature());
namesBuilder.add(name);
typesBuilder.add(type);
hiveColumnIndexes[columnIndex] = column.getHiveColumnIndex();
if (getParquetType(column, fileSchema, useParquetColumnNames) == null) {
BlockBuilder blockBuilder =
type.createBlockBuilder(new BlockBuilderStatus(), MAX_VECTOR_LENGTH);
for (int i = 0; i < MAX_VECTOR_LENGTH; i++) {
blockBuilder.appendNull();
}
constantBlocks[columnIndex] = blockBuilder.build();
}
}
types = typesBuilder.build();
columnNames = namesBuilder.build();
}
private static Block toBlock(Type type, SliceInput input, int length) {
BlockBuilder builder = type.createBlockBuilder(new BlockBuilderStatus(), 1, length);
if (type.getJavaType() == long.class) {
type.writeLong(builder, input.readLong());
} else if (type.getJavaType() == double.class) {
type.writeDouble(builder, input.readDouble());
} else if (type.getJavaType() == Slice.class) {
type.writeSlice(builder, input.readSlice(length));
} else if (type.getJavaType() == boolean.class) {
type.writeBoolean(builder, input.readByte() != 0);
} else {
throw new IllegalArgumentException("Unsupported type: " + type.getJavaType().getSimpleName());
}
return builder.build();
}
@Override
public void start() {
if (builder == null) {
if (nullBuilder == null
|| (nullBuilder.getPositionCount() >= NULL_BUILDER_POSITIONS_THRESHOLD
&& nullBuilder.getSizeInBytes() >= NULL_BUILDER_SIZE_IN_BYTES_THRESHOLD)) {
nullBuilder =
mapType.createBlockBuilder(
new BlockBuilderStatus(), NULL_BUILDER_POSITIONS_THRESHOLD);
}
currentEntryBuilder = nullBuilder.beginBlockEntry();
} else {
currentEntryBuilder = builder.beginBlockEntry();
}
entryConverter.beforeValue(currentEntryBuilder);
}
@NotNull
public static Object evaluatePropertyValue(
Expression expression, Type expectedType, Session session, Metadata metadata) {
Object value =
evaluateConstantExpression(expression, expectedType, metadata, session, ImmutableSet.of());
// convert to object value type of SQL type
BlockBuilder blockBuilder = expectedType.createBlockBuilder(new BlockBuilderStatus(), 1);
BlockUtils.appendObject(expectedType, blockBuilder, value);
Object objectValue = expectedType.getObjectValue(session.toConnectorSession(), blockBuilder, 0);
if (objectValue == null) {
throw new PrestoException(
INVALID_SESSION_PROPERTY, "Session property value must not be null");
}
return objectValue;
}
public static Object evaluatePropertyValue(
Expression expression,
Type expectedType,
Session session,
Metadata metadata,
List<Expression> parameters) {
Expression rewritten =
ExpressionTreeRewriter.rewriteWith(new ParameterRewriter(parameters), expression);
Object value =
evaluateConstantExpression(rewritten, expectedType, metadata, session, parameters);
// convert to object value type of SQL type
BlockBuilder blockBuilder = expectedType.createBlockBuilder(new BlockBuilderStatus(), 1);
writeNativeValue(expectedType, blockBuilder, value);
Object objectValue = expectedType.getObjectValue(session.toConnectorSession(), blockBuilder, 0);
if (objectValue == null) {
throw new PrestoException(
INVALID_SESSION_PROPERTY, "Session property value must not be null");
}
return objectValue;
}
private static Object toTypeObjectValue(Session session, Type type, Object value) {
BlockBuilder blockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), 1);
writeNativeValue(type, blockBuilder, value);
return type.getObjectValue(session.toConnectorSession(), blockBuilder, 0);
}
public static Block intersect(Type type, Block leftArray, Block rightArray) {
int leftPositionCount = leftArray.getPositionCount();
int rightPositionCount = rightArray.getPositionCount();
if (leftPositionCount == 0) {
return leftArray;
}
if (rightPositionCount == 0) {
return rightArray;
}
int[] leftPositions = new int[leftPositionCount];
int[] rightPositions = new int[rightPositionCount];
for (int i = 0; i < leftPositionCount; i++) {
leftPositions[i] = i;
}
for (int i = 0; i < rightPositionCount; i++) {
rightPositions[i] = i;
}
IntArrays.quickSort(leftPositions, IntBlockCompare(type, leftArray));
IntArrays.quickSort(rightPositions, IntBlockCompare(type, rightArray));
int entrySize;
if (leftPositionCount < rightPositionCount) {
entrySize = (int) Math.ceil(leftArray.getSizeInBytes() / (double) leftPositionCount);
} else {
entrySize = (int) Math.ceil(rightArray.getSizeInBytes() / (double) rightPositionCount);
}
BlockBuilder resultBlockBuilder =
type.createBlockBuilder(
new BlockBuilderStatus(),
Math.min(leftArray.getPositionCount(), rightArray.getPositionCount()),
entrySize);
int leftCurrentPosition = 0;
int rightCurrentPosition = 0;
int leftBasePosition;
int rightBasePosition;
while (leftCurrentPosition < leftPositionCount && rightCurrentPosition < rightPositionCount) {
leftBasePosition = leftCurrentPosition;
rightBasePosition = rightCurrentPosition;
int compareValue =
type.compareTo(
leftArray,
leftPositions[leftCurrentPosition],
rightArray,
rightPositions[rightCurrentPosition]);
if (compareValue > 0) {
rightCurrentPosition++;
} else if (compareValue < 0) {
leftCurrentPosition++;
} else {
type.appendTo(leftArray, leftPositions[leftCurrentPosition], resultBlockBuilder);
leftCurrentPosition++;
rightCurrentPosition++;
while (leftCurrentPosition < leftPositionCount
&& type.equalTo(
leftArray,
leftPositions[leftBasePosition],
leftArray,
leftPositions[leftCurrentPosition])) {
leftCurrentPosition++;
}
while (rightCurrentPosition < rightPositionCount
&& type.equalTo(
rightArray,
rightPositions[rightBasePosition],
rightArray,
rightPositions[rightCurrentPosition])) {
rightCurrentPosition++;
}
}
}
return resultBlockBuilder.build();
}
