@Setup
public void setup() {
Random random = new Random();
RowExpression[] arguments = new RowExpression[1 + inListCount];
switch (type) {
case StandardTypes.BIGINT:
prestoType = BIGINT;
for (int i = 1; i <= inListCount; i++) {
arguments[i] = constant((long) random.nextInt(), BIGINT);
}
break;
case StandardTypes.DOUBLE:
prestoType = DOUBLE;
for (int i = 1; i <= inListCount; i++) {
arguments[i] = constant(random.nextDouble(), DOUBLE);
}
break;
case StandardTypes.VARCHAR:
prestoType = VARCHAR;
for (int i = 1; i <= inListCount; i++) {
arguments[i] = constant(Slices.utf8Slice(Long.toString(random.nextLong())), VARCHAR);
}
break;
default:
throw new IllegalStateException();
}
arguments[0] = field(0, prestoType);
RowExpression project = field(0, prestoType);
PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(prestoType));
for (int i = 0; i < 10_000; i++) {
pageBuilder.declarePosition();
switch (type) {
case StandardTypes.BIGINT:
BIGINT.writeLong(pageBuilder.getBlockBuilder(0), random.nextInt());
break;
case StandardTypes.DOUBLE:
DOUBLE.writeDouble(pageBuilder.getBlockBuilder(0), random.nextDouble());
break;
case StandardTypes.VARCHAR:
VARCHAR.writeSlice(
pageBuilder.getBlockBuilder(0), Slices.utf8Slice(Long.toString(random.nextLong())));
break;
}
}
inputPage = pageBuilder.build();
RowExpression filter =
call(
new Signature(IN, SCALAR, parseTypeSignature(StandardTypes.BOOLEAN)),
BOOLEAN,
arguments);
processor =
new ExpressionCompiler(MetadataManager.createTestMetadataManager())
.compilePageProcessor(filter, ImmutableList.of(project))
.get();
}
private void parseStringColumn(int column) {
// don't include column number in message because it causes boxing which is expensive here
checkArgument(!isPartitionColumn[column], "Column is a partition key");
loaded[column] = true;
Object fieldData = rowInspector.getStructFieldData(rowData, structFields[column]);
if (fieldData == null) {
nulls[column] = true;
} else {
Object fieldValue =
((PrimitiveObjectInspector) fieldInspectors[column]).getPrimitiveJavaObject(fieldData);
checkState(fieldValue != null, "fieldValue should not be null");
Slice value;
if (fieldValue instanceof String) {
value = Slices.utf8Slice((String) fieldValue);
} else if (fieldValue instanceof byte[]) {
value = Slices.wrappedBuffer((byte[]) fieldValue);
} else if (fieldValue instanceof HiveVarchar) {
value = Slices.utf8Slice(((HiveVarchar) fieldValue).getValue());
} else {
throw new IllegalStateException(
"unsupported string field type: " + fieldValue.getClass().getName());
}
Type type = types[column];
if (isVarcharType(type)) {
value = truncateToLength(value, type);
}
slices[column] = value;
nulls[column] = false;
}
}
private void parseStringColumn(int column) {
// don't include column number in message because it causes boxing which is expensive here
checkArgument(!isPartitionColumn[column], "Column is a partition key");
loaded[column] = true;
Object fieldData = rowInspector.getStructFieldData(rowData, structFields[column]);
if (fieldData == null) {
nulls[column] = true;
} else if (hiveTypes[column] == HiveType.MAP
|| hiveTypes[column] == HiveType.LIST
|| hiveTypes[column] == HiveType.STRUCT) {
// temporarily special case MAP, LIST, and STRUCT types as strings
slices[column] =
Slices.wrappedBuffer(
SerDeUtils.getJsonBytes(sessionTimeZone, fieldData, fieldInspectors[column]));
nulls[column] = false;
} else {
Object fieldValue =
((PrimitiveObjectInspector) fieldInspectors[column]).getPrimitiveJavaObject(fieldData);
checkState(fieldValue != null, "fieldValue should not be null");
if (fieldValue instanceof String) {
slices[column] = Slices.utf8Slice((String) fieldValue);
} else if (fieldValue instanceof byte[]) {
slices[column] = Slices.wrappedBuffer((byte[]) fieldValue);
} else {
throw new IllegalStateException(
"unsupported string field type: " + fieldValue.getClass().getName());
}
nulls[column] = false;
}
}
@Test
public void testUtf8Conversion() {
String s = "apple \u2603 snowman";
Slice slice = Slices.copiedBuffer(s, UTF_8);
assertEquals(Slices.utf8Slice(s), slice);
assertEquals(slice.toStringUtf8(), s);
assertEquals(Slices.utf8Slice(s).toStringUtf8(), s);
}
private static Block createMapBlock(int positionCount, Block keyBlock, Block valueBlock) {
InterleavedBlock interleavedBlock = new InterleavedBlock(new Block[] {keyBlock, valueBlock});
int[] offsets = new int[positionCount];
int mapSize = keyBlock.getPositionCount() / positionCount;
for (int i = 0; i < positionCount; i++) {
offsets[i] = mapSize * 2 * i;
}
return new ArrayBlock(
interleavedBlock, Slices.wrappedIntArray(offsets), 0, Slices.allocate(positionCount));
}
private static Slice dateFormat(
ISOChronology chronology, Locale locale, long timestamp, Slice formatString) {
DateTimeFormatter formatter =
DATETIME_FORMATTER_CACHE.get(formatString).withChronology(chronology).withLocale(locale);
return Slices.copiedBuffer(formatter.print(timestamp), Charsets.UTF_8);
}
public GroupByIdBlock getGroupIds(Page page) {
int positionCount = page.getPositionCount();
int groupIdBlockSize = SINGLE_LONG.getFixedSize() * positionCount;
BlockBuilder blockBuilder =
new BlockBuilder(
SINGLE_LONG, groupIdBlockSize, Slices.allocate(groupIdBlockSize).getOutput());
// open cursors for group blocks
BlockCursor[] cursors = new BlockCursor[channels.length];
for (int i = 0; i < channels.length; i++) {
cursors[i] = page.getBlock(channels[i]).cursor();
}
// use cursors in hash strategy to provide value for "current" row
hashStrategy.setCurrentRow(cursors);
for (int position = 0; position < positionCount; position++) {
for (BlockCursor cursor : cursors) {
checkState(cursor.advanceNextPosition());
}
int groupId = pagePositionToGroupId.get(CURRENT_ROW_ADDRESS);
if (groupId < 0) {
groupId = addNewGroup(cursors);
}
blockBuilder.append(groupId);
}
UncompressedBlock block = blockBuilder.build();
return new GroupByIdBlock(nextGroupId, block);
}
public ChannelBuilder(Type type, int blockSize) {
checkNotNull(type, "type is null");
this.type = type;
this.slice = Slices.allocate(blockSize);
this.sliceOutput = slice.getOutput();
this.positionOffsets = new IntArrayList(1024);
}
public static Slice toJson(Type rowType, ConnectorSession session, Slice row) {
Object object = rowType.getObjectValue(session, createBlock(rowType, row), 0);
try {
return Slices.utf8Slice(OBJECT_MAPPER.get().writeValueAsString(object));
} catch (JsonProcessingException e) {
throw Throwables.propagate(e);
}
}
private void testIsFull(VariableWidthBlockBuilder blockBuilder) {
assertTrue(blockBuilder.isEmpty());
while (!blockBuilder.isFull()) {
VARCHAR.writeSlice(blockBuilder, Slices.allocate(VARCHAR_VALUE_SIZE));
}
assertEquals(blockBuilder.getPositionCount(), EXPECTED_ENTRY_COUNT);
assertEquals(blockBuilder.isFull(), true);
}
@Description("convert ASCII character code to string")
@ScalarFunction
@SqlType(VarcharType.class)
public static Slice chr(@SqlType(BigintType.class) long n) {
Slice slice = Slices.allocate(1);
slice.setByte(0, Ints.saturatedCast(n));
return slice;
}
private void newBuffer() {
this.buffer = allocator.allocate(PageFormat.PAGE_HEADER_SIZE + fixedRecordSize);
this.bufferSlice = Slices.wrappedBuffer(buffer.array(), buffer.offset(), buffer.capacity());
this.count = 0;
this.position = PageFormat.PAGE_HEADER_SIZE;
this.stringReferences.clear();
this.stringReferenceSize = 0;
}
private static Block createKeyBlock(int positionCount, List<String> keys) {
Block keyDictionaryBlock = createSliceArrayBlock(keys);
int[] keyIds = new int[positionCount * keys.size()];
for (int i = 0; i < keyIds.length; i++) {
keyIds[i] = i % keys.size();
}
return new DictionaryBlock(
positionCount * keys.size(), keyDictionaryBlock, Slices.wrappedIntArray(keyIds));
}
private static Slice formatDatetime(
ISOChronology chronology, Locale locale, long timestamp, Slice formatString) {
String pattern = formatString.toString(Charsets.UTF_8);
DateTimeFormatter formatter =
DateTimeFormat.forPattern(pattern).withChronology(chronology).withLocale(locale);
String datetimeString = formatter.print(timestamp);
return Slices.wrappedBuffer(datetimeString.getBytes(Charsets.UTF_8));
}
@Description("converts all the alphabets in the string to upper case")
@ScalarFunction
@SqlType(VarcharType.class)
public static Slice upper(@SqlType(VarcharType.class) Slice slice) {
Slice upper = Slices.allocate(slice.length());
for (int i = 0; i < slice.length(); i++) {
upper.setByte(i, Ascii.toUpperCase((char) slice.getByte(i)));
}
return upper;
}
@Description("concatenates given strings")
@ScalarFunction
@SqlType(VarcharType.class)
public static Slice concat(
@SqlType(VarcharType.class) Slice str1, @SqlType(VarcharType.class) Slice str2) {
Slice concat = Slices.allocate(str1.length() + str2.length());
concat.setBytes(0, str1);
concat.setBytes(str1.length(), str2);
return concat;
}
@Description("reverses the given string")
@ScalarFunction
@SqlType(VarcharType.class)
public static Slice reverse(@SqlType(VarcharType.class) Slice slice) {
Slice reverse = Slices.allocate(slice.length());
for (int i = 0, j = slice.length() - 1; i < slice.length(); i++, j--) {
reverse.setByte(j, slice.getByte(i));
}
return reverse;
}
@Test
public void testToString() {
assertEquals(Slices.copiedBuffer("apple", UTF_8).toString(UTF_8), "apple");
for (int size = 0; size < 100; size++) {
for (int index = 0; index < size; index++) {
assertToStrings(allocate(size), index);
}
}
}
@Description("greedily replaces occurrences of a pattern with a string")
@ScalarFunction
@SqlType(VarcharType.class)
public static Slice replace(
@SqlType(VarcharType.class) Slice str,
@SqlType(VarcharType.class) Slice search,
@SqlType(VarcharType.class) Slice replace) {
String replaced =
str.toString(Charsets.UTF_8)
.replace(search.toString(Charsets.UTF_8), replace.toString(Charsets.UTF_8));
return Slices.copiedBuffer(replaced, Charsets.UTF_8);
}
public static Slice charPartitionKey(String value, String name, Type columnType) {
Slice partitionKey = trimSpaces(Slices.utf8Slice(value));
CharType charType = checkType(columnType, CharType.class, "columnType");
if (SliceUtf8.countCodePoints(partitionKey) > charType.getLength()) {
throw new PrestoException(
HIVE_INVALID_PARTITION_VALUE,
format(
"Invalid partition value '%s' for %s partition key: %s",
value, columnType.toString(), name));
}
return partitionKey;
}
private Page generateZeroPage(List<Type> types, int rowsCount, int fieldLength) {
byte[] constantBytes = new byte[fieldLength];
Arrays.fill(constantBytes, (byte) 42);
Slice constantSlice = Slices.wrappedBuffer(constantBytes);
Block[] blocks = new Block[types.size()];
for (int i = 0; i < blocks.length; i++) {
blocks[i] = createZeroBlock(types.get(i), rowsCount, constantSlice);
}
return new Page(rowsCount, blocks);
}
private static Block createDictionaryValueBlock(int positionCount, int mapSize) {
double distinctRatio = 0.82;
int dictionarySize = (int) (positionCount * mapSize * distinctRatio);
List<String> dictionaryStrings = new ArrayList<>(dictionarySize);
for (int i = 0; i < dictionarySize; i++) {
int wordLength = ThreadLocalRandom.current().nextInt(5, 10);
dictionaryStrings.add(randomString(wordLength));
}
Block dictionaryBlock = createSliceArrayBlock(dictionaryStrings);
int[] keyIds = new int[positionCount * mapSize];
for (int i = 0; i < keyIds.length; i++) {
keyIds[i] = ThreadLocalRandom.current().nextInt(0, dictionarySize);
}
return new DictionaryBlock(
positionCount * mapSize, dictionaryBlock, Slices.wrappedIntArray(keyIds));
}
@Override
public Block copyPositions(List<Integer> positions) {
int finalLength = positions.stream().mapToInt(this::getLength).sum();
SliceOutput newSlice = Slices.allocate(finalLength).getOutput();
int[] newOffsets = new int[positions.size() + 1];
boolean[] newValueIsNull = new boolean[positions.size()];
for (int i = 0; i < positions.size(); i++) {
int position = positions.get(i);
if (isEntryNull(position)) {
newValueIsNull[i] = true;
} else {
newSlice.appendBytes(
sliceOutput
.getUnderlyingSlice()
.getBytes(getPositionOffset(position), getLength(position)));
}
newOffsets[i + 1] = newSlice.size();
}
return new VariableWidthBlock(positions.size(), newSlice.slice(), newOffsets, newValueIsNull);
}
@Test
public void testMemoryMappedReads() throws IOException {
Path path = Files.createTempFile("longs", null);
ImmutableList<Long> values = createRandomLongs(20000);
Slice output = allocate(values.size() * Longs.BYTES);
for (int i = 0; i < values.size(); i++) {
output.setLong(i * Longs.BYTES, values.get(i));
}
Files.write(path, output.getBytes());
Slice slice = Slices.mapFileReadOnly(path.toFile());
for (int i = 0; i < values.size(); i++) {
long actual = slice.getLong(i * Longs.BYTES);
long expected = values.get(i);
assertEquals(actual, expected);
}
assertEquals(slice.getBytes(), output.getBytes());
}
@SuppressWarnings("CharUsedInArithmeticContext")
private static void assertToStrings(Slice slice, int index) {
// fill slice with FF
slice.fill((byte) 0xFF);
// set and get the value
char[] chars = new char[(slice.length() - index) / 2];
for (int i = 0; i < chars.length; i++) {
chars[i] = (char) ('a' + (i % 26));
}
String string = new String(chars);
Slice value = Slices.copiedBuffer(string, UTF_8);
slice.setBytes(index, value);
assertEquals(slice.toString(index, value.length(), UTF_8), string);
for (int length = 0; length < value.length(); length++) {
slice.fill((byte) 0xFF);
slice.setBytes(index, value, 0, length);
assertEquals(slice.toString(index, length, UTF_8), string.substring(0, length));
}
}
public GenericHiveRecordCursor(
RecordReader<K, V> recordReader,
long totalBytes,
Properties splitSchema,
List<HivePartitionKey> partitionKeys,
List<HiveColumnHandle> columns,
DateTimeZone hiveStorageTimeZone,
DateTimeZone sessionTimeZone) {
checkNotNull(recordReader, "recordReader is null");
checkArgument(totalBytes >= 0, "totalBytes is negative");
checkNotNull(splitSchema, "splitSchema is null");
checkNotNull(partitionKeys, "partitionKeys is null");
checkNotNull(columns, "columns is null");
checkArgument(!columns.isEmpty(), "columns is empty");
checkNotNull(hiveStorageTimeZone, "hiveStorageTimeZone is null");
checkNotNull(sessionTimeZone, "sessionTimeZone is null");
this.recordReader = recordReader;
this.totalBytes = totalBytes;
this.key = recordReader.createKey();
this.value = recordReader.createValue();
this.hiveStorageTimeZone = hiveStorageTimeZone;
this.sessionTimeZone = sessionTimeZone;
this.deserializer = getDeserializer(splitSchema);
this.rowInspector = getTableObjectInspector(deserializer);
int size = columns.size();
String[] names = new String[size];
this.types = new Type[size];
this.hiveTypes = new HiveType[size];
this.structFields = new StructField[size];
this.fieldInspectors = new ObjectInspector[size];
this.isPartitionColumn = new boolean[size];
this.loaded = new boolean[size];
this.booleans = new boolean[size];
this.longs = new long[size];
this.doubles = new double[size];
this.slices = new Slice[size];
this.nulls = new boolean[size];
// initialize data columns
for (int i = 0; i < columns.size(); i++) {
HiveColumnHandle column = columns.get(i);
names[i] = column.getName();
types[i] = column.getType();
hiveTypes[i] = column.getHiveType();
if (!column.isPartitionKey()) {
StructField field = rowInspector.getStructFieldRef(column.getName());
structFields[i] = field;
fieldInspectors[i] = field.getFieldObjectInspector();
}
isPartitionColumn[i] = column.isPartitionKey();
}
// parse requested partition columns
Map<String, HivePartitionKey> partitionKeysByName =
uniqueIndex(partitionKeys, HivePartitionKey.nameGetter());
for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) {
HiveColumnHandle column = columns.get(columnIndex);
if (column.isPartitionKey()) {
HivePartitionKey partitionKey = partitionKeysByName.get(column.getName());
checkArgument(partitionKey != null, "Unknown partition key %s", column.getName());
byte[] bytes = partitionKey.getValue().getBytes(Charsets.UTF_8);
Type type = types[columnIndex];
if (BOOLEAN.equals(type)) {
if (isTrue(bytes, 0, bytes.length)) {
booleans[columnIndex] = true;
} else if (isFalse(bytes, 0, bytes.length)) {
booleans[columnIndex] = false;
} else {
String valueString = new String(bytes, Charsets.UTF_8);
throw new IllegalArgumentException(
String.format(
"Invalid partition value '%s' for BOOLEAN partition key %s",
valueString, names[columnIndex]));
}
} else if (BIGINT.equals(type)) {
if (bytes.length == 0) {
throw new IllegalArgumentException(
String.format(
"Invalid partition value '' for BIGINT partition key %s", names[columnIndex]));
}
longs[columnIndex] = parseLong(bytes, 0, bytes.length);
} else if (DOUBLE.equals(type)) {
if (bytes.length == 0) {
throw new IllegalArgumentException(
String.format(
"Invalid partition value '' for DOUBLE partition key %s", names[columnIndex]));
}
doubles[columnIndex] = parseDouble(bytes, 0, bytes.length);
} else if (VARCHAR.equals(type)) {
slices[columnIndex] = Slices.wrappedBuffer(Arrays.copyOf(bytes, bytes.length));
} else {
throw new UnsupportedOperationException("Unsupported column type: " + type);
}
}
}
}
protected Slice allocate(int size) {
return Slices.allocate(size);
}
private static void serializePrimitive(
Type type, BlockBuilder builder, Object object, PrimitiveObjectInspector inspector) {
requireNonNull(builder, "parent builder is null");
if (object == null) {
builder.appendNull();
return;
}
switch (inspector.getPrimitiveCategory()) {
case BOOLEAN:
BooleanType.BOOLEAN.writeBoolean(builder, ((BooleanObjectInspector) inspector).get(object));
return;
case BYTE:
TinyintType.TINYINT.writeLong(builder, ((ByteObjectInspector) inspector).get(object));
return;
case SHORT:
SmallintType.SMALLINT.writeLong(builder, ((ShortObjectInspector) inspector).get(object));
return;
case INT:
IntegerType.INTEGER.writeLong(builder, ((IntObjectInspector) inspector).get(object));
return;
case LONG:
BigintType.BIGINT.writeLong(builder, ((LongObjectInspector) inspector).get(object));
return;
case FLOAT:
DoubleType.DOUBLE.writeDouble(builder, ((FloatObjectInspector) inspector).get(object));
return;
case DOUBLE:
DoubleType.DOUBLE.writeDouble(builder, ((DoubleObjectInspector) inspector).get(object));
return;
case STRING:
type.writeSlice(
builder,
Slices.utf8Slice(((StringObjectInspector) inspector).getPrimitiveJavaObject(object)));
return;
case VARCHAR:
type.writeSlice(
builder,
Slices.utf8Slice(
((HiveVarcharObjectInspector) inspector)
.getPrimitiveJavaObject(object)
.getValue()));
return;
case CHAR:
CharType charType = checkType(type, CharType.class, "type");
HiveChar hiveChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(object);
type.writeSlice(
builder,
trimSpacesAndTruncateToLength(
Slices.utf8Slice(hiveChar.getValue()), charType.getLength()));
return;
case DATE:
DateType.DATE.writeLong(builder, formatDateAsLong(object, (DateObjectInspector) inspector));
return;
case TIMESTAMP:
TimestampType.TIMESTAMP.writeLong(
builder, formatTimestampAsLong(object, (TimestampObjectInspector) inspector));
return;
case BINARY:
VARBINARY.writeSlice(
builder,
Slices.wrappedBuffer(
((BinaryObjectInspector) inspector).getPrimitiveJavaObject(object)));
return;
case DECIMAL:
DecimalType decimalType = checkType(type, DecimalType.class, "type");
HiveDecimalWritable hiveDecimal =
((HiveDecimalObjectInspector) inspector).getPrimitiveWritableObject(object);
if (decimalType.isShort()) {
decimalType.writeLong(
builder, DecimalUtils.getShortDecimalValue(hiveDecimal, decimalType.getScale()));
} else {
decimalType.writeSlice(
builder, DecimalUtils.getLongDecimalValue(hiveDecimal, decimalType.getScale()));
}
return;
}
throw new RuntimeException("Unknown primitive type: " + inspector.getPrimitiveCategory());
}
public static Expression toExpression(Object object, Type type) {
requireNonNull(type, "type is null");
if (object instanceof Expression) {
return (Expression) object;
}
if (object == null) {
if (type.equals(UNKNOWN)) {
return new NullLiteral();
}
return new Cast(new NullLiteral(), type.getTypeSignature().toString(), false, true);
}
checkArgument(
Primitives.wrap(type.getJavaType()).isInstance(object),
"object.getClass (%s) and type.getJavaType (%s) do not agree",
object.getClass(),
type.getJavaType());
if (type.equals(BIGINT)) {
return new LongLiteral(object.toString());
}
if (type.equals(DOUBLE)) {
Double value = (Double) object;
// WARNING: the ORC predicate code depends on NaN and infinity not appearing in a tuple
// domain, so
// if you remove this, you will need to update the TupleDomainOrcPredicate
if (value.isNaN()) {
return new FunctionCall(new QualifiedName("nan"), ImmutableList.<Expression>of());
} else if (value.equals(Double.NEGATIVE_INFINITY)) {
return ArithmeticUnaryExpression.negative(
new FunctionCall(new QualifiedName("infinity"), ImmutableList.<Expression>of()));
} else if (value.equals(Double.POSITIVE_INFINITY)) {
return new FunctionCall(new QualifiedName("infinity"), ImmutableList.<Expression>of());
} else {
return new DoubleLiteral(object.toString());
}
}
if (type instanceof VarcharType) {
if (object instanceof String) {
object = Slices.utf8Slice((String) object);
}
if (object instanceof Slice) {
Slice value = (Slice) object;
int length = SliceUtf8.countCodePoints(value);
if (length == ((VarcharType) type).getLength()) {
return new StringLiteral(value.toStringUtf8());
}
return new Cast(
new StringLiteral(value.toStringUtf8()), type.getDisplayName(), false, true);
}
throw new IllegalArgumentException(
"object must be instance of Slice or String when type is VARCHAR");
}
if (type.equals(BOOLEAN)) {
return new BooleanLiteral(object.toString());
}
if (object instanceof Block) {
SliceOutput output = new DynamicSliceOutput(((Block) object).getSizeInBytes());
BlockSerdeUtil.writeBlock(output, (Block) object);
object = output.slice();
// This if condition will evaluate to true: object instanceof Slice && !type.equals(VARCHAR)
}
if (object instanceof Slice) {
// HACK: we need to serialize VARBINARY in a format that can be embedded in an expression to
// be
// able to encode it in the plan that gets sent to workers.
// We do this by transforming the in-memory varbinary into a call to
// from_base64(<base64-encoded value>)
FunctionCall fromBase64 =
new FunctionCall(
new QualifiedName("from_base64"),
ImmutableList.of(
new StringLiteral(VarbinaryFunctions.toBase64((Slice) object).toStringUtf8())));
Signature signature = FunctionRegistry.getMagicLiteralFunctionSignature(type);
return new FunctionCall(new QualifiedName(signature.getName()), ImmutableList.of(fromBase64));
}
Signature signature = FunctionRegistry.getMagicLiteralFunctionSignature(type);
Expression rawLiteral = toExpression(object, FunctionRegistry.typeForMagicLiteral(type));
return new FunctionCall(new QualifiedName(signature.getName()), ImmutableList.of(rawLiteral));
}
@Override
public Block readBlock(Type type) throws IOException {
if (!rowGroupOpen) {
openRowGroup();
}
if (readOffset > 0) {
if (presentStream != null) {
// skip ahead the present bit reader, but count the set bits
// and use this as the skip size for the field readers
readOffset = presentStream.countBitsSet(readOffset);
}
for (StreamReader structField : structFields) {
structField.prepareNextRead(readOffset);
}
}
List<Type> typeParameters = type.getTypeParameters();
boolean[] nullVector = new boolean[nextBatchSize];
Block[] blocks = new Block[typeParameters.size()];
if (presentStream == null) {
for (int i = 0; i < typeParameters.size(); i++) {
StreamReader structField = structFields[i];
structField.prepareNextRead(nextBatchSize);
blocks[i] = structField.readBlock(typeParameters.get(i));
}
} else {
int nullValues = presentStream.getUnsetBits(nextBatchSize, nullVector);
if (nullValues != nextBatchSize) {
for (int i = 0; i < typeParameters.size(); i++) {
StreamReader structField = structFields[i];
structField.prepareNextRead(nextBatchSize - nullValues);
blocks[i] = structField.readBlock(typeParameters.get(i));
}
} else {
for (int i = 0; i < typeParameters.size(); i++) {
blocks[i] = typeParameters.get(i).createBlockBuilder(new BlockBuilderStatus(), 0).build();
}
}
}
// Build offsets for array block (null valued have no positions)
int[] offsets = new int[nextBatchSize];
offsets[0] = (nullVector[0] ? 0 : typeParameters.size());
for (int i = 1; i < nextBatchSize; i++) {
offsets[i] = offsets[i - 1] + (nullVector[i] ? 0 : typeParameters.size());
}
// Struct is represented as an array block holding an interleaved block
InterleavedBlock interleavedBlock = new InterleavedBlock(blocks);
ArrayBlock arrayBlock =
new ArrayBlock(
interleavedBlock,
Slices.wrappedIntArray(offsets),
0,
Slices.wrappedBooleanArray(nullVector));
readOffset = 0;
nextBatchSize = 0;
return arrayBlock;
}
