@Override
public void serialize(MaxOrMinByState state, BlockBuilder out) {
SliceOutput sliceOutput = new DynamicSliceOutput((int) state.getEstimatedSize());
int keyLength = 0;
if (state.getKey() != null && !state.getKey().isNull(0)) {
keyLength = state.getKey().getLength(0);
}
sliceOutput.writeInt(keyLength);
int valueLength = 0;
if (state.getValue() != null && !state.getValue().isNull(0)) {
valueLength = state.getValue().getLength(0);
}
sliceOutput.writeInt(valueLength);
if (state.getKey() != null && !state.getKey().isNull(0)) {
appendTo(keyType, sliceOutput, state.getKey());
}
if (state.getValue() != null && !state.getValue().isNull(0)) {
appendTo(valueType, sliceOutput, state.getValue());
}
Slice slice = sliceOutput.slice();
out.writeBytes(slice, 0, slice.length());
out.closeEntry();
}
@Override
public void writeBlock(SliceOutput sliceOutput, Block block) {
int positionCount = block.getPositionCount();
sliceOutput.appendInt(positionCount);
encodeNullsAsBits(sliceOutput, block);
for (int position = 0; position < positionCount; position++) {
if (!block.isNull(position)) {
sliceOutput.writeLong(block.getLong(position, 0));
}
}
}
private static void appendTo(Type type, SliceOutput output, Block block) {
if (type.getJavaType() == long.class) {
output.appendLong(type.getLong(block, 0));
} else if (type.getJavaType() == double.class) {
output.appendDouble(type.getDouble(block, 0));
} else if (type.getJavaType() == Slice.class) {
output.appendBytes(type.getSlice(block, 0));
} else if (type.getJavaType() == boolean.class) {
output.appendByte(type.getBoolean(block, 0) ? 1 : 0);
} else {
throw new IllegalArgumentException("Unsupported type: " + type.getJavaType().getSimpleName());
}
}
@Override
public Block build() {
if (currentEntrySize > 0) {
throw new IllegalStateException("Current entry must be closed before the block can be built");
}
return new VariableWidthBlock(positions, sliceOutput.slice(), offsets, valueIsNull);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("VariableWidthBlockBuilder{");
sb.append("positionCount=").append(positions);
sb.append(", size=").append(sliceOutput.size());
sb.append('}');
return sb.toString();
}
private void recordNewPosition() {
if (positions == offsets.length) {
offsets = Arrays.copyOf(offsets, offsets.length * 2);
}
offsets[positions] = sliceOutput.size();
positions++;
}
@Override
public BlockBuilder appendNull() {
recordNewPosition();
sliceOutput.writeByte(1);
entryAdded(0);
return this;
}
@Override
public Block getRegion(int positionOffset, int length) {
int positionCount = getPositionCount();
if (positionOffset < 0 || length < 0 || positionOffset + length > positionCount) {
throw new IndexOutOfBoundsException(
"Invalid position " + positionOffset + " in block with " + positionCount + " positions");
}
return new VariableWidthBlock(
positionOffset, length, sliceOutput.slice(), offsets, valueIsNull);
}
@Override
public Block copyPositions(List<Integer> positions) {
int finalLength = positions.stream().mapToInt(this::getLength).sum();
SliceOutput newSlice = Slices.allocate(finalLength).getOutput();
int[] newOffsets = new int[positions.size() + 1];
boolean[] newValueIsNull = new boolean[positions.size()];
for (int i = 0; i < positions.size(); i++) {
int position = positions.get(i);
if (isEntryNull(position)) {
newValueIsNull[i] = true;
} else {
newSlice.appendBytes(
sliceOutput
.getUnderlyingSlice()
.getBytes(getPositionOffset(position), getLength(position)));
}
newOffsets[i + 1] = newSlice.size();
}
return new VariableWidthBlock(positions.size(), newSlice.slice(), newOffsets, newValueIsNull);
}
@Override
public BlockBuilder appendSlice(Slice value, int offset, int length) {
recordNewPosition();
sliceOutput.writeByte(0);
int bytesWritten = type.writeSlice(sliceOutput, value, offset, length);
entryAdded(bytesWritten);
return this;
}
private void entryAdded(int bytesWritten, boolean isNull) {
if (valueIsNull.length <= positions) {
growCapacity();
}
valueIsNull[positions] = isNull;
offsets[positions + 1] = sliceOutput.size();
positions++;
blockBuilderStatus.addBytes(SIZE_OF_BYTE + SIZE_OF_INT + bytesWritten);
}
@Override
public Block copyRegion(int positionOffset, int length) {
int positionCount = getPositionCount();
if (positionOffset < 0 || length < 0 || positionOffset + length > positionCount) {
throw new IndexOutOfBoundsException(
"Invalid position " + positionOffset + " in block with " + positionCount + " positions");
}
int[] newOffsets = Arrays.copyOfRange(offsets, positionOffset, positionOffset + length + 1);
boolean[] newValueIsNull =
Arrays.copyOfRange(valueIsNull, positionOffset, positionOffset + length);
return new VariableWidthBlock(length, sliceOutput.slice(), newOffsets, newValueIsNull);
}
@Override
public void reset(BlockBuilderStatus blockBuilderStatus) {
this.blockBuilderStatus = requireNonNull(blockBuilderStatus, "blockBuilderStatus is null");
int newSize = calculateBlockResetSize(positions);
valueIsNull = new boolean[newSize];
offsets = new int[newSize + 1];
sliceOutput = new DynamicSliceOutput(calculateBlockResetSize(sliceOutput.size()));
positions = 0;
currentEntrySize = 0;
updateArraysDataSize();
}
@Override
public void writeTo(
List<Page> pages,
Class<?> type,
Type genericType,
Annotation[] annotations,
MediaType mediaType,
MultivaluedMap<String, Object> httpHeaders,
OutputStream output)
throws IOException, WebApplicationException {
try {
SliceOutput sliceOutput = new OutputStreamSliceOutput(output);
PagesSerde.writePages(blockEncodingSerde, sliceOutput, pages);
// We use flush instead of close, because the underlying stream would be closed and that is
// not allowed.
sliceOutput.flush();
} catch (RuntimeIOException e) {
// EOF exception occurs when the client disconnects while writing data
// This is not a "server" problem so we don't want to log this
if (!(e.getCause() instanceof EOFException)) {
throw e;
}
}
}
@Override
public int getSizeInBytes() {
return sliceOutput.size();
}
@Override
public RandomAccessBlock build() {
return new VariableWidthRandomAccessBlock(
type, sliceOutput.slice(), Arrays.copyOf(offsets, positions));
}
@Override
protected Slice getRawSlice() {
return sliceOutput.getUnderlyingSlice();
}
private void entryAdded(int bytesWritten) {
blockBuilderStatus.addBytes(SIZE_OF_BYTE + bytesWritten);
if (sliceOutput.size() >= blockBuilderStatus.getMaxBlockSizeInBytes()) {
blockBuilderStatus.setFull();
}
}
public boolean append(BlockCursor cursor) {
// the extra BYTE here is for the null flag
int writableBytes = sliceOutput.writableBytes() - SIZE_OF_BYTE;
boolean isNull = cursor.isNull();
if (type == Type.FIXED_INT_64) {
if (writableBytes < SIZE_OF_LONG) {
return false;
}
positionOffsets.add(sliceOutput.size());
sliceOutput.writeByte(isNull ? 1 : 0);
sliceOutput.appendLong(isNull ? 0 : cursor.getLong());
} else if (type == Type.DOUBLE) {
if (writableBytes < SIZE_OF_DOUBLE) {
return false;
}
positionOffsets.add(sliceOutput.size());
sliceOutput.writeByte(isNull ? 1 : 0);
sliceOutput.appendDouble(isNull ? 0 : cursor.getDouble());
} else if (type == Type.BOOLEAN) {
if (writableBytes < SIZE_OF_BYTE) {
return false;
}
positionOffsets.add(sliceOutput.size());
sliceOutput.writeByte(isNull ? 1 : 0);
sliceOutput.writeByte(!isNull && cursor.getBoolean() ? 1 : 0);
} else if (type == Type.VARIABLE_BINARY) {
int sliceLength =
isNull ? 0 : getVariableBinaryLength(cursor.getRawSlice(), cursor.getRawOffset());
if (writableBytes < SIZE_OF_INT + sliceLength) {
return false;
}
int startingOffset = sliceOutput.size();
positionOffsets.add(startingOffset);
sliceOutput.writeByte(isNull ? 1 : 0);
sliceOutput.appendInt(sliceLength + SIZE_OF_BYTE + SIZE_OF_INT);
if (!isNull) {
sliceOutput.writeBytes(
cursor.getRawSlice(),
cursor.getRawOffset() + SIZE_OF_BYTE + SIZE_OF_INT,
sliceLength);
}
} else {
throw new IllegalArgumentException("Unsupported type " + type);
}
return true;
}
public static Expression toExpression(Object object, Type type) {
requireNonNull(type, "type is null");
if (object instanceof Expression) {
return (Expression) object;
}
if (object == null) {
if (type.equals(UNKNOWN)) {
return new NullLiteral();
}
return new Cast(new NullLiteral(), type.getTypeSignature().toString(), false, true);
}
checkArgument(
Primitives.wrap(type.getJavaType()).isInstance(object),
"object.getClass (%s) and type.getJavaType (%s) do not agree",
object.getClass(),
type.getJavaType());
if (type.equals(BIGINT)) {
return new LongLiteral(object.toString());
}
if (type.equals(DOUBLE)) {
Double value = (Double) object;
// WARNING: the ORC predicate code depends on NaN and infinity not appearing in a tuple
// domain, so
// if you remove this, you will need to update the TupleDomainOrcPredicate
if (value.isNaN()) {
return new FunctionCall(new QualifiedName("nan"), ImmutableList.<Expression>of());
} else if (value.equals(Double.NEGATIVE_INFINITY)) {
return ArithmeticUnaryExpression.negative(
new FunctionCall(new QualifiedName("infinity"), ImmutableList.<Expression>of()));
} else if (value.equals(Double.POSITIVE_INFINITY)) {
return new FunctionCall(new QualifiedName("infinity"), ImmutableList.<Expression>of());
} else {
return new DoubleLiteral(object.toString());
}
}
if (type instanceof VarcharType) {
if (object instanceof String) {
object = Slices.utf8Slice((String) object);
}
if (object instanceof Slice) {
Slice value = (Slice) object;
int length = SliceUtf8.countCodePoints(value);
if (length == ((VarcharType) type).getLength()) {
return new StringLiteral(value.toStringUtf8());
}
return new Cast(
new StringLiteral(value.toStringUtf8()), type.getDisplayName(), false, true);
}
throw new IllegalArgumentException(
"object must be instance of Slice or String when type is VARCHAR");
}
if (type.equals(BOOLEAN)) {
return new BooleanLiteral(object.toString());
}
if (object instanceof Block) {
SliceOutput output = new DynamicSliceOutput(((Block) object).getSizeInBytes());
BlockSerdeUtil.writeBlock(output, (Block) object);
object = output.slice();
// This if condition will evaluate to true: object instanceof Slice && !type.equals(VARCHAR)
}
if (object instanceof Slice) {
// HACK: we need to serialize VARBINARY in a format that can be embedded in an expression to
// be
// able to encode it in the plan that gets sent to workers.
// We do this by transforming the in-memory varbinary into a call to
// from_base64(<base64-encoded value>)
FunctionCall fromBase64 =
new FunctionCall(
new QualifiedName("from_base64"),
ImmutableList.of(
new StringLiteral(VarbinaryFunctions.toBase64((Slice) object).toStringUtf8())));
Signature signature = FunctionRegistry.getMagicLiteralFunctionSignature(type);
return new FunctionCall(new QualifiedName(signature.getName()), ImmutableList.of(fromBase64));
}
Signature signature = FunctionRegistry.getMagicLiteralFunctionSignature(type);
Expression rawLiteral = toExpression(object, FunctionRegistry.typeForMagicLiteral(type));
return new FunctionCall(new QualifiedName(signature.getName()), ImmutableList.of(rawLiteral));
}
public UncompressedBlock build() {
checkState(!positionOffsets.isEmpty(), "Cannot build an empty block");
return new UncompressedBlock(
positionOffsets.size(), new TupleInfo(type), sliceOutput.slice());
}
@Override
public int getSizeInBytes() {
long arraysSizeInBytes = (Integer.BYTES + Byte.BYTES) * (long) positions;
return intSaturatedCast(sliceOutput.size() + arraysSizeInBytes);
}
@Override
public int getRetainedSizeInBytes() {
return intSaturatedCast(
INSTANCE_SIZE + sliceOutput.getRetainedSize() + arraysRetainedSizeInBytes);
}
@Override
public void appendTo(Slice slice, int offset, SliceOutput sliceOutput) {
// copy full value including length
int length = getLength(slice, offset);
sliceOutput.writeBytes(slice, offset, length);
}
@Override
public int writeSlice(SliceOutput sliceOutput, Slice value, int offset, int length) {
sliceOutput.writeInt(length);
sliceOutput.writeBytes(value, offset, length);
return length + SIZE_OF_INT;
}
@Override
public BlockBuilder writeByte(int value) {
sliceOutput.writeByte(value);
currentEntrySize += SIZE_OF_BYTE;
return this;
}
@Override
public BlockBuilder writeInt(int value) {
sliceOutput.writeInt(value);
currentEntrySize += SIZE_OF_INT;
return this;
}
@Override
public BlockBuilder writeLong(long value) {
sliceOutput.writeLong(value);
currentEntrySize += SIZE_OF_LONG;
return this;
}
private static Slice blockToSlice(Block block) {
// This function is strictly for testing use only
SliceOutput sliceOutput = new DynamicSliceOutput(1000);
BlockSerdeUtil.writeBlock(sliceOutput, block.copyRegion(0, block.getPositionCount()));
return sliceOutput.slice();
}
@Override
public BlockBuilder writeBytes(Slice source, int sourceIndex, int length) {
sliceOutput.writeBytes(source, sourceIndex, length);
currentEntrySize += length;
return this;
}