@Benchmark
public Page benchmark() {
PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(prestoType));
int count = processor.process(null, inputPage, 0, inputPage.getPositionCount(), pageBuilder);
checkState(count == inputPage.getPositionCount());
return pageBuilder.build();
}
public PagePartitioner(
PartitionFunction partitionFunction,
List<Integer> partitionChannels,
List<Optional<NullableValue>> partitionConstants,
OptionalInt nullChannel,
OutputBuffer outputBuffer,
List<Type> sourceTypes,
DataSize maxMemory) {
this.partitionFunction = requireNonNull(partitionFunction, "partitionFunction is null");
this.partitionChannels = requireNonNull(partitionChannels, "partitionChannels is null");
this.partitionConstants =
requireNonNull(partitionConstants, "partitionConstants is null")
.stream()
.map(constant -> constant.map(NullableValue::asBlock))
.collect(toImmutableList());
this.nullChannel = requireNonNull(nullChannel, "nullChannel is null");
this.outputBuffer = requireNonNull(outputBuffer, "outputBuffer is null");
this.sourceTypes = requireNonNull(sourceTypes, "sourceTypes is null");
int pageSize =
Math.min(
PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES,
((int) maxMemory.toBytes()) / partitionFunction.getPartitionCount());
pageSize = Math.max(1, pageSize);
ImmutableList.Builder<PageBuilder> pageBuilders = ImmutableList.builder();
for (int i = 0; i < partitionFunction.getPartitionCount(); i++) {
pageBuilders.add(PageBuilder.withMaxPageSize(pageSize, sourceTypes));
}
this.pageBuilders = pageBuilders.build();
}
public ListenableFuture<?> flush(boolean force) {
// add all full pages to output buffer
List<ListenableFuture<?>> blockedFutures = new ArrayList<>();
for (int partition = 0; partition < pageBuilders.size(); partition++) {
PageBuilder partitionPageBuilder = pageBuilders.get(partition);
if (!partitionPageBuilder.isEmpty() && (force || partitionPageBuilder.isFull())) {
Page pagePartition = partitionPageBuilder.build();
partitionPageBuilder.reset();
blockedFutures.add(outputBuffer.enqueue(partition, pagePartition));
pagesAdded.incrementAndGet();
rowsAdded.addAndGet(pagePartition.getPositionCount());
}
}
ListenableFuture<?> future = Futures.allAsList(blockedFutures);
if (future.isDone()) {
return NOT_BLOCKED;
}
return future;
}
public ListenableFuture<?> partitionPage(Page page) {
requireNonNull(page, "page is null");
Page partitionFunctionArgs = getPartitionFunctionArguments(page);
for (int position = 0; position < page.getPositionCount(); position++) {
if (nullChannel.isPresent() && page.getBlock(nullChannel.getAsInt()).isNull(position)) {
for (PageBuilder pageBuilder : pageBuilders) {
pageBuilder.declarePosition();
for (int channel = 0; channel < sourceTypes.size(); channel++) {
Type type = sourceTypes.get(channel);
type.appendTo(page.getBlock(channel), position, pageBuilder.getBlockBuilder(channel));
}
}
} else {
int partition = partitionFunction.getPartition(partitionFunctionArgs, position);
PageBuilder pageBuilder = pageBuilders.get(partition);
pageBuilder.declarePosition();
for (int channel = 0; channel < sourceTypes.size(); channel++) {
Type type = sourceTypes.get(channel);
type.appendTo(page.getBlock(channel), position, pageBuilder.getBlockBuilder(channel));
}
}
}
return flush(false);
}
@Setup
public void setup() {
Random random = new Random();
RowExpression[] arguments = new RowExpression[1 + inListCount];
switch (type) {
case StandardTypes.BIGINT:
prestoType = BIGINT;
for (int i = 1; i <= inListCount; i++) {
arguments[i] = constant((long) random.nextInt(), BIGINT);
}
break;
case StandardTypes.DOUBLE:
prestoType = DOUBLE;
for (int i = 1; i <= inListCount; i++) {
arguments[i] = constant(random.nextDouble(), DOUBLE);
}
break;
case StandardTypes.VARCHAR:
prestoType = VARCHAR;
for (int i = 1; i <= inListCount; i++) {
arguments[i] = constant(Slices.utf8Slice(Long.toString(random.nextLong())), VARCHAR);
}
break;
default:
throw new IllegalStateException();
}
arguments[0] = field(0, prestoType);
RowExpression project = field(0, prestoType);
PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(prestoType));
for (int i = 0; i < 10_000; i++) {
pageBuilder.declarePosition();
switch (type) {
case StandardTypes.BIGINT:
BIGINT.writeLong(pageBuilder.getBlockBuilder(0), random.nextInt());
break;
case StandardTypes.DOUBLE:
DOUBLE.writeDouble(pageBuilder.getBlockBuilder(0), random.nextDouble());
break;
case StandardTypes.VARCHAR:
VARCHAR.writeSlice(
pageBuilder.getBlockBuilder(0), Slices.utf8Slice(Long.toString(random.nextLong())));
break;
}
}
inputPage = pageBuilder.build();
RowExpression filter =
call(
new Signature(IN, SCALAR, parseTypeSignature(StandardTypes.BOOLEAN)),
BOOLEAN,
arguments);
processor =
new ExpressionCompiler(MetadataManager.createTestMetadataManager())
.compilePageProcessor(filter, ImmutableList.of(project))
.get();
}
private Page extractOutput() {
// INVARIANT: pagesIndex contains the full grouped & sorted data for one or more partitions
// Iterate through the positions sequentially until we have one full page
while (!pageBuilder.isFull()) {
if (partition == null || !partition.hasNext()) {
int partitionStart = partition == null ? 0 : partition.getPartitionEnd();
if (partitionStart >= pagesIndex.getPositionCount()) {
// Finished all of the partitions in the current pagesIndex
partition = null;
pagesIndex.clear();
// Try to extract more partitions from the pendingInput
if (pendingInput != null && processPendingInput()) {
partitionStart = 0;
} else if (state == State.FINISHING) {
state = State.FINISHED;
// Output the remaining page if we have anything buffered
if (!pageBuilder.isEmpty()) {
Page page = pageBuilder.build();
pageBuilder.reset();
return page;
}
return null;
} else {
state = State.NEEDS_INPUT;
return null;
}
}
int partitionEnd = findGroupEnd(pagesIndex, unGroupedPartitionHashStrategy, partitionStart);
partition =
new WindowPartition(
pagesIndex,
partitionStart,
partitionEnd,
outputChannels,
windowFunctions,
frameInfo,
peerGroupHashStrategy);
}
partition.processNextRow(pageBuilder);
}
Page page = pageBuilder.build();
pageBuilder.reset();
return page;
}
@Test(dataProvider = "hashEnabledValues")
public void testSingleChannel(boolean hashEnabled) throws Exception {
List<Type> joinTypes = ImmutableList.<Type>of(VARCHAR);
List<Integer> joinChannels = Ints.asList(0);
// compile a single channel hash strategy
PagesHashStrategyFactory pagesHashStrategyFactory =
joinCompiler.compilePagesHashStrategyFactory(joinTypes, joinChannels);
// create hash strategy with a single channel blocks -- make sure there is some overlap in
// values
List<Block> channel =
ImmutableList.of(
BlockAssertions.createStringSequenceBlock(10, 20),
BlockAssertions.createStringSequenceBlock(20, 30),
BlockAssertions.createStringSequenceBlock(15, 25));
Optional<Integer> hashChannel = Optional.empty();
List<List<Block>> channels = ImmutableList.of(channel);
if (hashEnabled) {
ImmutableList.Builder<Block> hashChannelBuilder = ImmutableList.builder();
for (Block block : channel) {
hashChannelBuilder.add(TypeUtils.getHashBlock(joinTypes, block));
}
hashChannel = Optional.of(1);
channels = ImmutableList.of(channel, hashChannelBuilder.build());
}
PagesHashStrategy hashStrategy =
pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel);
// verify channel count
assertEquals(hashStrategy.getChannelCount(), 1);
// verify hashStrategy is consistent with equals and hash code from block
for (int leftBlockIndex = 0; leftBlockIndex < channel.size(); leftBlockIndex++) {
Block leftBlock = channel.get(leftBlockIndex);
PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(VARCHAR));
for (int leftBlockPosition = 0;
leftBlockPosition < leftBlock.getPositionCount();
leftBlockPosition++) {
// hash code of position must match block hash
assertEquals(
hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition),
hashPosition(VARCHAR, leftBlock, leftBlockPosition));
// position must be equal to itself
assertTrue(
hashStrategy.positionEqualsPositionIgnoreNulls(
leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
// check equality of every position against every other position in the block
for (int rightBlockIndex = 0; rightBlockIndex < channel.size(); rightBlockIndex++) {
Block rightBlock = channel.get(rightBlockIndex);
for (int rightBlockPosition = 0;
rightBlockPosition < rightBlock.getPositionCount();
rightBlockPosition++) {
boolean expected =
positionEqualsPosition(
VARCHAR, leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
assertEquals(
hashStrategy.positionEqualsRow(
leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)),
expected);
assertEquals(
hashStrategy.rowEqualsRow(
leftBlockPosition,
new Page(leftBlock),
rightBlockPosition,
new Page(rightBlock)),
expected);
assertEquals(
hashStrategy.positionEqualsRowIgnoreNulls(
leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)),
expected);
assertEquals(
hashStrategy.positionEqualsPositionIgnoreNulls(
leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition),
expected);
assertEquals(
hashStrategy.positionEqualsPosition(
leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition),
expected);
}
}
// check equality of every position against every other position in the block cursor
for (int rightBlockIndex = 0; rightBlockIndex < channel.size(); rightBlockIndex++) {
Block rightBlock = channel.get(rightBlockIndex);
for (int rightBlockPosition = 0;
rightBlockPosition < rightBlock.getPositionCount();
rightBlockPosition++) {
boolean expected =
positionEqualsPosition(
VARCHAR, leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
assertEquals(
hashStrategy.positionEqualsRow(
leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)),
expected);
assertEquals(
hashStrategy.rowEqualsRow(
leftBlockPosition,
new Page(leftBlock),
rightBlockPosition,
new Page(rightBlock)),
expected);
assertEquals(
hashStrategy.positionEqualsRowIgnoreNulls(
leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)),
expected);
assertEquals(
hashStrategy.positionEqualsPositionIgnoreNulls(
leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition),
expected);
assertEquals(
hashStrategy.positionEqualsPosition(
leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition),
expected);
}
}
// write position to output block
pageBuilder.declarePosition();
hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
}
// verify output block matches
assertBlockEquals(VARCHAR, pageBuilder.build().getBlock(0), leftBlock);
}
}
@Test(dataProvider = "hashEnabledValues")
public void testMultiChannel(boolean hashEnabled) throws Exception {
// compile a single channel hash strategy
JoinCompiler joinCompiler = new JoinCompiler();
List<Type> types = ImmutableList.<Type>of(VARCHAR, VARCHAR, BIGINT, DOUBLE, BOOLEAN);
List<Type> joinTypes = ImmutableList.<Type>of(VARCHAR, BIGINT, DOUBLE, BOOLEAN);
List<Integer> joinChannels = Ints.asList(1, 2, 3, 4);
// crate hash strategy with a single channel blocks -- make sure there is some overlap in values
List<Block> extraChannel =
ImmutableList.of(
BlockAssertions.createStringSequenceBlock(10, 20),
BlockAssertions.createStringSequenceBlock(20, 30),
BlockAssertions.createStringSequenceBlock(15, 25));
List<Block> varcharChannel =
ImmutableList.of(
BlockAssertions.createStringSequenceBlock(10, 20),
BlockAssertions.createStringSequenceBlock(20, 30),
BlockAssertions.createStringSequenceBlock(15, 25));
List<Block> longChannel =
ImmutableList.of(
BlockAssertions.createLongSequenceBlock(10, 20),
BlockAssertions.createLongSequenceBlock(20, 30),
BlockAssertions.createLongSequenceBlock(15, 25));
List<Block> doubleChannel =
ImmutableList.of(
BlockAssertions.createDoubleSequenceBlock(10, 20),
BlockAssertions.createDoubleSequenceBlock(20, 30),
BlockAssertions.createDoubleSequenceBlock(15, 25));
List<Block> booleanChannel =
ImmutableList.of(
BlockAssertions.createBooleanSequenceBlock(10, 20),
BlockAssertions.createBooleanSequenceBlock(20, 30),
BlockAssertions.createBooleanSequenceBlock(15, 25));
Optional<Integer> hashChannel = Optional.empty();
ImmutableList<List<Block>> channels =
ImmutableList.of(extraChannel, varcharChannel, longChannel, doubleChannel, booleanChannel);
List<Block> precomputedHash = ImmutableList.of();
if (hashEnabled) {
ImmutableList.Builder<Block> hashChannelBuilder = ImmutableList.builder();
for (int i = 0; i < 3; i++) {
hashChannelBuilder.add(
TypeUtils.getHashBlock(
joinTypes,
varcharChannel.get(i),
longChannel.get(i),
doubleChannel.get(i),
booleanChannel.get(i)));
}
hashChannel = Optional.of(5);
precomputedHash = hashChannelBuilder.build();
channels =
ImmutableList.of(
extraChannel,
varcharChannel,
longChannel,
doubleChannel,
booleanChannel,
precomputedHash);
types = ImmutableList.<Type>of(VARCHAR, VARCHAR, BIGINT, DOUBLE, BOOLEAN, BIGINT);
}
PagesHashStrategyFactory pagesHashStrategyFactory =
joinCompiler.compilePagesHashStrategyFactory(types, joinChannels);
PagesHashStrategy hashStrategy =
pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel);
// todo add tests for filter function
PagesHashStrategy expectedHashStrategy =
new SimplePagesHashStrategy(types, channels, joinChannels, hashChannel);
// verify channel count
assertEquals(hashStrategy.getChannelCount(), types.size());
// verify size
long sizeInBytes =
channels.stream().flatMap(List::stream).mapToLong(Block::getRetainedSizeInBytes).sum();
assertEquals(hashStrategy.getSizeInBytes(), sizeInBytes);
// verify hashStrategy is consistent with equals and hash code from block
for (int leftBlockIndex = 0; leftBlockIndex < varcharChannel.size(); leftBlockIndex++) {
PageBuilder pageBuilder = new PageBuilder(types);
Block[] leftBlocks = new Block[4];
leftBlocks[0] = varcharChannel.get(leftBlockIndex);
leftBlocks[1] = longChannel.get(leftBlockIndex);
leftBlocks[2] = doubleChannel.get(leftBlockIndex);
leftBlocks[3] = booleanChannel.get(leftBlockIndex);
int leftPositionCount = varcharChannel.get(leftBlockIndex).getPositionCount();
for (int leftBlockPosition = 0; leftBlockPosition < leftPositionCount; leftBlockPosition++) {
// hash code of position must match block hash
assertEquals(
hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition),
expectedHashStrategy.hashPosition(leftBlockIndex, leftBlockPosition));
// position must be equal to itself
assertTrue(
hashStrategy.positionEqualsPositionIgnoreNulls(
leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
assertTrue(
hashStrategy.positionEqualsPosition(
leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
// check equality of every position against every other position in the block
for (int rightBlockIndex = 0; rightBlockIndex < varcharChannel.size(); rightBlockIndex++) {
Block rightBlock = varcharChannel.get(rightBlockIndex);
for (int rightBlockPosition = 0;
rightBlockPosition < rightBlock.getPositionCount();
rightBlockPosition++) {
assertEquals(
hashStrategy.positionEqualsPositionIgnoreNulls(
leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition),
expectedHashStrategy.positionEqualsPositionIgnoreNulls(
leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition));
assertEquals(
hashStrategy.positionEqualsPosition(
leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition),
expectedHashStrategy.positionEqualsPosition(
leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition));
}
}
// check equality of every position against every other position in the block cursor
for (int rightBlockIndex = 0; rightBlockIndex < varcharChannel.size(); rightBlockIndex++) {
Block[] rightBlocks = new Block[4];
rightBlocks[0] = varcharChannel.get(rightBlockIndex);
rightBlocks[1] = longChannel.get(rightBlockIndex);
rightBlocks[2] = doubleChannel.get(rightBlockIndex);
rightBlocks[3] = booleanChannel.get(rightBlockIndex);
int rightPositionCount = varcharChannel.get(rightBlockIndex).getPositionCount();
for (int rightPosition = 0; rightPosition < rightPositionCount; rightPosition++) {
boolean expected =
expectedHashStrategy.positionEqualsRow(
leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks));
assertEquals(
hashStrategy.positionEqualsRow(
leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)),
expected);
assertEquals(
hashStrategy.rowEqualsRow(
leftBlockPosition, new Page(leftBlocks), rightPosition, new Page(rightBlocks)),
expected);
assertEquals(
hashStrategy.positionEqualsRowIgnoreNulls(
leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)),
expected);
}
}
// write position to output block
pageBuilder.declarePosition();
hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
}
// verify output block matches
Page page = pageBuilder.build();
if (hashEnabled) {
assertPageEquals(
types,
page,
new Page(
extraChannel.get(leftBlockIndex),
varcharChannel.get(leftBlockIndex),
longChannel.get(leftBlockIndex),
doubleChannel.get(leftBlockIndex),
booleanChannel.get(leftBlockIndex),
precomputedHash.get(leftBlockIndex)));
} else {
assertPageEquals(
types,
page,
new Page(
extraChannel.get(leftBlockIndex),
varcharChannel.get(leftBlockIndex),
longChannel.get(leftBlockIndex),
doubleChannel.get(leftBlockIndex),
booleanChannel.get(leftBlockIndex)));
}
}
}
protected void appendTo(PageBuilder pageBuilder, int outputChannelOffset) {
BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(outputChannelOffset);
elementType.appendTo(arrayBlock, position, blockBuilder);
position++;
}