@Benchmark
 public Page benchmark() {
   PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(prestoType));
   int count = processor.process(null, inputPage, 0, inputPage.getPositionCount(), pageBuilder);
   checkState(count == inputPage.getPositionCount());
   return pageBuilder.build();
 }
    public PagePartitioner(
        PartitionFunction partitionFunction,
        List<Integer> partitionChannels,
        List<Optional<NullableValue>> partitionConstants,
        OptionalInt nullChannel,
        OutputBuffer outputBuffer,
        List<Type> sourceTypes,
        DataSize maxMemory) {
      this.partitionFunction = requireNonNull(partitionFunction, "partitionFunction is null");
      this.partitionChannels = requireNonNull(partitionChannels, "partitionChannels is null");
      this.partitionConstants =
          requireNonNull(partitionConstants, "partitionConstants is null")
              .stream()
              .map(constant -> constant.map(NullableValue::asBlock))
              .collect(toImmutableList());
      this.nullChannel = requireNonNull(nullChannel, "nullChannel is null");
      this.outputBuffer = requireNonNull(outputBuffer, "outputBuffer is null");
      this.sourceTypes = requireNonNull(sourceTypes, "sourceTypes is null");

      int pageSize =
          Math.min(
              PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES,
              ((int) maxMemory.toBytes()) / partitionFunction.getPartitionCount());
      pageSize = Math.max(1, pageSize);

      ImmutableList.Builder<PageBuilder> pageBuilders = ImmutableList.builder();
      for (int i = 0; i < partitionFunction.getPartitionCount(); i++) {
        pageBuilders.add(PageBuilder.withMaxPageSize(pageSize, sourceTypes));
      }
      this.pageBuilders = pageBuilders.build();
    }
    public ListenableFuture<?> flush(boolean force) {
      // add all full pages to output buffer
      List<ListenableFuture<?>> blockedFutures = new ArrayList<>();
      for (int partition = 0; partition < pageBuilders.size(); partition++) {
        PageBuilder partitionPageBuilder = pageBuilders.get(partition);
        if (!partitionPageBuilder.isEmpty() && (force || partitionPageBuilder.isFull())) {
          Page pagePartition = partitionPageBuilder.build();
          partitionPageBuilder.reset();

          blockedFutures.add(outputBuffer.enqueue(partition, pagePartition));
          pagesAdded.incrementAndGet();
          rowsAdded.addAndGet(pagePartition.getPositionCount());
        }
      }
      ListenableFuture<?> future = Futures.allAsList(blockedFutures);
      if (future.isDone()) {
        return NOT_BLOCKED;
      }
      return future;
    }
    public ListenableFuture<?> partitionPage(Page page) {
      requireNonNull(page, "page is null");

      Page partitionFunctionArgs = getPartitionFunctionArguments(page);
      for (int position = 0; position < page.getPositionCount(); position++) {
        if (nullChannel.isPresent() && page.getBlock(nullChannel.getAsInt()).isNull(position)) {
          for (PageBuilder pageBuilder : pageBuilders) {
            pageBuilder.declarePosition();

            for (int channel = 0; channel < sourceTypes.size(); channel++) {
              Type type = sourceTypes.get(channel);
              type.appendTo(page.getBlock(channel), position, pageBuilder.getBlockBuilder(channel));
            }
          }
        } else {
          int partition = partitionFunction.getPartition(partitionFunctionArgs, position);

          PageBuilder pageBuilder = pageBuilders.get(partition);
          pageBuilder.declarePosition();

          for (int channel = 0; channel < sourceTypes.size(); channel++) {
            Type type = sourceTypes.get(channel);
            type.appendTo(page.getBlock(channel), position, pageBuilder.getBlockBuilder(channel));
          }
        }
      }
      return flush(false);
    }
  @Setup
  public void setup() {
    Random random = new Random();
    RowExpression[] arguments = new RowExpression[1 + inListCount];
    switch (type) {
      case StandardTypes.BIGINT:
        prestoType = BIGINT;
        for (int i = 1; i <= inListCount; i++) {
          arguments[i] = constant((long) random.nextInt(), BIGINT);
        }
        break;
      case StandardTypes.DOUBLE:
        prestoType = DOUBLE;
        for (int i = 1; i <= inListCount; i++) {
          arguments[i] = constant(random.nextDouble(), DOUBLE);
        }
        break;
      case StandardTypes.VARCHAR:
        prestoType = VARCHAR;
        for (int i = 1; i <= inListCount; i++) {
          arguments[i] = constant(Slices.utf8Slice(Long.toString(random.nextLong())), VARCHAR);
        }
        break;
      default:
        throw new IllegalStateException();
    }

    arguments[0] = field(0, prestoType);
    RowExpression project = field(0, prestoType);

    PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(prestoType));
    for (int i = 0; i < 10_000; i++) {
      pageBuilder.declarePosition();

      switch (type) {
        case StandardTypes.BIGINT:
          BIGINT.writeLong(pageBuilder.getBlockBuilder(0), random.nextInt());
          break;
        case StandardTypes.DOUBLE:
          DOUBLE.writeDouble(pageBuilder.getBlockBuilder(0), random.nextDouble());
          break;
        case StandardTypes.VARCHAR:
          VARCHAR.writeSlice(
              pageBuilder.getBlockBuilder(0), Slices.utf8Slice(Long.toString(random.nextLong())));
          break;
      }
    }
    inputPage = pageBuilder.build();

    RowExpression filter =
        call(
            new Signature(IN, SCALAR, parseTypeSignature(StandardTypes.BOOLEAN)),
            BOOLEAN,
            arguments);

    processor =
        new ExpressionCompiler(MetadataManager.createTestMetadataManager())
            .compilePageProcessor(filter, ImmutableList.of(project))
            .get();
  }
Ejemplo n.º 6
0
  private Page extractOutput() {
    // INVARIANT: pagesIndex contains the full grouped & sorted data for one or more partitions

    // Iterate through the positions sequentially until we have one full page
    while (!pageBuilder.isFull()) {
      if (partition == null || !partition.hasNext()) {
        int partitionStart = partition == null ? 0 : partition.getPartitionEnd();

        if (partitionStart >= pagesIndex.getPositionCount()) {
          // Finished all of the partitions in the current pagesIndex
          partition = null;
          pagesIndex.clear();

          // Try to extract more partitions from the pendingInput
          if (pendingInput != null && processPendingInput()) {
            partitionStart = 0;
          } else if (state == State.FINISHING) {
            state = State.FINISHED;
            // Output the remaining page if we have anything buffered
            if (!pageBuilder.isEmpty()) {
              Page page = pageBuilder.build();
              pageBuilder.reset();
              return page;
            }
            return null;
          } else {
            state = State.NEEDS_INPUT;
            return null;
          }
        }

        int partitionEnd = findGroupEnd(pagesIndex, unGroupedPartitionHashStrategy, partitionStart);
        partition =
            new WindowPartition(
                pagesIndex,
                partitionStart,
                partitionEnd,
                outputChannels,
                windowFunctions,
                frameInfo,
                peerGroupHashStrategy);
      }

      partition.processNextRow(pageBuilder);
    }

    Page page = pageBuilder.build();
    pageBuilder.reset();
    return page;
  }
Ejemplo n.º 7
0
  @Test(dataProvider = "hashEnabledValues")
  public void testSingleChannel(boolean hashEnabled) throws Exception {
    List<Type> joinTypes = ImmutableList.<Type>of(VARCHAR);
    List<Integer> joinChannels = Ints.asList(0);

    // compile a single channel hash strategy
    PagesHashStrategyFactory pagesHashStrategyFactory =
        joinCompiler.compilePagesHashStrategyFactory(joinTypes, joinChannels);

    // create hash strategy with a single channel blocks -- make sure there is some overlap in
    // values
    List<Block> channel =
        ImmutableList.of(
            BlockAssertions.createStringSequenceBlock(10, 20),
            BlockAssertions.createStringSequenceBlock(20, 30),
            BlockAssertions.createStringSequenceBlock(15, 25));

    Optional<Integer> hashChannel = Optional.empty();
    List<List<Block>> channels = ImmutableList.of(channel);
    if (hashEnabled) {
      ImmutableList.Builder<Block> hashChannelBuilder = ImmutableList.builder();
      for (Block block : channel) {
        hashChannelBuilder.add(TypeUtils.getHashBlock(joinTypes, block));
      }
      hashChannel = Optional.of(1);
      channels = ImmutableList.of(channel, hashChannelBuilder.build());
    }
    PagesHashStrategy hashStrategy =
        pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel);

    // verify channel count
    assertEquals(hashStrategy.getChannelCount(), 1);

    // verify hashStrategy is consistent with equals and hash code from block
    for (int leftBlockIndex = 0; leftBlockIndex < channel.size(); leftBlockIndex++) {
      Block leftBlock = channel.get(leftBlockIndex);

      PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(VARCHAR));

      for (int leftBlockPosition = 0;
          leftBlockPosition < leftBlock.getPositionCount();
          leftBlockPosition++) {
        // hash code of position must match block hash
        assertEquals(
            hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition),
            hashPosition(VARCHAR, leftBlock, leftBlockPosition));

        // position must be equal to itself
        assertTrue(
            hashStrategy.positionEqualsPositionIgnoreNulls(
                leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));

        // check equality of every position against every other position in the block
        for (int rightBlockIndex = 0; rightBlockIndex < channel.size(); rightBlockIndex++) {
          Block rightBlock = channel.get(rightBlockIndex);
          for (int rightBlockPosition = 0;
              rightBlockPosition < rightBlock.getPositionCount();
              rightBlockPosition++) {
            boolean expected =
                positionEqualsPosition(
                    VARCHAR, leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
            assertEquals(
                hashStrategy.positionEqualsRow(
                    leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)),
                expected);
            assertEquals(
                hashStrategy.rowEqualsRow(
                    leftBlockPosition,
                    new Page(leftBlock),
                    rightBlockPosition,
                    new Page(rightBlock)),
                expected);
            assertEquals(
                hashStrategy.positionEqualsRowIgnoreNulls(
                    leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)),
                expected);
            assertEquals(
                hashStrategy.positionEqualsPositionIgnoreNulls(
                    leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition),
                expected);
            assertEquals(
                hashStrategy.positionEqualsPosition(
                    leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition),
                expected);
          }
        }

        // check equality of every position against every other position in the block cursor
        for (int rightBlockIndex = 0; rightBlockIndex < channel.size(); rightBlockIndex++) {
          Block rightBlock = channel.get(rightBlockIndex);
          for (int rightBlockPosition = 0;
              rightBlockPosition < rightBlock.getPositionCount();
              rightBlockPosition++) {
            boolean expected =
                positionEqualsPosition(
                    VARCHAR, leftBlock, leftBlockPosition, rightBlock, rightBlockPosition);
            assertEquals(
                hashStrategy.positionEqualsRow(
                    leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)),
                expected);
            assertEquals(
                hashStrategy.rowEqualsRow(
                    leftBlockPosition,
                    new Page(leftBlock),
                    rightBlockPosition,
                    new Page(rightBlock)),
                expected);
            assertEquals(
                hashStrategy.positionEqualsRowIgnoreNulls(
                    leftBlockIndex, leftBlockPosition, rightBlockPosition, new Page(rightBlock)),
                expected);
            assertEquals(
                hashStrategy.positionEqualsPositionIgnoreNulls(
                    leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition),
                expected);
            assertEquals(
                hashStrategy.positionEqualsPosition(
                    leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition),
                expected);
          }
        }

        // write position to output block
        pageBuilder.declarePosition();
        hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
      }

      // verify output block matches
      assertBlockEquals(VARCHAR, pageBuilder.build().getBlock(0), leftBlock);
    }
  }
Ejemplo n.º 8
0
  @Test(dataProvider = "hashEnabledValues")
  public void testMultiChannel(boolean hashEnabled) throws Exception {
    // compile a single channel hash strategy
    JoinCompiler joinCompiler = new JoinCompiler();
    List<Type> types = ImmutableList.<Type>of(VARCHAR, VARCHAR, BIGINT, DOUBLE, BOOLEAN);
    List<Type> joinTypes = ImmutableList.<Type>of(VARCHAR, BIGINT, DOUBLE, BOOLEAN);
    List<Integer> joinChannels = Ints.asList(1, 2, 3, 4);

    // crate hash strategy with a single channel blocks -- make sure there is some overlap in values
    List<Block> extraChannel =
        ImmutableList.of(
            BlockAssertions.createStringSequenceBlock(10, 20),
            BlockAssertions.createStringSequenceBlock(20, 30),
            BlockAssertions.createStringSequenceBlock(15, 25));
    List<Block> varcharChannel =
        ImmutableList.of(
            BlockAssertions.createStringSequenceBlock(10, 20),
            BlockAssertions.createStringSequenceBlock(20, 30),
            BlockAssertions.createStringSequenceBlock(15, 25));
    List<Block> longChannel =
        ImmutableList.of(
            BlockAssertions.createLongSequenceBlock(10, 20),
            BlockAssertions.createLongSequenceBlock(20, 30),
            BlockAssertions.createLongSequenceBlock(15, 25));
    List<Block> doubleChannel =
        ImmutableList.of(
            BlockAssertions.createDoubleSequenceBlock(10, 20),
            BlockAssertions.createDoubleSequenceBlock(20, 30),
            BlockAssertions.createDoubleSequenceBlock(15, 25));
    List<Block> booleanChannel =
        ImmutableList.of(
            BlockAssertions.createBooleanSequenceBlock(10, 20),
            BlockAssertions.createBooleanSequenceBlock(20, 30),
            BlockAssertions.createBooleanSequenceBlock(15, 25));

    Optional<Integer> hashChannel = Optional.empty();
    ImmutableList<List<Block>> channels =
        ImmutableList.of(extraChannel, varcharChannel, longChannel, doubleChannel, booleanChannel);
    List<Block> precomputedHash = ImmutableList.of();
    if (hashEnabled) {
      ImmutableList.Builder<Block> hashChannelBuilder = ImmutableList.builder();
      for (int i = 0; i < 3; i++) {
        hashChannelBuilder.add(
            TypeUtils.getHashBlock(
                joinTypes,
                varcharChannel.get(i),
                longChannel.get(i),
                doubleChannel.get(i),
                booleanChannel.get(i)));
      }
      hashChannel = Optional.of(5);
      precomputedHash = hashChannelBuilder.build();
      channels =
          ImmutableList.of(
              extraChannel,
              varcharChannel,
              longChannel,
              doubleChannel,
              booleanChannel,
              precomputedHash);
      types = ImmutableList.<Type>of(VARCHAR, VARCHAR, BIGINT, DOUBLE, BOOLEAN, BIGINT);
    }

    PagesHashStrategyFactory pagesHashStrategyFactory =
        joinCompiler.compilePagesHashStrategyFactory(types, joinChannels);
    PagesHashStrategy hashStrategy =
        pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel);
    // todo add tests for filter function
    PagesHashStrategy expectedHashStrategy =
        new SimplePagesHashStrategy(types, channels, joinChannels, hashChannel);

    // verify channel count
    assertEquals(hashStrategy.getChannelCount(), types.size());
    // verify size
    long sizeInBytes =
        channels.stream().flatMap(List::stream).mapToLong(Block::getRetainedSizeInBytes).sum();
    assertEquals(hashStrategy.getSizeInBytes(), sizeInBytes);

    // verify hashStrategy is consistent with equals and hash code from block
    for (int leftBlockIndex = 0; leftBlockIndex < varcharChannel.size(); leftBlockIndex++) {
      PageBuilder pageBuilder = new PageBuilder(types);

      Block[] leftBlocks = new Block[4];
      leftBlocks[0] = varcharChannel.get(leftBlockIndex);
      leftBlocks[1] = longChannel.get(leftBlockIndex);
      leftBlocks[2] = doubleChannel.get(leftBlockIndex);
      leftBlocks[3] = booleanChannel.get(leftBlockIndex);

      int leftPositionCount = varcharChannel.get(leftBlockIndex).getPositionCount();
      for (int leftBlockPosition = 0; leftBlockPosition < leftPositionCount; leftBlockPosition++) {
        // hash code of position must match block hash
        assertEquals(
            hashStrategy.hashPosition(leftBlockIndex, leftBlockPosition),
            expectedHashStrategy.hashPosition(leftBlockIndex, leftBlockPosition));

        // position must be equal to itself
        assertTrue(
            hashStrategy.positionEqualsPositionIgnoreNulls(
                leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));
        assertTrue(
            hashStrategy.positionEqualsPosition(
                leftBlockIndex, leftBlockPosition, leftBlockIndex, leftBlockPosition));

        // check equality of every position against every other position in the block
        for (int rightBlockIndex = 0; rightBlockIndex < varcharChannel.size(); rightBlockIndex++) {
          Block rightBlock = varcharChannel.get(rightBlockIndex);
          for (int rightBlockPosition = 0;
              rightBlockPosition < rightBlock.getPositionCount();
              rightBlockPosition++) {
            assertEquals(
                hashStrategy.positionEqualsPositionIgnoreNulls(
                    leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition),
                expectedHashStrategy.positionEqualsPositionIgnoreNulls(
                    leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition));
            assertEquals(
                hashStrategy.positionEqualsPosition(
                    leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition),
                expectedHashStrategy.positionEqualsPosition(
                    leftBlockIndex, leftBlockPosition, rightBlockIndex, rightBlockPosition));
          }
        }

        // check equality of every position against every other position in the block cursor
        for (int rightBlockIndex = 0; rightBlockIndex < varcharChannel.size(); rightBlockIndex++) {
          Block[] rightBlocks = new Block[4];
          rightBlocks[0] = varcharChannel.get(rightBlockIndex);
          rightBlocks[1] = longChannel.get(rightBlockIndex);
          rightBlocks[2] = doubleChannel.get(rightBlockIndex);
          rightBlocks[3] = booleanChannel.get(rightBlockIndex);

          int rightPositionCount = varcharChannel.get(rightBlockIndex).getPositionCount();
          for (int rightPosition = 0; rightPosition < rightPositionCount; rightPosition++) {
            boolean expected =
                expectedHashStrategy.positionEqualsRow(
                    leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks));

            assertEquals(
                hashStrategy.positionEqualsRow(
                    leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)),
                expected);
            assertEquals(
                hashStrategy.rowEqualsRow(
                    leftBlockPosition, new Page(leftBlocks), rightPosition, new Page(rightBlocks)),
                expected);
            assertEquals(
                hashStrategy.positionEqualsRowIgnoreNulls(
                    leftBlockIndex, leftBlockPosition, rightPosition, new Page(rightBlocks)),
                expected);
          }
        }

        // write position to output block
        pageBuilder.declarePosition();
        hashStrategy.appendTo(leftBlockIndex, leftBlockPosition, pageBuilder, 0);
      }

      // verify output block matches
      Page page = pageBuilder.build();
      if (hashEnabled) {
        assertPageEquals(
            types,
            page,
            new Page(
                extraChannel.get(leftBlockIndex),
                varcharChannel.get(leftBlockIndex),
                longChannel.get(leftBlockIndex),
                doubleChannel.get(leftBlockIndex),
                booleanChannel.get(leftBlockIndex),
                precomputedHash.get(leftBlockIndex)));
      } else {
        assertPageEquals(
            types,
            page,
            new Page(
                extraChannel.get(leftBlockIndex),
                varcharChannel.get(leftBlockIndex),
                longChannel.get(leftBlockIndex),
                doubleChannel.get(leftBlockIndex),
                booleanChannel.get(leftBlockIndex)));
      }
    }
  }
 protected void appendTo(PageBuilder pageBuilder, int outputChannelOffset) {
   BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(outputChannelOffset);
   elementType.appendTo(arrayBlock, position, blockBuilder);
   position++;
 }