@Benchmark
 public Page benchmark() {
   PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(prestoType));
   int count = processor.process(null, inputPage, 0, inputPage.getPositionCount(), pageBuilder);
   checkState(count == inputPage.getPositionCount());
   return pageBuilder.build();
 }
  @Override
  public ConnectorPageSource createPageSource(
      ConnectorTransactionHandle transactionHandle,
      ConnectorSession session,
      ConnectorSplit split,
      List<ColumnHandle> columns) {
    InternalTable table = getInternalTable(transactionHandle, session, split, columns);

    List<Integer> channels = new ArrayList<>();
    for (ColumnHandle column : columns) {
      String columnName =
          checkType(column, InformationSchemaColumnHandle.class, "column").getColumnName();
      int columnIndex = table.getColumnIndex(columnName);
      channels.add(columnIndex);
    }

    ImmutableList.Builder<Page> pages = ImmutableList.builder();
    for (Page page : table.getPages()) {
      Block[] blocks = new Block[channels.size()];
      for (int index = 0; index < blocks.length; index++) {
        blocks[index] = page.getBlock(channels.get(index));
      }
      pages.add(new Page(page.getPositionCount(), blocks));
    }
    return new FixedPageSource(pages.build());
  }
Esempio n. 3
0
 private static Page rearrangePage(Page page, int[] channels) {
   Block[] newBlocks = new Block[channels.length];
   for (int i = 0; i < channels.length; i++) {
     newBlocks[i] = page.getBlock(channels[i]);
   }
   return new Page(page.getPositionCount(), newBlocks);
 }
    public ListenableFuture<?> partitionPage(Page page) {
      requireNonNull(page, "page is null");

      Page partitionFunctionArgs = getPartitionFunctionArguments(page);
      for (int position = 0; position < page.getPositionCount(); position++) {
        if (nullChannel.isPresent() && page.getBlock(nullChannel.getAsInt()).isNull(position)) {
          for (PageBuilder pageBuilder : pageBuilders) {
            pageBuilder.declarePosition();

            for (int channel = 0; channel < sourceTypes.size(); channel++) {
              Type type = sourceTypes.get(channel);
              type.appendTo(page.getBlock(channel), position, pageBuilder.getBlockBuilder(channel));
            }
          }
        } else {
          int partition = partitionFunction.getPartition(partitionFunctionArgs, position);

          PageBuilder pageBuilder = pageBuilders.get(partition);
          pageBuilder.declarePosition();

          for (int channel = 0; channel < sourceTypes.size(); channel++) {
            Type type = sourceTypes.get(channel);
            type.appendTo(page.getBlock(channel), position, pageBuilder.getBlockBuilder(channel));
          }
        }
      }
      return flush(false);
    }
Esempio n. 5
0
 @Override
 public Page getOutput() {
   Page page = exchangeClient.pollPage();
   if (page != null) {
     operatorContext.recordGeneratedInput(page.getSizeInBytes(), page.getPositionCount());
   }
   return page;
 }
Esempio n. 6
0
 @Override
 public Page getOutput() {
   if (!pages.hasNext()) {
     return null;
   }
   Page page = pages.next();
   if (page != null) {
     operatorContext.recordGeneratedInput(page.getSizeInBytes(), page.getPositionCount());
   }
   return page;
 }
 private Page getPartitionFunctionArguments(Page page) {
   Block[] blocks = new Block[partitionChannels.size()];
   for (int i = 0; i < blocks.length; i++) {
     Optional<Block> partitionConstant = partitionConstants.get(i);
     if (partitionConstant.isPresent()) {
       blocks[i] = new RunLengthEncodedBlock(partitionConstant.get(), page.getPositionCount());
     } else {
       blocks[i] = page.getBlock(partitionChannels.get(i));
     }
   }
   return new Page(page.getPositionCount(), blocks);
 }
Esempio n. 8
0
  private static boolean executeFilterWithNoInputColumns(Operator operator) {
    Page page = getAtMostOnePage(operator, ZERO_CHANNEL_PAGE);

    boolean value;
    if (page != null) {
      assertEquals(page.getPositionCount(), 1);
      assertEquals(page.getChannelCount(), 0);
      value = true;
    } else {
      value = false;
    }
    return value;
  }
Esempio n. 9
0
  private Object selectSingleValue(Operator operator) {
    Page output = getAtMostOnePage(operator, SOURCE_PAGE);

    assertNotNull(output);
    assertEquals(output.getPositionCount(), 1);
    assertEquals(output.getChannelCount(), 1);
    Type type = operator.getTypes().get(0);

    Block block = output.getBlock(0);
    assertEquals(block.getPositionCount(), 1);

    return type.getObjectValue(session.toConnectorSession(), block, 0);
  }
  @Override
  public void addInput(Page page) {
    requireNonNull(page, "page is null");
    checkState(isBlocked().isDone(), "output is already blocked");

    if (page.getPositionCount() == 0) {
      return;
    }

    page = pagePreprocessor.apply(page);
    blocked = partitionFunction.partitionPage(page);

    operatorContext.recordGeneratedOutput(page.getSizeInBytes(), page.getPositionCount());
  }
Esempio n. 11
0
  private static boolean executeFilter(Operator operator) {
    Page page = getAtMostOnePage(operator, SOURCE_PAGE);

    boolean value;
    if (page != null) {
      assertEquals(page.getPositionCount(), 1);
      assertEquals(page.getChannelCount(), 1);

      assertTrue(operator.getTypes().get(0).getBoolean(page.getBlock(0), 0));
      value = true;
    } else {
      value = false;
    }
    return value;
  }
Esempio n. 12
0
  /**
   * @return the unused section of the page, or null if fully applied. pagesIndex guaranteed to have
   *     at least one row after this method returns
   */
  private Page updatePagesIndex(Page page) {
    checkArgument(page.getPositionCount() > 0);

    // TODO: Fix pagesHashStrategy to allow specifying channels for comparison, it currently
    // requires us to rearrange the right side blocks in consecutive channel order
    Page preGroupedPage = rearrangePage(page, preGroupedChannels);
    if (pagesIndex.getPositionCount() == 0
        || pagesIndex.positionEqualsRow(
            preGroupedPartitionHashStrategy, 0, 0, preGroupedPage.getBlocks())) {
      // Find the position where the pre-grouped columns change
      int groupEnd = findGroupEnd(preGroupedPage, preGroupedPartitionHashStrategy, 0);

      // Add the section of the page that contains values for the current group
      pagesIndex.addPage(page.getRegion(0, groupEnd));

      if (page.getPositionCount() - groupEnd > 0) {
        // Save the remaining page, which may contain multiple partitions
        return page.getRegion(groupEnd, page.getPositionCount() - groupEnd);
      } else {
        // Page fully consumed
        return null;
      }
    } else {
      // We had previous results buffered, but the new page starts with new group values
      return page;
    }
  }
Esempio n. 13
0
 @SuppressWarnings("NumericCastThatLosesPrecision")
 @Override
 public int getBucket(Page page, int position) {
   long hash = 0;
   for (Block block : page.getBlocks()) {
     long value = BIGINT.getLong(block, position);
     hash = (hash * 31) + XxHash64.hash(value);
   }
   int value = (int) (hash & Integer.MAX_VALUE);
   return value % bucketCount;
 }
    public ListenableFuture<?> flush(boolean force) {
      // add all full pages to output buffer
      List<ListenableFuture<?>> blockedFutures = new ArrayList<>();
      for (int partition = 0; partition < pageBuilders.size(); partition++) {
        PageBuilder partitionPageBuilder = pageBuilders.get(partition);
        if (!partitionPageBuilder.isEmpty() && (force || partitionPageBuilder.isFull())) {
          Page pagePartition = partitionPageBuilder.build();
          partitionPageBuilder.reset();

          blockedFutures.add(outputBuffer.enqueue(partition, pagePartition));
          pagesAdded.incrementAndGet();
          rowsAdded.addAndGet(pagePartition.getPositionCount());
        }
      }
      ListenableFuture<?> future = Futures.allAsList(blockedFutures);
      if (future.isDone()) {
        return NOT_BLOCKED;
      }
      return future;
    }
Esempio n. 15
0
File: Row.java Progetto: cdma/presto
  public static Row extractRow(Page page, int position, List<Type> types) {
    checkArgument(page.getChannelCount() == types.size(), "channelCount does not match");
    checkArgument(
        position < page.getPositionCount(),
        "Requested position %s from a page with positionCount %s ",
        position,
        page.getPositionCount());

    RowBuilder rowBuilder = new RowBuilder();
    for (int channel = 0; channel < page.getChannelCount(); channel++) {
      Block block = page.getBlock(channel);
      Type type = types.get(channel);
      int size;
      Object value = getNativeContainerValue(type, block, position);
      if (value == null) {
        size = SIZE_OF_BYTE;
      } else if (type.getJavaType() == boolean.class) {
        size = SIZE_OF_BYTE;
      } else if (type.getJavaType() == long.class) {
        size = SIZE_OF_LONG;
      } else if (type.getJavaType() == double.class) {
        size = SIZE_OF_DOUBLE;
      } else if (type.getJavaType() == Slice.class) {
        size = ((Slice) value).length();
      } else if (type.getJavaType() == Block.class) {
        size = ((Block) value).getSizeInBytes();
      } else {
        throw new AssertionError("Unimplemented type: " + type);
      }
      rowBuilder.add(nativeContainerToOrcValue(type, value), size);
    }
    Row row = rowBuilder.build();
    verify(
        row.getColumns().size() == types.size(),
        "Column count in row: %s Expected column count: %s",
        row.getColumns().size(),
        types.size());
    return row;
  }
Esempio n. 16
0
  // Assumes input grouped on relevant pagesHashStrategy columns
  private static int findGroupEnd(
      Page page, PagesHashStrategy pagesHashStrategy, int startPosition) {
    checkArgument(page.getPositionCount() > 0, "Must have at least one position");
    checkPositionIndex(startPosition, page.getPositionCount(), "startPosition out of bounds");

    // Short circuit if the whole page has the same value
    if (pagesHashStrategy.rowEqualsRow(
        startPosition, page.getBlocks(), page.getPositionCount() - 1, page.getBlocks())) {
      return page.getPositionCount();
    }

    // TODO: do position binary search
    int endPosition = startPosition + 1;
    while (endPosition < page.getPositionCount()
        && pagesHashStrategy.rowEqualsRow(
            endPosition - 1, page.getBlocks(), endPosition, page.getBlocks())) {
      endPosition++;
    }
    return endPosition;
  }
Esempio n. 17
0
  @Override
  public void addInput(Page page) {
    checkState(state == State.NEEDS_INPUT, "Operator can not take input at this time");
    requireNonNull(page, "page is null");
    checkState(pendingInput == null, "Operator already has pending input");

    if (page.getPositionCount() == 0) {
      return;
    }

    pendingInput = page;
    if (processPendingInput()) {
      state = State.HAS_OUTPUT;
    }
    operatorContext.setMemoryReservation(pagesIndex.getEstimatedSize().toBytes());
  }
  @Override
  public void addInput(Page page) {
    requireNonNull(page, "page is null");
    checkState(!finishing, "Operator is finishing");
    checkState(channelSet != null, "Set has not been built yet");
    checkState(outputPage == null, "Operator still has pending output");

    // create the block builder for the new boolean column
    // we know the exact size required for the block
    BlockBuilder blockBuilder = BOOLEAN.createFixedSizeBlockBuilder(page.getPositionCount());

    Page probeJoinPage = new Page(page.getBlock(probeJoinChannel));

    // update hashing strategy to use probe cursor
    for (int position = 0; position < page.getPositionCount(); position++) {
      if (probeJoinPage.getBlock(0).isNull(position)) {
        throw new PrestoException(
            NOT_SUPPORTED,
            "NULL values are not allowed on the probe side of SemiJoin operator. See the query plan for details.");
      } else {
        boolean contains = channelSet.contains(position, probeJoinPage);
        if (!contains && channelSet.containsNull()) {
          blockBuilder.appendNull();
        } else {
          BOOLEAN.writeBoolean(blockBuilder, contains);
        }
      }
    }

    // add the new boolean column to the page
    Block[] sourceBlocks = page.getBlocks();
    Block[] outputBlocks =
        new Block[sourceBlocks.length + 1]; // +1 for the single boolean output channel

    System.arraycopy(sourceBlocks, 0, outputBlocks, 0, sourceBlocks.length);
    outputBlocks[sourceBlocks.length] = blockBuilder.build();

    outputPage = new Page(outputBlocks);
  }
 private static void assertPageEquals(Page expectedPage, Page actualPage) {
   assertEquals(actualPage.getPositionCount(), expectedPage.getPositionCount());
   assertEquals(actualPage.getChannelCount(), expectedPage.getChannelCount());
 }