public PartitionedOutputOperator(
OperatorContext operatorContext,
List<Type> sourceTypes,
Function<Page, Page> pagePreprocessor,
PartitionFunction partitionFunction,
List<Integer> partitionChannels,
List<Optional<NullableValue>> partitionConstants,
OptionalInt nullChannel,
OutputBuffer outputBuffer,
DataSize maxMemory) {
this.operatorContext = requireNonNull(operatorContext, "operatorContext is null");
this.pagePreprocessor = requireNonNull(pagePreprocessor, "pagePreprocessor is null");
this.partitionFunction =
new PagePartitioner(
partitionFunction,
partitionChannels,
partitionConstants,
nullChannel,
outputBuffer,
sourceTypes,
maxMemory);
operatorContext.setInfoSupplier(this::getInfo);
// TODO: We should try to make this more accurate
// Recalculating the retained size of all the PageBuilders is somewhat expensive,
// so we only do it once here rather than in addInput(), and assume that the size will be
// constant.
operatorContext
.getSystemMemoryContext()
.newLocalMemoryContext()
.setBytes(this.partitionFunction.getRetainedSizeInBytes());
}
public CounterStat getOutputPositions() {
OperatorContext inputOperator = getLast(operatorContexts, null);
if (inputOperator != null) {
return inputOperator.getOutputPositions();
} else {
return new CounterStat();
}
}
public CounterStat getInputDataSize() {
OperatorContext inputOperator = getFirst(operatorContexts, null);
if (inputOperator != null) {
return inputOperator.getInputDataSize();
} else {
return new CounterStat();
}
}
public OperatorContext addOperatorContext(
int operatorId, PlanNodeId planNodeId, String operatorType) {
checkArgument(operatorId >= 0, "operatorId is negative");
for (OperatorContext operatorContext : operatorContexts) {
checkArgument(
operatorId != operatorContext.getOperatorId(),
"A context already exists for operatorId %s",
operatorId);
}
OperatorContext operatorContext =
new OperatorContext(operatorId, planNodeId, operatorType, this, executor);
operatorContexts.add(operatorContext);
return operatorContext;
}
@Override
public void addInput(Page page) {
checkNotNull(page, "page is null");
checkState(!finished, "Already finished");
inMemoryExchange.addPage(page);
operatorContext.recordGeneratedOutput(page.getDataSize(), page.getPositionCount());
}
@Override
public Page getOutput() {
Page page = exchangeClient.pollPage();
if (page != null) {
operatorContext.recordGeneratedInput(page.getSizeInBytes(), page.getPositionCount());
}
return page;
}
@Override
public Page getOutput() {
if (state == State.NEEDS_INPUT || state == State.FINISHED) {
return null;
}
Page page = extractOutput();
operatorContext.setMemoryReservation(pagesIndex.getEstimatedSize().toBytes());
return page;
}
@Override
public void finish() {
if (finished) {
return;
}
ChannelSet channelSet = channelSetBuilder.build();
setSupplier.setChannelSet(channelSet);
operatorContext.recordGeneratedOutput(channelSet.getEstimatedSize(), channelSet.size());
finished = true;
}
@Override
public Page getOutput() {
if (!pages.hasNext()) {
return null;
}
Page page = pages.next();
if (page != null) {
operatorContext.recordGeneratedInput(page.getSizeInBytes(), page.getPositionCount());
}
return page;
}
public ExchangeOperator(
OperatorContext operatorContext,
List<Type> types,
PlanNodeId sourceId,
ExchangeClient exchangeClient) {
this.operatorContext = checkNotNull(operatorContext, "operatorContext is null");
this.sourceId = checkNotNull(sourceId, "sourceId is null");
this.exchangeClient = checkNotNull(exchangeClient, "exchangeClient is null");
this.types = checkNotNull(types, "types is null");
operatorContext.setInfoSupplier(exchangeClient::getStatus);
}
@Override
public void addInput(Page page) {
requireNonNull(page, "page is null");
checkState(isBlocked().isDone(), "output is already blocked");
if (page.getPositionCount() == 0) {
return;
}
page = pagePreprocessor.apply(page);
blocked = partitionFunction.partitionPage(page);
operatorContext.recordGeneratedOutput(page.getSizeInBytes(), page.getPositionCount());
}
@Override
public void addInput(Page page) {
checkState(state == State.NEEDS_INPUT, "Operator can not take input at this time");
requireNonNull(page, "page is null");
checkState(pendingInput == null, "Operator already has pending input");
if (page.getPositionCount() == 0) {
return;
}
pendingInput = page;
if (processPendingInput()) {
state = State.HAS_OUTPUT;
}
operatorContext.setMemoryReservation(pagesIndex.getEstimatedSize().toBytes());
}
public InMemoryJoinHash(
LongArrayList addresses,
PagesHashStrategy pagesHashStrategy,
OperatorContext operatorContext) {
this.addresses = checkNotNull(addresses, "addresses is null");
this.pagesHashStrategy = checkNotNull(pagesHashStrategy, "pagesHashStrategy is null");
this.channelCount = pagesHashStrategy.getChannelCount();
checkNotNull(operatorContext, "operatorContext is null");
// reserve memory for the arrays
int hashSize = HashCommon.arraySize(addresses.size(), 0.75f);
operatorContext.reserveMemory(sizeOfIntArray(hashSize) + sizeOfIntArray(addresses.size()));
mask = hashSize - 1;
key = new int[hashSize];
Arrays.fill(key, -1);
this.positionLinks = new int[addresses.size()];
Arrays.fill(positionLinks, -1);
// index pages
for (int position = 0; position < addresses.size(); position++) {
int pos = ((int) Murmur3.hash64(hashPosition(position))) & mask;
// look for an empty slot or a slot containing this key
while (key[pos] != -1) {
int currentKey = key[pos];
if (positionEqualsPosition(currentKey, position)) {
// found a slot for this key
// link the new key position to the current key position
positionLinks[position] = currentKey;
// key[pos] updated outside of this loop
break;
}
// increment position and mask to handler wrap around
pos = (pos + 1) & mask;
}
key[pos] = position;
}
}
