@Override
public ActualProperties visitTableWriter(
TableWriterNode node, List<ActualProperties> inputProperties) {
ActualProperties properties = Iterables.getOnlyElement(inputProperties);
if (properties.isCoordinatorOnly()) {
return ActualProperties.builder().global(coordinatorSingleStreamPartition()).build();
}
return ActualProperties.builder()
.global(properties.isSingleNode() ? singleStreamPartition() : arbitraryPartition())
.build();
}
@Override
public ActualProperties visitFilter(FilterNode node, List<ActualProperties> inputProperties) {
ActualProperties properties = Iterables.getOnlyElement(inputProperties);
DomainTranslator.ExtractionResult decomposedPredicate =
DomainTranslator.fromPredicate(metadata, session, node.getPredicate(), types);
Map<Symbol, NullableValue> constants = new HashMap<>(properties.getConstants());
constants.putAll(
extractFixedValues(decomposedPredicate.getTupleDomain()).orElse(ImmutableMap.of()));
return ActualProperties.builderFrom(properties).constants(constants).build();
}
@Override
public ActualProperties visitProject(ProjectNode node, List<ActualProperties> inputProperties) {
ActualProperties properties = Iterables.getOnlyElement(inputProperties);
Map<Symbol, Symbol> identities = computeIdentityTranslations(node.getAssignments());
ActualProperties translatedProperties =
properties.translate(column -> Optional.ofNullable(identities.get(column)));
// Extract additional constants
Map<Symbol, NullableValue> constants = new HashMap<>();
for (Map.Entry<Symbol, Expression> assignment : node.getAssignments().entrySet()) {
Expression expression = assignment.getValue();
IdentityHashMap<Expression, Type> expressionTypes =
getExpressionTypes(
session,
metadata,
parser,
types,
expression,
emptyList() /* parameters already replaced */);
Type type = requireNonNull(expressionTypes.get(expression));
ExpressionInterpreter optimizer =
ExpressionInterpreter.expressionOptimizer(
expression, metadata, session, expressionTypes);
// TODO:
// We want to use a symbol resolver that looks up in the constants from the input subplan
// to take advantage of constant-folding for complex expressions
// However, that currently causes errors when those expressions operate on arrays or row
// types
// ("ROW comparison not supported for fields with null elements", etc)
Object value = optimizer.optimize(NoOpSymbolResolver.INSTANCE);
if (value instanceof SymbolReference) {
Symbol symbol = Symbol.from((SymbolReference) value);
NullableValue existingConstantValue = constants.get(symbol);
if (existingConstantValue != null) {
constants.put(assignment.getKey(), new NullableValue(type, value));
}
} else if (!(value instanceof Expression)) {
constants.put(assignment.getKey(), new NullableValue(type, value));
}
}
constants.putAll(translatedProperties.getConstants());
return ActualProperties.builderFrom(translatedProperties).constants(constants).build();
}
@Override
public ActualProperties visitAggregation(
AggregationNode node, List<ActualProperties> inputProperties) {
ActualProperties properties = Iterables.getOnlyElement(inputProperties);
ActualProperties translated =
properties.translate(
symbol ->
node.getGroupingKeys().contains(symbol)
? Optional.of(symbol)
: Optional.<Symbol>empty());
return ActualProperties.builderFrom(translated)
.local(LocalProperties.grouped(node.getGroupingKeys()))
.build();
}
@Override
public ActualProperties visitDistinctLimit(
DistinctLimitNode node, List<ActualProperties> inputProperties) {
ActualProperties properties = Iterables.getOnlyElement(inputProperties);
return ActualProperties.builderFrom(properties)
.local(LocalProperties.grouped(node.getDistinctSymbols()))
.build();
}
public static ActualProperties deriveProperties(
PlanNode node,
List<ActualProperties> inputProperties,
Metadata metadata,
Session session,
Map<Symbol, Type> types,
SqlParser parser) {
ActualProperties output =
node.accept(new Visitor(metadata, session, types, parser), inputProperties);
// TODO: ideally this logic would be somehow moved to PlanSanityChecker
verify(
node instanceof SemiJoinNode
|| inputProperties.stream().noneMatch(ActualProperties::isNullsReplicated)
|| output.isNullsReplicated(),
"SemiJoinNode is the only node that can strip null replication");
return output;
}
@Override
public ActualProperties visitSort(SortNode node, List<ActualProperties> inputProperties) {
ActualProperties properties = Iterables.getOnlyElement(inputProperties);
List<SortingProperty<Symbol>> localProperties =
node.getOrderBy()
.stream()
.map(column -> new SortingProperty<>(column, node.getOrderings().get(column)))
.collect(toImmutableList());
return ActualProperties.builderFrom(properties).local(localProperties).build();
}
@Override
public ActualProperties visitWindow(WindowNode node, List<ActualProperties> inputProperties) {
ActualProperties properties = Iterables.getOnlyElement(inputProperties);
// If the input is completely pre-partitioned and sorted, then the original input properties
// will be respected
if (ImmutableSet.copyOf(node.getPartitionBy()).equals(node.getPrePartitionedInputs())
&& node.getPreSortedOrderPrefix() == node.getOrderBy().size()) {
return properties;
}
ImmutableList.Builder<LocalProperty<Symbol>> localProperties = ImmutableList.builder();
// If the WindowNode has pre-partitioned inputs, then it will not change the order of those
// inputs at output,
// so we should just propagate those underlying local properties that guarantee the
// pre-partitioning.
// TODO: come up with a more general form of this operation for other streaming operators
if (!node.getPrePartitionedInputs().isEmpty()) {
GroupingProperty<Symbol> prePartitionedProperty =
new GroupingProperty<>(node.getPrePartitionedInputs());
for (LocalProperty<Symbol> localProperty : properties.getLocalProperties()) {
if (!prePartitionedProperty.isSimplifiedBy(localProperty)) {
break;
}
localProperties.add(localProperty);
}
}
if (!node.getPartitionBy().isEmpty()) {
localProperties.add(new GroupingProperty<>(node.getPartitionBy()));
}
for (Symbol column : node.getOrderBy()) {
localProperties.add(new SortingProperty<>(column, node.getOrderings().get(column)));
}
return ActualProperties.builderFrom(properties)
.local(LocalProperties.normalizeAndPrune(localProperties.build()))
.build();
}
@Override
public ActualProperties visitTopNRowNumber(
TopNRowNumberNode node, List<ActualProperties> inputProperties) {
ActualProperties properties = Iterables.getOnlyElement(inputProperties);
ImmutableList.Builder<LocalProperty<Symbol>> localProperties = ImmutableList.builder();
localProperties.add(new GroupingProperty<>(node.getPartitionBy()));
for (Symbol column : node.getOrderBy()) {
localProperties.add(new SortingProperty<>(column, node.getOrderings().get(column)));
}
return ActualProperties.builderFrom(properties).local(localProperties.build()).build();
}
@Override
public ActualProperties visitTableScan(
TableScanNode node, List<ActualProperties> inputProperties) {
checkArgument(node.getLayout().isPresent(), "table layout has not yet been chosen");
TableLayout layout = metadata.getLayout(session, node.getLayout().get());
Map<ColumnHandle, Symbol> assignments =
ImmutableBiMap.copyOf(node.getAssignments()).inverse();
ActualProperties.Builder properties = ActualProperties.builder();
// Globally constant assignments
Map<ColumnHandle, NullableValue> globalConstants = new HashMap<>();
extractFixedValues(node.getCurrentConstraint())
.orElse(ImmutableMap.of())
.entrySet()
.stream()
.filter(entry -> !entry.getValue().isNull())
.forEach(entry -> globalConstants.put(entry.getKey(), entry.getValue()));
Map<Symbol, NullableValue> symbolConstants =
globalConstants
.entrySet()
.stream()
.filter(entry -> assignments.containsKey(entry.getKey()))
.collect(toMap(entry -> assignments.get(entry.getKey()), Map.Entry::getValue));
properties.constants(symbolConstants);
// Partitioning properties
properties.global(deriveGlobalProperties(layout, assignments, globalConstants));
// Append the global constants onto the local properties to maximize their translation
// potential
List<LocalProperty<ColumnHandle>> constantAppendedLocalProperties =
ImmutableList.<LocalProperty<ColumnHandle>>builder()
.addAll(
globalConstants
.keySet()
.stream()
.map(column -> new ConstantProperty<>(column))
.iterator())
.addAll(layout.getLocalProperties())
.build();
properties.local(
LocalProperties.translate(
constantAppendedLocalProperties,
column -> Optional.ofNullable(assignments.get(column))));
return properties.build();
}
@Override
public ActualProperties visitIndexJoin(
IndexJoinNode node, List<ActualProperties> inputProperties) {
// TODO: include all equivalent columns in partitioning properties
ActualProperties probeProperties = inputProperties.get(0);
ActualProperties indexProperties = inputProperties.get(1);
switch (node.getType()) {
case INNER:
return ActualProperties.builderFrom(probeProperties)
.constants(
ImmutableMap.<Symbol, NullableValue>builder()
.putAll(probeProperties.getConstants())
.putAll(indexProperties.getConstants())
.build())
.build();
case SOURCE_OUTER:
return ActualProperties.builderFrom(probeProperties)
.constants(probeProperties.getConstants())
.build();
default:
throw new UnsupportedOperationException("Unsupported join type: " + node.getType());
}
}
@Override
public ActualProperties visitTableFinish(
TableFinishNode node, List<ActualProperties> inputProperties) {
return ActualProperties.builder().global(coordinatorSingleStreamPartition()).build();
}
@Override
public ActualProperties visitValues(ValuesNode node, List<ActualProperties> context) {
return ActualProperties.builder().global(singleStreamPartition()).build();
}
@Override
public ActualProperties visitGroupId(GroupIdNode node, List<ActualProperties> inputProperties) {
ActualProperties properties = Iterables.getOnlyElement(inputProperties);
return properties.translate(translateGroupIdSymbols(node));
}
@Override
public ActualProperties visitExplainAnalyze(
ExplainAnalyzeNode node, List<ActualProperties> inputProperties) {
return ActualProperties.builder().global(coordinatorSingleStreamPartition()).build();
}
@Override
public ActualProperties visitJoin(JoinNode node, List<ActualProperties> inputProperties) {
// TODO: include all equivalent columns in partitioning properties
ActualProperties probeProperties = inputProperties.get(0);
ActualProperties buildProperties = inputProperties.get(1);
switch (node.getType()) {
case INNER:
return ActualProperties.builderFrom(probeProperties)
.constants(
ImmutableMap.<Symbol, NullableValue>builder()
.putAll(probeProperties.getConstants())
.putAll(buildProperties.getConstants())
.build())
.build();
case LEFT:
return ActualProperties.builderFrom(probeProperties)
.constants(probeProperties.getConstants())
.build();
case RIGHT:
return ActualProperties.builderFrom(buildProperties)
.constants(buildProperties.getConstants())
.local(ImmutableList.of())
.build();
case FULL:
// We can't say anything about the partitioning scheme because any partition of
// a hash-partitioned join can produce nulls in case of a lack of matches
return ActualProperties.builder()
.global(
probeProperties.isSingleNode() ? singleStreamPartition() : arbitraryPartition())
.build();
default:
throw new UnsupportedOperationException("Unsupported join type: " + node.getType());
}
}
@Override
public ActualProperties visitExchange(
ExchangeNode node, List<ActualProperties> inputProperties) {
checkArgument(
node.getScope() != REMOTE
|| inputProperties.stream().noneMatch(ActualProperties::isNullsReplicated),
"Null replicated inputs should not be remotely exchanged");
Set<Map.Entry<Symbol, NullableValue>> entries = null;
for (int sourceIndex = 0; sourceIndex < node.getSources().size(); sourceIndex++) {
Map<Symbol, Symbol> inputToOutput = exchangeInputToOutput(node, sourceIndex);
ActualProperties translated =
inputProperties
.get(sourceIndex)
.translate(symbol -> Optional.ofNullable(inputToOutput.get(symbol)));
entries =
(entries == null)
? translated.getConstants().entrySet()
: Sets.intersection(entries, translated.getConstants().entrySet());
}
checkState(entries != null);
Map<Symbol, NullableValue> constants =
entries.stream().collect(toMap(Map.Entry::getKey, Map.Entry::getValue));
// Local exchanges are only created in AddLocalExchanges, at the end of optimization, and
// local exchanges do not produce global properties as represented by ActualProperties.
// This is acceptable because AddLocalExchanges does not use global properties and is only
// interested in the local properties.
// TODO: implement full properties for local exchanges
if (node.getScope() == LOCAL) {
return ActualProperties.builder().constants(constants).build();
}
switch (node.getType()) {
case GATHER:
boolean coordinatorOnly =
node.getPartitioningScheme().getPartitioning().getHandle().isCoordinatorOnly();
return ActualProperties.builder()
.global(
coordinatorOnly ? coordinatorSingleStreamPartition() : singleStreamPartition())
.constants(constants)
.build();
case REPARTITION:
return ActualProperties.builder()
.global(
partitionedOn(
node.getPartitioningScheme().getPartitioning(),
Optional.of(node.getPartitioningScheme().getPartitioning()))
.withReplicatedNulls(node.getPartitioningScheme().isReplicateNulls()))
.constants(constants)
.build();
case REPLICATE:
// TODO: this should have the same global properties as the stream taking the replicated
// data
return ActualProperties.builder()
.global(arbitraryPartition())
.constants(constants)
.build();
}
throw new UnsupportedOperationException("not yet implemented");
}
