public static ExpressionAnalyzer createConstantAnalyzer(Metadata metadata, Session session) {
return createWithoutSubqueries(
metadata.getFunctionRegistry(),
metadata.getTypeManager(),
session,
EXPRESSION_NOT_CONSTANT,
"Constant expression cannot contain a subquery");
}
private ByteCodeNode compileExpression(
CallSiteBinder callSiteBinder,
RowExpression expression,
boolean sourceIsCursor,
CompilerContext context,
Block getSessionByteCode) {
ByteCodeExpressionVisitor visitor =
new ByteCodeExpressionVisitor(
callSiteBinder, getSessionByteCode, metadata.getFunctionRegistry(), sourceIsCursor);
return expression.accept(visitor, context);
}
private RowExpression toRowExpression(
Expression projection, IdentityHashMap<Expression, Type> expressionTypes) {
return SqlToRowExpressionTranslator.translate(
projection,
SCALAR,
expressionTypes,
metadata.getFunctionRegistry(),
metadata.getTypeManager(),
session,
false);
}
@Override
public PlanNode visitWindow(WindowNode node, RewriteContext<Context> context) {
if (!node.getWindowFunctions()
.values()
.stream()
.map(function -> function.getFunctionCall().getName())
.allMatch(metadata.getFunctionRegistry()::isAggregationFunction)) {
return node;
}
// Don't need this restriction if we can prove that all order by symbols are deterministically
// produced
if (!node.getOrderBy().isEmpty()) {
return node;
}
// Only RANGE frame type currently supported for aggregation functions because it guarantees
// the
// same value for each peer group.
// ROWS frame type requires the ordering to be fully deterministic (e.g. deterministically
// sorted on all columns)
if (node.getFrames()
.stream()
.map(WindowNode.Frame::getType)
.anyMatch(
type ->
type
!= WindowFrame.Type
.RANGE)) { // TODO: extract frames of type RANGE and allow optimization on
// them
return node;
}
// Lookup symbols can only be passed through if they are part of the partitioning
Set<Symbol> partitionByLookupSymbols =
context
.get()
.getLookupSymbols()
.stream()
.filter(node.getPartitionBy()::contains)
.collect(toImmutableSet());
if (partitionByLookupSymbols.isEmpty()) {
return node;
}
return context.defaultRewrite(
node, new Context(partitionByLookupSymbols, context.get().getSuccess()));
}
@Override
protected Object visitGenericLiteral(GenericLiteral node, ConnectorSession session) {
Type type = metadata.getType(parseTypeSignature(node.getType()));
if (type == null) {
throw new SemanticException(TYPE_MISMATCH, node, "Unknown type: " + node.getType());
}
if (JSON.equals(type)) {
ScalarFunctionImplementation operator =
metadata
.getFunctionRegistry()
.getScalarFunctionImplementation(
new Signature(
"json_parse", SCALAR, JSON.getTypeSignature(), VARCHAR.getTypeSignature()));
try {
return ExpressionInterpreter.invoke(
session, operator, ImmutableList.<Object>of(utf8Slice(node.getValue())));
} catch (Throwable throwable) {
throw Throwables.propagate(throwable);
}
}
ScalarFunctionImplementation operator;
try {
Signature signature = metadata.getFunctionRegistry().getCoercion(VARCHAR, type);
operator = metadata.getFunctionRegistry().getScalarFunctionImplementation(signature);
} catch (IllegalArgumentException e) {
throw new SemanticException(TYPE_MISMATCH, node, "No literal form for type %s", type);
}
try {
return ExpressionInterpreter.invoke(
session, operator, ImmutableList.<Object>of(utf8Slice(node.getValue())));
} catch (Throwable throwable) {
throw Throwables.propagate(throwable);
}
}
private MethodDefinition generateProjectMethod(
ClassDefinition classDefinition,
CallSiteBinder callSiteBinder,
CachedInstanceBinder cachedInstanceBinder,
String methodName,
RowExpression projection) {
Parameter session = arg("session", ConnectorSession.class);
List<Parameter> inputs = toBlockParameters(getInputChannels(projection));
Parameter position = arg("position", int.class);
Parameter output = arg("output", BlockBuilder.class);
MethodDefinition method =
classDefinition.declareMethod(
a(PUBLIC),
methodName,
type(void.class),
ImmutableList.<Parameter>builder()
.add(session)
.addAll(inputs)
.add(position)
.add(output)
.build());
method.comment("Projection: %s", projection.toString());
Scope scope = method.getScope();
BytecodeBlock body = method.getBody();
Variable wasNullVariable = scope.declareVariable("wasNull", body, constantFalse());
BytecodeExpressionVisitor visitor =
new BytecodeExpressionVisitor(
callSiteBinder,
cachedInstanceBinder,
fieldReferenceCompiler(callSiteBinder, position, wasNullVariable),
metadata.getFunctionRegistry());
body.getVariable(output)
.comment("evaluate projection: " + projection.toString())
.append(projection.accept(visitor, scope))
.append(generateWrite(callSiteBinder, scope, wasNullVariable, projection.getType()))
.ret();
return method;
}
public static ExpressionAnalyzer create(
Analysis analysis,
Session session,
Metadata metadata,
SqlParser sqlParser,
AccessControl accessControl,
boolean experimentalSyntaxEnabled) {
return new ExpressionAnalyzer(
metadata.getFunctionRegistry(),
metadata.getTypeManager(),
node ->
new StatementAnalyzer(
analysis,
metadata,
sqlParser,
accessControl,
session,
experimentalSyntaxEnabled,
Optional.empty()),
session);
}
private void generateFilterMethod(
ClassDefinition classDefinition,
CallSiteBinder callSiteBinder,
CachedInstanceBinder cachedInstanceBinder,
RowExpression filter) {
Parameter session = arg("session", ConnectorSession.class);
Parameter position = arg("position", int.class);
List<Parameter> blocks = toBlockParameters(getInputChannels(filter));
MethodDefinition method =
classDefinition.declareMethod(
a(PUBLIC),
"filter",
type(boolean.class),
ImmutableList.<Parameter>builder().add(session).addAll(blocks).add(position).build());
method.comment("Filter: %s", filter.toString());
BytecodeBlock body = method.getBody();
Scope scope = method.getScope();
Variable wasNullVariable = scope.declareVariable("wasNull", body, constantFalse());
BytecodeExpressionVisitor visitor =
new BytecodeExpressionVisitor(
callSiteBinder,
cachedInstanceBinder,
fieldReferenceCompiler(callSiteBinder, position, wasNullVariable),
metadata.getFunctionRegistry());
BytecodeNode visitorBody = filter.accept(visitor, scope);
Variable result = scope.declareVariable(boolean.class, "result");
body.append(visitorBody)
.putVariable(result)
.append(
new IfStatement()
.condition(wasNullVariable)
.ifTrue(constantFalse().ret())
.ifFalse(result.ret()));
}
private InternalTable buildPartitions(
Session session, String catalogName, Map<String, NullableValue> filters) {
QualifiedObjectName tableName = extractQualifiedTableName(catalogName, filters);
InternalTable.Builder table =
InternalTable.builder(informationSchemaTableColumns(TABLE_INTERNAL_PARTITIONS));
Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
if (!tableHandle.isPresent()) {
throw new TableNotFoundException(tableName.asSchemaTableName());
}
List<TableLayoutResult> layouts =
metadata.getLayouts(
session, tableHandle.get(), Constraint.<ColumnHandle>alwaysTrue(), Optional.empty());
if (layouts.size() == 1) {
Map<ColumnHandle, String> columnHandles =
ImmutableBiMap.copyOf(metadata.getColumnHandles(session, tableHandle.get())).inverse();
Map<ColumnHandle, MethodHandle> methodHandles = new HashMap<>();
for (ColumnHandle columnHandle : columnHandles.keySet()) {
try {
ColumnMetadata columnMetadata =
metadata.getColumnMetadata(session, tableHandle.get(), columnHandle);
Signature operator =
metadata.getFunctionRegistry().getCoercion(columnMetadata.getType(), VARCHAR);
MethodHandle methodHandle =
metadata
.getFunctionRegistry()
.getScalarFunctionImplementation(operator)
.getMethodHandle();
methodHandles.put(columnHandle, methodHandle);
} catch (OperatorNotFoundException exception) {
// Do not put the columnHandle in the map.
}
}
TableLayout layout = Iterables.getOnlyElement(layouts).getLayout();
layout
.getDiscretePredicates()
.ifPresent(
domains -> {
int partitionNumber = 1;
for (TupleDomain<ColumnHandle> domain : domains) {
for (Entry<ColumnHandle, NullableValue> entry :
TupleDomain.extractFixedValues(domain).get().entrySet()) {
ColumnHandle columnHandle = entry.getKey();
String columnName = columnHandles.get(columnHandle);
String value = null;
if (entry.getValue().getValue() != null) {
if (methodHandles.containsKey(columnHandle)) {
try {
value =
((Slice)
methodHandles
.get(columnHandle)
.invokeWithArguments(entry.getValue().getValue()))
.toStringUtf8();
} catch (Throwable throwable) {
throw Throwables.propagate(throwable);
}
} else {
// OperatorNotFoundException was thrown for this columnHandle
value = "<UNREPRESENTABLE VALUE>";
}
}
table.add(
catalogName,
tableName.getSchemaName(),
tableName.getObjectName(),
partitionNumber,
columnName,
value);
}
partitionNumber++;
}
});
}
return table.build();
}
private InternalTable buildPartitions(
Session session, String catalogName, Map<String, SerializableNativeValue> filters) {
QualifiedTableName tableName = extractQualifiedTableName(catalogName, filters);
InternalTable.Builder table =
InternalTable.builder(informationSchemaTableColumns(TABLE_INTERNAL_PARTITIONS));
Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
checkArgument(tableHandle.isPresent(), "Table %s does not exist", tableName);
Map<ColumnHandle, String> columnHandles =
ImmutableBiMap.copyOf(metadata.getColumnHandles(session, tableHandle.get())).inverse();
List<TableLayoutResult> layouts =
metadata.getLayouts(
session, tableHandle.get(), Constraint.<ColumnHandle>alwaysTrue(), Optional.empty());
if (layouts.size() == 1) {
TableLayout layout = Iterables.getOnlyElement(layouts).getLayout();
layout
.getDiscretePredicates()
.ifPresent(
domains -> {
int partitionNumber = 1;
for (TupleDomain<ColumnHandle> domain : domains) {
for (Entry<ColumnHandle, SerializableNativeValue> entry :
domain.extractNullableFixedValues().entrySet()) {
ColumnHandle columnHandle = entry.getKey();
String columnName = columnHandles.get(columnHandle);
String value = null;
if (entry.getValue().getValue() != null) {
ColumnMetadata columnMetadata =
metadata.getColumnMetadata(session, tableHandle.get(), columnHandle);
try {
FunctionInfo operator =
metadata
.getFunctionRegistry()
.getCoercion(columnMetadata.getType(), VARCHAR);
value =
((Slice)
operator
.getMethodHandle()
.invokeWithArguments(entry.getValue().getValue()))
.toStringUtf8();
} catch (OperatorNotFoundException e) {
value = "<UNREPRESENTABLE VALUE>";
} catch (Throwable throwable) {
throw Throwables.propagate(throwable);
}
}
table.add(
catalogName,
tableName.getSchemaName(),
tableName.getTableName(),
partitionNumber,
columnName,
value);
}
partitionNumber++;
}
});
}
return table.build();
}
public PlanOptimizers(
Metadata metadata,
SqlParser sqlParser,
FeaturesConfig featuresConfig,
boolean forceSingleNode) {
ImmutableList.Builder<PlanOptimizer> builder = ImmutableList.builder();
builder.add(
new DesugaringOptimizer(
metadata,
sqlParser), // Clean up all the sugar in expressions, e.g. AtTimeZone, must be run
// before all the other optimizers
new CanonicalizeExpressions(),
new ImplementFilteredAggregations(),
new ImplementSampleAsFilter(),
new SimplifyExpressions(metadata, sqlParser),
new UnaliasSymbolReferences(),
new PruneIdentityProjections(),
new SetFlatteningOptimizer(),
new ImplementIntersectAndExceptAsUnion(),
new LimitPushDown(), // Run the LimitPushDown after flattening set operators to make it
// easier to do the set flattening
new PruneUnreferencedOutputs(),
new MergeProjections(),
new TransformExistsApplyToScalarApply(metadata),
new TransformQuantifiedComparisonApplyToScalarApply(metadata),
new RemoveUnreferencedScalarInputApplyNodes(),
new TransformUncorrelatedInPredicateSubqueryToSemiJoin(),
new TransformUncorrelatedScalarToJoin(),
new TransformCorrelatedScalarAggregationToJoin(metadata),
new PredicatePushDown(metadata, sqlParser),
new MergeProjections(),
new SimplifyExpressions(
metadata,
sqlParser), // Re-run the SimplifyExpressions to simplify any recomposed expressions
// from other optimizations
new ProjectionPushDown(),
new UnaliasSymbolReferences(), // Run again because predicate pushdown and projection
// pushdown might add more projections
new PruneUnreferencedOutputs(), // Make sure to run this before index join. Filtered
// projections may not have all the columns.
new IndexJoinOptimizer(
metadata), // Run this after projections and filters have been fully simplified and
// pushed down
new CountConstantOptimizer(),
new WindowFilterPushDown(
metadata), // This must run after PredicatePushDown and LimitPushDown so that it
// squashes any successive filter nodes and limits
new MergeWindows(),
new ReorderWindows(), // Should be after MergeWindows to avoid unnecessary reordering of
// mergeable windows
new MergeProjections(),
new PruneUnreferencedOutputs(), // Make sure to run this at the end to help clean the plan
// for logging/execution and not remove info that other
// optimizers might need at an earlier point
new PruneIdentityProjections(), // This MUST run after PruneUnreferencedOutputs as it may
// introduce new redundant projections
new MetadataQueryOptimizer(metadata),
new EliminateCrossJoins(), // This can pull up Filter and Project nodes from between Joins,
// so we need to push them down again
new PredicatePushDown(metadata, sqlParser),
new ProjectionPushDown());
if (featuresConfig.isOptimizeSingleDistinct()) {
builder.add(new SingleDistinctOptimizer());
builder.add(new PruneUnreferencedOutputs());
}
builder.add(new OptimizeMixedDistinctAggregations(metadata));
if (!forceSingleNode) {
builder.add(new PushTableWriteThroughUnion()); // Must run before AddExchanges
builder.add(new AddExchanges(metadata, sqlParser));
}
builder.add(new PickLayout(metadata));
builder.add(new EmptyDeleteOptimizer()); // Run after table scan is removed by PickLayout
builder.add(
new PredicatePushDown(
metadata,
sqlParser)); // Run predicate push down one more time in case we can leverage new
// information from layouts' effective predicate
builder.add(new ProjectionPushDown());
builder.add(new MergeProjections());
builder.add(
new UnaliasSymbolReferences()); // Run unalias after merging projections to simplify
// projections more efficiently
builder.add(new PruneUnreferencedOutputs());
builder.add(new PruneIdentityProjections());
// Optimizers above this don't understand local exchanges, so be careful moving this.
builder.add(new AddLocalExchanges(metadata, sqlParser));
// Optimizers above this do not need to care about aggregations with the type other than SINGLE
// This optimizer must be run after all exchange-related optimizers
builder.add(new PartialAggregationPushDown(metadata.getFunctionRegistry()));
builder.add(new PruneIdentityProjections());
// DO NOT add optimizers that change the plan shape (computations) after this point
// Precomputed hashes - this assumes that partitioning will not change
builder.add(new HashGenerationOptimizer());
builder.add(new MetadataDeleteOptimizer(metadata));
builder.add(new BeginTableWrite(metadata)); // HACK! see comments in BeginTableWrite
// TODO: consider adding a formal final plan sanitization optimizer that prepares the plan for
// transmission/execution/logging
// TODO: figure out how to improve the set flattening optimizer so that it can run at any point
this.optimizers = builder.build();
}
