public static ExpressionAnalyzer createConstantAnalyzer(Metadata metadata, Session session) { return createWithoutSubqueries( metadata.getFunctionRegistry(), metadata.getTypeManager(), session, EXPRESSION_NOT_CONSTANT, "Constant expression cannot contain a subquery"); }
private ByteCodeNode compileExpression( CallSiteBinder callSiteBinder, RowExpression expression, boolean sourceIsCursor, CompilerContext context, Block getSessionByteCode) { ByteCodeExpressionVisitor visitor = new ByteCodeExpressionVisitor( callSiteBinder, getSessionByteCode, metadata.getFunctionRegistry(), sourceIsCursor); return expression.accept(visitor, context); }
private RowExpression toRowExpression( Expression projection, IdentityHashMap<Expression, Type> expressionTypes) { return SqlToRowExpressionTranslator.translate( projection, SCALAR, expressionTypes, metadata.getFunctionRegistry(), metadata.getTypeManager(), session, false); }
@Override public PlanNode visitWindow(WindowNode node, RewriteContext<Context> context) { if (!node.getWindowFunctions() .values() .stream() .map(function -> function.getFunctionCall().getName()) .allMatch(metadata.getFunctionRegistry()::isAggregationFunction)) { return node; } // Don't need this restriction if we can prove that all order by symbols are deterministically // produced if (!node.getOrderBy().isEmpty()) { return node; } // Only RANGE frame type currently supported for aggregation functions because it guarantees // the // same value for each peer group. // ROWS frame type requires the ordering to be fully deterministic (e.g. deterministically // sorted on all columns) if (node.getFrames() .stream() .map(WindowNode.Frame::getType) .anyMatch( type -> type != WindowFrame.Type .RANGE)) { // TODO: extract frames of type RANGE and allow optimization on // them return node; } // Lookup symbols can only be passed through if they are part of the partitioning Set<Symbol> partitionByLookupSymbols = context .get() .getLookupSymbols() .stream() .filter(node.getPartitionBy()::contains) .collect(toImmutableSet()); if (partitionByLookupSymbols.isEmpty()) { return node; } return context.defaultRewrite( node, new Context(partitionByLookupSymbols, context.get().getSuccess())); }
@Override protected Object visitGenericLiteral(GenericLiteral node, ConnectorSession session) { Type type = metadata.getType(parseTypeSignature(node.getType())); if (type == null) { throw new SemanticException(TYPE_MISMATCH, node, "Unknown type: " + node.getType()); } if (JSON.equals(type)) { ScalarFunctionImplementation operator = metadata .getFunctionRegistry() .getScalarFunctionImplementation( new Signature( "json_parse", SCALAR, JSON.getTypeSignature(), VARCHAR.getTypeSignature())); try { return ExpressionInterpreter.invoke( session, operator, ImmutableList.<Object>of(utf8Slice(node.getValue()))); } catch (Throwable throwable) { throw Throwables.propagate(throwable); } } ScalarFunctionImplementation operator; try { Signature signature = metadata.getFunctionRegistry().getCoercion(VARCHAR, type); operator = metadata.getFunctionRegistry().getScalarFunctionImplementation(signature); } catch (IllegalArgumentException e) { throw new SemanticException(TYPE_MISMATCH, node, "No literal form for type %s", type); } try { return ExpressionInterpreter.invoke( session, operator, ImmutableList.<Object>of(utf8Slice(node.getValue()))); } catch (Throwable throwable) { throw Throwables.propagate(throwable); } }
private MethodDefinition generateProjectMethod( ClassDefinition classDefinition, CallSiteBinder callSiteBinder, CachedInstanceBinder cachedInstanceBinder, String methodName, RowExpression projection) { Parameter session = arg("session", ConnectorSession.class); List<Parameter> inputs = toBlockParameters(getInputChannels(projection)); Parameter position = arg("position", int.class); Parameter output = arg("output", BlockBuilder.class); MethodDefinition method = classDefinition.declareMethod( a(PUBLIC), methodName, type(void.class), ImmutableList.<Parameter>builder() .add(session) .addAll(inputs) .add(position) .add(output) .build()); method.comment("Projection: %s", projection.toString()); Scope scope = method.getScope(); BytecodeBlock body = method.getBody(); Variable wasNullVariable = scope.declareVariable("wasNull", body, constantFalse()); BytecodeExpressionVisitor visitor = new BytecodeExpressionVisitor( callSiteBinder, cachedInstanceBinder, fieldReferenceCompiler(callSiteBinder, position, wasNullVariable), metadata.getFunctionRegistry()); body.getVariable(output) .comment("evaluate projection: " + projection.toString()) .append(projection.accept(visitor, scope)) .append(generateWrite(callSiteBinder, scope, wasNullVariable, projection.getType())) .ret(); return method; }
public static ExpressionAnalyzer create( Analysis analysis, Session session, Metadata metadata, SqlParser sqlParser, AccessControl accessControl, boolean experimentalSyntaxEnabled) { return new ExpressionAnalyzer( metadata.getFunctionRegistry(), metadata.getTypeManager(), node -> new StatementAnalyzer( analysis, metadata, sqlParser, accessControl, session, experimentalSyntaxEnabled, Optional.empty()), session); }
private void generateFilterMethod( ClassDefinition classDefinition, CallSiteBinder callSiteBinder, CachedInstanceBinder cachedInstanceBinder, RowExpression filter) { Parameter session = arg("session", ConnectorSession.class); Parameter position = arg("position", int.class); List<Parameter> blocks = toBlockParameters(getInputChannels(filter)); MethodDefinition method = classDefinition.declareMethod( a(PUBLIC), "filter", type(boolean.class), ImmutableList.<Parameter>builder().add(session).addAll(blocks).add(position).build()); method.comment("Filter: %s", filter.toString()); BytecodeBlock body = method.getBody(); Scope scope = method.getScope(); Variable wasNullVariable = scope.declareVariable("wasNull", body, constantFalse()); BytecodeExpressionVisitor visitor = new BytecodeExpressionVisitor( callSiteBinder, cachedInstanceBinder, fieldReferenceCompiler(callSiteBinder, position, wasNullVariable), metadata.getFunctionRegistry()); BytecodeNode visitorBody = filter.accept(visitor, scope); Variable result = scope.declareVariable(boolean.class, "result"); body.append(visitorBody) .putVariable(result) .append( new IfStatement() .condition(wasNullVariable) .ifTrue(constantFalse().ret()) .ifFalse(result.ret())); }
private InternalTable buildPartitions( Session session, String catalogName, Map<String, NullableValue> filters) { QualifiedObjectName tableName = extractQualifiedTableName(catalogName, filters); InternalTable.Builder table = InternalTable.builder(informationSchemaTableColumns(TABLE_INTERNAL_PARTITIONS)); Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName); if (!tableHandle.isPresent()) { throw new TableNotFoundException(tableName.asSchemaTableName()); } List<TableLayoutResult> layouts = metadata.getLayouts( session, tableHandle.get(), Constraint.<ColumnHandle>alwaysTrue(), Optional.empty()); if (layouts.size() == 1) { Map<ColumnHandle, String> columnHandles = ImmutableBiMap.copyOf(metadata.getColumnHandles(session, tableHandle.get())).inverse(); Map<ColumnHandle, MethodHandle> methodHandles = new HashMap<>(); for (ColumnHandle columnHandle : columnHandles.keySet()) { try { ColumnMetadata columnMetadata = metadata.getColumnMetadata(session, tableHandle.get(), columnHandle); Signature operator = metadata.getFunctionRegistry().getCoercion(columnMetadata.getType(), VARCHAR); MethodHandle methodHandle = metadata .getFunctionRegistry() .getScalarFunctionImplementation(operator) .getMethodHandle(); methodHandles.put(columnHandle, methodHandle); } catch (OperatorNotFoundException exception) { // Do not put the columnHandle in the map. } } TableLayout layout = Iterables.getOnlyElement(layouts).getLayout(); layout .getDiscretePredicates() .ifPresent( domains -> { int partitionNumber = 1; for (TupleDomain<ColumnHandle> domain : domains) { for (Entry<ColumnHandle, NullableValue> entry : TupleDomain.extractFixedValues(domain).get().entrySet()) { ColumnHandle columnHandle = entry.getKey(); String columnName = columnHandles.get(columnHandle); String value = null; if (entry.getValue().getValue() != null) { if (methodHandles.containsKey(columnHandle)) { try { value = ((Slice) methodHandles .get(columnHandle) .invokeWithArguments(entry.getValue().getValue())) .toStringUtf8(); } catch (Throwable throwable) { throw Throwables.propagate(throwable); } } else { // OperatorNotFoundException was thrown for this columnHandle value = "<UNREPRESENTABLE VALUE>"; } } table.add( catalogName, tableName.getSchemaName(), tableName.getObjectName(), partitionNumber, columnName, value); } partitionNumber++; } }); } return table.build(); }
private InternalTable buildPartitions( Session session, String catalogName, Map<String, SerializableNativeValue> filters) { QualifiedTableName tableName = extractQualifiedTableName(catalogName, filters); InternalTable.Builder table = InternalTable.builder(informationSchemaTableColumns(TABLE_INTERNAL_PARTITIONS)); Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName); checkArgument(tableHandle.isPresent(), "Table %s does not exist", tableName); Map<ColumnHandle, String> columnHandles = ImmutableBiMap.copyOf(metadata.getColumnHandles(session, tableHandle.get())).inverse(); List<TableLayoutResult> layouts = metadata.getLayouts( session, tableHandle.get(), Constraint.<ColumnHandle>alwaysTrue(), Optional.empty()); if (layouts.size() == 1) { TableLayout layout = Iterables.getOnlyElement(layouts).getLayout(); layout .getDiscretePredicates() .ifPresent( domains -> { int partitionNumber = 1; for (TupleDomain<ColumnHandle> domain : domains) { for (Entry<ColumnHandle, SerializableNativeValue> entry : domain.extractNullableFixedValues().entrySet()) { ColumnHandle columnHandle = entry.getKey(); String columnName = columnHandles.get(columnHandle); String value = null; if (entry.getValue().getValue() != null) { ColumnMetadata columnMetadata = metadata.getColumnMetadata(session, tableHandle.get(), columnHandle); try { FunctionInfo operator = metadata .getFunctionRegistry() .getCoercion(columnMetadata.getType(), VARCHAR); value = ((Slice) operator .getMethodHandle() .invokeWithArguments(entry.getValue().getValue())) .toStringUtf8(); } catch (OperatorNotFoundException e) { value = "<UNREPRESENTABLE VALUE>"; } catch (Throwable throwable) { throw Throwables.propagate(throwable); } } table.add( catalogName, tableName.getSchemaName(), tableName.getTableName(), partitionNumber, columnName, value); } partitionNumber++; } }); } return table.build(); }
public PlanOptimizers( Metadata metadata, SqlParser sqlParser, FeaturesConfig featuresConfig, boolean forceSingleNode) { ImmutableList.Builder<PlanOptimizer> builder = ImmutableList.builder(); builder.add( new DesugaringOptimizer( metadata, sqlParser), // Clean up all the sugar in expressions, e.g. AtTimeZone, must be run // before all the other optimizers new CanonicalizeExpressions(), new ImplementFilteredAggregations(), new ImplementSampleAsFilter(), new SimplifyExpressions(metadata, sqlParser), new UnaliasSymbolReferences(), new PruneIdentityProjections(), new SetFlatteningOptimizer(), new ImplementIntersectAndExceptAsUnion(), new LimitPushDown(), // Run the LimitPushDown after flattening set operators to make it // easier to do the set flattening new PruneUnreferencedOutputs(), new MergeProjections(), new TransformExistsApplyToScalarApply(metadata), new TransformQuantifiedComparisonApplyToScalarApply(metadata), new RemoveUnreferencedScalarInputApplyNodes(), new TransformUncorrelatedInPredicateSubqueryToSemiJoin(), new TransformUncorrelatedScalarToJoin(), new TransformCorrelatedScalarAggregationToJoin(metadata), new PredicatePushDown(metadata, sqlParser), new MergeProjections(), new SimplifyExpressions( metadata, sqlParser), // Re-run the SimplifyExpressions to simplify any recomposed expressions // from other optimizations new ProjectionPushDown(), new UnaliasSymbolReferences(), // Run again because predicate pushdown and projection // pushdown might add more projections new PruneUnreferencedOutputs(), // Make sure to run this before index join. Filtered // projections may not have all the columns. new IndexJoinOptimizer( metadata), // Run this after projections and filters have been fully simplified and // pushed down new CountConstantOptimizer(), new WindowFilterPushDown( metadata), // This must run after PredicatePushDown and LimitPushDown so that it // squashes any successive filter nodes and limits new MergeWindows(), new ReorderWindows(), // Should be after MergeWindows to avoid unnecessary reordering of // mergeable windows new MergeProjections(), new PruneUnreferencedOutputs(), // Make sure to run this at the end to help clean the plan // for logging/execution and not remove info that other // optimizers might need at an earlier point new PruneIdentityProjections(), // This MUST run after PruneUnreferencedOutputs as it may // introduce new redundant projections new MetadataQueryOptimizer(metadata), new EliminateCrossJoins(), // This can pull up Filter and Project nodes from between Joins, // so we need to push them down again new PredicatePushDown(metadata, sqlParser), new ProjectionPushDown()); if (featuresConfig.isOptimizeSingleDistinct()) { builder.add(new SingleDistinctOptimizer()); builder.add(new PruneUnreferencedOutputs()); } builder.add(new OptimizeMixedDistinctAggregations(metadata)); if (!forceSingleNode) { builder.add(new PushTableWriteThroughUnion()); // Must run before AddExchanges builder.add(new AddExchanges(metadata, sqlParser)); } builder.add(new PickLayout(metadata)); builder.add(new EmptyDeleteOptimizer()); // Run after table scan is removed by PickLayout builder.add( new PredicatePushDown( metadata, sqlParser)); // Run predicate push down one more time in case we can leverage new // information from layouts' effective predicate builder.add(new ProjectionPushDown()); builder.add(new MergeProjections()); builder.add( new UnaliasSymbolReferences()); // Run unalias after merging projections to simplify // projections more efficiently builder.add(new PruneUnreferencedOutputs()); builder.add(new PruneIdentityProjections()); // Optimizers above this don't understand local exchanges, so be careful moving this. builder.add(new AddLocalExchanges(metadata, sqlParser)); // Optimizers above this do not need to care about aggregations with the type other than SINGLE // This optimizer must be run after all exchange-related optimizers builder.add(new PartialAggregationPushDown(metadata.getFunctionRegistry())); builder.add(new PruneIdentityProjections()); // DO NOT add optimizers that change the plan shape (computations) after this point // Precomputed hashes - this assumes that partitioning will not change builder.add(new HashGenerationOptimizer()); builder.add(new MetadataDeleteOptimizer(metadata)); builder.add(new BeginTableWrite(metadata)); // HACK! see comments in BeginTableWrite // TODO: consider adding a formal final plan sanitization optimizer that prepares the plan for // transmission/execution/logging // TODO: figure out how to improve the set flattening optimizer so that it can run at any point this.optimizers = builder.build(); }