private void helpTestNextBatch(int tupleBatchSize, Mode mode) throws Exception { ProjectIntoNode node = new ProjectIntoNode(2); TupleSource tupleSource = new FakeDataTupleSource(NUM_ROWS); RelationalNode child = new FakeRelationalNode(1, tupleSource, tupleBatchSize); node.addChild(child); node.setIntoGroup(new GroupSymbol("myGroup")); // $NON-NLS-1$ ElementSymbol elementSymbol_1 = new ElementSymbol("myGroup.myElement1"); // $NON-NLS-1$ ElementSymbol elementSymbol_2 = new ElementSymbol("myGroup.myElement2"); // $NON-NLS-1$ elementSymbol_1.setType(Integer.class); elementSymbol_2.setType(String.class); List<ElementSymbol> elements = Arrays.asList(elementSymbol_1, elementSymbol_2); node.setIntoElements(elements); child.setElements(elements); node.setMode(mode); node.setModelName("myModel"); // $NON-NLS-1$ CommandContext context = new CommandContext(); BufferManager bm = BufferManagerFactory.getTestBufferManager(tupleBatchSize, tupleBatchSize); ProcessorDataManager dataManager = new FakePDM(tupleBatchSize); child.initialize(context, bm, dataManager); node.initialize(context, bm, dataManager); node.open(); TupleBatch batch = null; // Do the remaining batches while (true) { try { batch = node.nextBatch(); break; } catch (BlockedException e) { // Normal } } assertNotNull(batch); List[] tuples = batch.getAllTuples(); assertEquals(1, tuples.length); Object[] columns = tuples[0].toArray(); assertNotNull(columns); assertEquals(1, columns.length); // Should have inserted all rows assertEquals(new Integer(NUM_ROWS), columns[0]); }
protected RelationalNode convertNode(PlanNode node) throws TeiidComponentException, TeiidProcessingException { RelationalNode processNode = null; switch (node.getType()) { case NodeConstants.Types.PROJECT: GroupSymbol intoGroup = (GroupSymbol) node.getProperty(NodeConstants.Info.INTO_GROUP); if (intoGroup != null) { try { Insert insert = (Insert) node.getFirstChild().getProperty(Info.VIRTUAL_COMMAND); List<ElementSymbol> allIntoElements = insert.getVariables(); Object groupID = intoGroup.getMetadataID(); Object modelID = metadata.getModelID(groupID); String modelName = metadata.getFullName(modelID); if (metadata.isVirtualGroup(groupID)) { InsertPlanExecutionNode ipen = new InsertPlanExecutionNode(getID(), metadata); ipen.setProcessorPlan( (ProcessorPlan) node.getFirstChild().getProperty(Info.PROCESSOR_PLAN)); ipen.setReferences(insert.getValues()); processNode = ipen; } else { ProjectIntoNode pinode = new ProjectIntoNode(getID()); pinode.setIntoGroup(intoGroup); pinode.setIntoElements(allIntoElements); pinode.setModelName(modelName); pinode.setConstraint((Criteria) node.getProperty(Info.CONSTRAINT)); processNode = pinode; SourceCapabilities caps = capFinder.findCapabilities(modelName); if (caps.supportsCapability(Capability.INSERT_WITH_ITERATOR)) { pinode.setMode(org.teiid.query.processor.relational.ProjectIntoNode.Mode.ITERATOR); } else if (caps.supportsCapability(Capability.BATCHED_UPDATES)) { pinode.setMode(org.teiid.query.processor.relational.ProjectIntoNode.Mode.BATCH); } else { pinode.setMode(org.teiid.query.processor.relational.ProjectIntoNode.Mode.SINGLE); } } } catch (QueryMetadataException e) { throw new TeiidComponentException(QueryPlugin.Event.TEIID30247, e); } } else { List<Expression> symbols = (List) node.getProperty(NodeConstants.Info.PROJECT_COLS); ProjectNode pnode = new ProjectNode(getID()); pnode.setSelectSymbols(symbols); processNode = pnode; if (node.hasBooleanProperty(Info.HAS_WINDOW_FUNCTIONS)) { WindowFunctionProjectNode wfpn = new WindowFunctionProjectNode(getID()); // with partial projection the window function may already be pushed, we'll check for // that here ArrayList<Expression> filtered = new ArrayList<Expression>(); List<Expression> childSymbols = (List) node.getFirstChild().getProperty(NodeConstants.Info.OUTPUT_COLS); for (Expression ex : symbols) { ex = SymbolMap.getExpression(ex); if (childSymbols.contains(ex)) { continue; } filtered.add(ex); } Set<WindowFunction> windowFunctions = RuleAssignOutputElements.getWindowFunctions(filtered); if (!windowFunctions.isEmpty()) { // TODO: check for selecting all window functions List<Expression> outputElements = new ArrayList<Expression>(windowFunctions); // collect the other projected expressions for (Expression singleElementSymbol : (List<Expression>) node.getFirstChild().getProperty(Info.OUTPUT_COLS)) { outputElements.add(singleElementSymbol); } wfpn.setElements(outputElements); wfpn.init(); pnode.addChild(wfpn); } } } break; case NodeConstants.Types.JOIN: JoinType jtype = (JoinType) node.getProperty(NodeConstants.Info.JOIN_TYPE); JoinStrategyType stype = (JoinStrategyType) node.getProperty(NodeConstants.Info.JOIN_STRATEGY); JoinNode jnode = new JoinNode(getID()); jnode.setJoinType(jtype); jnode.setLeftDistinct(node.hasBooleanProperty(NodeConstants.Info.IS_LEFT_DISTINCT)); jnode.setRightDistinct(node.hasBooleanProperty(NodeConstants.Info.IS_RIGHT_DISTINCT)); List joinCrits = (List) node.getProperty(NodeConstants.Info.JOIN_CRITERIA); String depValueSource = (String) node.getProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE); SortOption leftSort = (SortOption) node.getProperty(NodeConstants.Info.SORT_LEFT); if (stype == JoinStrategyType.MERGE || stype == JoinStrategyType.ENHANCED_SORT) { MergeJoinStrategy mjStrategy = null; if (stype.equals(JoinStrategyType.ENHANCED_SORT)) { EnhancedSortMergeJoinStrategy esmjStrategy = new EnhancedSortMergeJoinStrategy( leftSort, (SortOption) node.getProperty(NodeConstants.Info.SORT_RIGHT)); esmjStrategy.setSemiDep(node.hasBooleanProperty(Info.IS_SEMI_DEP)); mjStrategy = esmjStrategy; } else { mjStrategy = new MergeJoinStrategy( leftSort, (SortOption) node.getProperty(NodeConstants.Info.SORT_RIGHT), false); } jnode.setJoinStrategy(mjStrategy); List leftExpressions = (List) node.getProperty(NodeConstants.Info.LEFT_EXPRESSIONS); List rightExpressions = (List) node.getProperty(NodeConstants.Info.RIGHT_EXPRESSIONS); jnode.setJoinExpressions(leftExpressions, rightExpressions); joinCrits = (List) node.getProperty(NodeConstants.Info.NON_EQUI_JOIN_CRITERIA); } else if (stype == JoinStrategyType.NESTED_TABLE) { NestedTableJoinStrategy ntjStrategy = new NestedTableJoinStrategy(); jnode.setJoinStrategy(ntjStrategy); SymbolMap references = (SymbolMap) node.getProperty(Info.LEFT_NESTED_REFERENCES); ntjStrategy.setLeftMap(references); references = (SymbolMap) node.getProperty(Info.RIGHT_NESTED_REFERENCES); ntjStrategy.setRightMap(references); } else { NestedLoopJoinStrategy nljStrategy = new NestedLoopJoinStrategy(); jnode.setJoinStrategy(nljStrategy); } Criteria joinCrit = Criteria.combineCriteria(joinCrits); jnode.setJoinCriteria(joinCrit); processNode = jnode; jnode.setDependentValueSource(depValueSource); break; case NodeConstants.Types.ACCESS: ProcessorPlan plan = (ProcessorPlan) node.getProperty(NodeConstants.Info.PROCESSOR_PLAN); if (plan != null) { PlanExecutionNode peNode = null; Criteria crit = (Criteria) node.getProperty(NodeConstants.Info.PROCEDURE_CRITERIA); if (crit != null) { List references = (List) node.getProperty(NodeConstants.Info.PROCEDURE_INPUTS); List defaults = (List) node.getProperty(NodeConstants.Info.PROCEDURE_DEFAULTS); peNode = new DependentProcedureExecutionNode(getID(), crit, references, defaults); } else { peNode = new PlanExecutionNode(getID()); } peNode.setProcessorPlan(plan); processNode = peNode; } else { AccessNode aNode = null; Command command = (Command) node.getProperty(NodeConstants.Info.ATOMIC_REQUEST); Object modelID = node.getProperty(NodeConstants.Info.MODEL_ID); if (modelID != null) { String fullName = metadata.getFullName(modelID); if (!capFinder.isValid(fullName)) { // TODO: we ideally want to handle the partial resutls case here differently // by adding a null node / and a source warning // for now it's just as easy to say that the user needs to take steps to // return static capabilities SourceCapabilities caps = capFinder.findCapabilities(fullName); Exception cause = null; if (caps != null) { cause = (Exception) caps.getSourceProperty(Capability.INVALID_EXCEPTION); } throw new QueryPlannerException( QueryPlugin.Event.TEIID30498, cause, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30498, fullName)); } } EvaluatableVisitor ev = null; if (node.hasBooleanProperty(NodeConstants.Info.IS_DEPENDENT_SET)) { if (command instanceof StoredProcedure) { List references = (List) node.getProperty(NodeConstants.Info.PROCEDURE_INPUTS); List defaults = (List) node.getProperty(NodeConstants.Info.PROCEDURE_DEFAULTS); Criteria crit = (Criteria) node.getProperty(NodeConstants.Info.PROCEDURE_CRITERIA); DependentProcedureAccessNode depAccessNode = new DependentProcedureAccessNode(getID(), crit, references, defaults); processNode = depAccessNode; aNode = depAccessNode; } else { // create dependent access node DependentAccessNode depAccessNode = new DependentAccessNode(getID()); if (modelID != null) { depAccessNode.setPushdown( CapabilitiesUtil.supports( Capability.DEPENDENT_JOIN, modelID, metadata, capFinder)); depAccessNode.setMaxSetSize( CapabilitiesUtil.getMaxInCriteriaSize(modelID, metadata, capFinder)); depAccessNode.setMaxPredicates( CapabilitiesUtil.getMaxDependentPredicates(modelID, metadata, capFinder)); depAccessNode.setUseBindings( CapabilitiesUtil.supports( Capability.DEPENDENT_JOIN_BINDINGS, modelID, metadata, capFinder)); // TODO: allow the translator to drive this property // simplistic check of whether this query is complex to re-execute Query query = (Query) command; if (query.getGroupBy() != null || query.getFrom().getClauses().size() > 1 || !(query.getFrom().getClauses().get(0) instanceof UnaryFromClause) || query.getWith() != null) { depAccessNode.setComplexQuery(true); } else { // check to see if there in an index on at least one of the dependent sets Set<GroupSymbol> groups = new HashSet<GroupSymbol>(query.getFrom().getGroups()); boolean found = false; for (Criteria crit : Criteria.separateCriteriaByAnd(query.getCriteria())) { if (crit instanceof DependentSetCriteria) { DependentSetCriteria dsc = (DependentSetCriteria) crit; if (NewCalculateCostUtil.getKeyUsed( ElementCollectorVisitor.getElements(dsc.getExpression(), true), groups, metadata, null) != null) { found = true; break; } } } if (!found) { depAccessNode.setComplexQuery(true); } } } processNode = depAccessNode; aNode = depAccessNode; } aNode.setShouldEvaluateExpressions(true); } else { // create access node aNode = new AccessNode(getID()); processNode = aNode; } // -- special handling for system tables. currently they cannot perform projection try { if (command instanceof Query) { processNode = correctProjectionInternalTables(node, aNode); } } catch (QueryMetadataException err) { throw new TeiidComponentException(QueryPlugin.Event.TEIID30248, err); } setRoutingName(aNode, node, command); boolean shouldEval = false; if (command instanceof Insert) { Insert insert = (Insert) command; if (insert.getQueryExpression() != null) { insert.setQueryExpression( (QueryCommand) aliasCommand(aNode, insert.getQueryExpression(), modelID)); } else { for (int i = 0; i < insert.getValues().size(); i++) { Expression ex = (Expression) insert.getValues().get(i); if (!CriteriaCapabilityValidatorVisitor.canPushLanguageObject( ex, modelID, metadata, capFinder, analysisRecord)) { // replace with an expression symbol to let the rewriter know that it should be // replaced insert.getValues().set(i, new ExpressionSymbol("x", ex)); shouldEval = true; } } } } else if (command instanceof QueryCommand) { command = aliasCommand(aNode, command, modelID); } ev = EvaluatableVisitor.needsEvaluation(command, modelID, metadata, capFinder); aNode.setShouldEvaluateExpressions( ev.requiresEvaluation(EvaluationLevel.PROCESSING) || shouldEval); aNode.setCommand(command); Map<GroupSymbol, PlanNode> subPlans = (Map<GroupSymbol, PlanNode>) node.getProperty(Info.SUB_PLANS); // it makes more sense to allow the multisource affect to be elevated above just access // nodes if (aNode.getModelId() != null && metadata.isMultiSource(aNode.getModelId())) { VDBMetaData vdb = context.getVdb(); aNode.setShouldEvaluateExpressions(true); // forces a rewrite aNode.setElements((List) node.getProperty(NodeConstants.Info.OUTPUT_COLS)); if (node.hasBooleanProperty(Info.IS_MULTI_SOURCE)) { Expression ex = rewriteMultiSourceCommand(aNode.getCommand()); aNode.setConnectorBindingExpression(ex); aNode.setMultiSource(true); } else { String sourceName = (String) node.getProperty(Info.SOURCE_NAME); aNode.setConnectorBindingExpression(new Constant(sourceName)); } } else if (subPlans == null) { if (!aNode.isShouldEvaluate()) { aNode.minimizeProject(command); } // check if valid to share this with other nodes if (ev != null && ev.getDeterminismLevel().compareTo(Determinism.COMMAND_DETERMINISTIC) >= 0 && command.areResultsCachable()) { checkForSharedSourceCommand(aNode); } } if (subPlans != null) { QueryCommand qc = (QueryCommand) command; if (qc.getWith() == null) { qc.setWith(new ArrayList<WithQueryCommand>(subPlans.size())); } Map<GroupSymbol, RelationalPlan> plans = new LinkedHashMap<GroupSymbol, RelationalPlan>(); for (Map.Entry<GroupSymbol, PlanNode> entry : subPlans.entrySet()) { RelationalPlan subPlan = convert(entry.getValue()); List<ElementSymbol> elems = ResolverUtil.resolveElementsInGroup(entry.getKey(), metadata); subPlan.setOutputElements(elems); plans.put(entry.getKey(), subPlan); WithQueryCommand withQueryCommand = new WithQueryCommand(entry.getKey(), elems, null); qc.getWith().add(withQueryCommand); } aNode.setSubPlans(plans); } } break; case NodeConstants.Types.SELECT: Criteria crit = (Criteria) node.getProperty(NodeConstants.Info.SELECT_CRITERIA); if (!node.hasCollectionProperty(Info.OUTPUT_COLS)) { // the late optimization to create a dependent join from a subquery introduces // criteria that have no output elements set // TODO that should be cleaner, but the logic currently expects to be run after join // implementation // and rerunning assign output elements seems excessive node.setProperty(Info.OUTPUT_COLS, node.getFirstChild().getProperty(Info.OUTPUT_COLS)); } SelectNode selnode = new SelectNode(getID()); selnode.setCriteria(crit); // in case the parent was a source selnode.setProjectedExpressions( (List<Expression>) node.getProperty(NodeConstants.Info.PROJECT_COLS)); processNode = selnode; break; case NodeConstants.Types.SORT: case NodeConstants.Types.DUP_REMOVE: if (node.getType() == NodeConstants.Types.DUP_REMOVE) { processNode = new DupRemoveNode(getID()); } else { SortNode sortNode = new SortNode(getID()); OrderBy orderBy = (OrderBy) node.getProperty(NodeConstants.Info.SORT_ORDER); if (orderBy != null) { sortNode.setSortElements(orderBy.getOrderByItems()); } if (node.hasBooleanProperty(NodeConstants.Info.IS_DUP_REMOVAL)) { sortNode.setMode(Mode.DUP_REMOVE_SORT); } processNode = sortNode; } break; case NodeConstants.Types.GROUP: GroupingNode gnode = new GroupingNode(getID()); gnode.setRollup(node.hasBooleanProperty(Info.ROLLUP)); SymbolMap groupingMap = (SymbolMap) node.getProperty(NodeConstants.Info.SYMBOL_MAP); gnode.setOutputMapping(groupingMap); gnode.setRemoveDuplicates(node.hasBooleanProperty(NodeConstants.Info.IS_DUP_REMOVAL)); List<Expression> gCols = (List) node.getProperty(NodeConstants.Info.GROUP_COLS); OrderBy orderBy = (OrderBy) node.getProperty(Info.SORT_ORDER); if (orderBy == null) { if (gCols != null) { LinkedHashSet<Expression> exprs = new LinkedHashSet<Expression>(); for (Expression ex : gCols) { exprs.add(SymbolMap.getExpression(ex)); } orderBy = new OrderBy( RuleChooseJoinStrategy.createExpressionSymbols( new ArrayList<Expression>(exprs))); } } else { HashSet<Expression> seen = new HashSet<Expression>(); for (int i = 0; i < gCols.size(); i++) { if (i < orderBy.getOrderByItems().size()) { OrderByItem orderByItem = orderBy.getOrderByItems().get(i); Expression ex = SymbolMap.getExpression(orderByItem.getSymbol()); if (!seen.add(ex)) { continue; } if (ex instanceof ElementSymbol) { ex = groupingMap.getMappedExpression((ElementSymbol) ex); orderByItem.setSymbol(new ExpressionSymbol("expr", ex)); // $NON-NLS-1$ } } else { orderBy.addVariable( new ExpressionSymbol("expr", gCols.get(i)), OrderBy.ASC); // $NON-NLS-1$ } } } if (orderBy != null) { gnode.setOrderBy(orderBy.getOrderByItems()); } processNode = gnode; break; case NodeConstants.Types.SOURCE: Object source = node.getProperty(NodeConstants.Info.TABLE_FUNCTION); if (source instanceof XMLTable) { XMLTable xt = (XMLTable) source; XMLTableNode xtn = new XMLTableNode(getID()); // we handle the projection filtering once here rather than repeating the // path analysis on a per plan basis updateGroupName(node, xt); Map<Expression, Integer> elementMap = RelationalNode.createLookupMap(xt.getProjectedSymbols()); List cols = (List) node.getProperty(NodeConstants.Info.OUTPUT_COLS); int[] projectionIndexes = RelationalNode.getProjectionIndexes(elementMap, cols); ArrayList<XMLColumn> filteredColumns = new ArrayList<XMLColumn>(projectionIndexes.length); for (int col : projectionIndexes) { filteredColumns.add(xt.getColumns().get(col)); } xt.getXQueryExpression().useDocumentProjection(filteredColumns, analysisRecord); xtn.setProjectedColumns(filteredColumns); xtn.setTable(xt); processNode = xtn; break; } if (source instanceof ObjectTable) { ObjectTable ot = (ObjectTable) source; ObjectTableNode otn = new ObjectTableNode(getID()); // we handle the projection filtering once here rather than repeating the // path analysis on a per plan basis updateGroupName(node, ot); Map<Expression, Integer> elementMap = RelationalNode.createLookupMap(ot.getProjectedSymbols()); List<Expression> cols = (List<Expression>) node.getProperty(NodeConstants.Info.OUTPUT_COLS); int[] projectionIndexes = RelationalNode.getProjectionIndexes(elementMap, cols); ArrayList<ObjectColumn> filteredColumns = new ArrayList<ObjectColumn>(projectionIndexes.length); for (int col : projectionIndexes) { filteredColumns.add(ot.getColumns().get(col)); } otn.setProjectedColumns(filteredColumns); otn.setTable(ot); processNode = otn; break; } if (source instanceof TextTable) { TextTableNode ttn = new TextTableNode(getID()); TextTable tt = (TextTable) source; updateGroupName(node, tt); ttn.setTable(tt); processNode = ttn; break; } if (source instanceof ArrayTable) { ArrayTableNode atn = new ArrayTableNode(getID()); ArrayTable at = (ArrayTable) source; updateGroupName(node, at); atn.setTable(at); processNode = atn; break; } SymbolMap symbolMap = (SymbolMap) node.getProperty(NodeConstants.Info.SYMBOL_MAP); if (symbolMap != null) { PlanNode child = node.getLastChild(); if (child.getType() == NodeConstants.Types.PROJECT || child.getType() == NodeConstants.Types.SELECT) { // update the project cols based upon the original output child.setProperty( NodeConstants.Info.PROJECT_COLS, child.getProperty(NodeConstants.Info.OUTPUT_COLS)); } child.setProperty( NodeConstants.Info.OUTPUT_COLS, node.getProperty(NodeConstants.Info.OUTPUT_COLS)); } return null; case NodeConstants.Types.SET_OP: Operation setOp = (Operation) node.getProperty(NodeConstants.Info.SET_OPERATION); boolean useAll = ((Boolean) node.getProperty(NodeConstants.Info.USE_ALL)).booleanValue(); if (setOp == Operation.UNION) { RelationalNode unionAllNode = new UnionAllNode(getID()); if (useAll) { processNode = unionAllNode; } else { boolean onlyDupRemoval = node.hasBooleanProperty(NodeConstants.Info.IS_DUP_REMOVAL); if (onlyDupRemoval) { processNode = new DupRemoveNode(getID()); } else { SortNode sNode = new SortNode(getID()); sNode.setMode(Mode.DUP_REMOVE_SORT); processNode = sNode; } unionAllNode.setElements((List) node.getProperty(NodeConstants.Info.OUTPUT_COLS)); processNode.addChild(unionAllNode); } } else { JoinNode joinAsSet = new JoinNode(getID()); joinAsSet.setJoinStrategy( new MergeJoinStrategy(SortOption.SORT_DISTINCT, SortOption.SORT_DISTINCT, true)); // If we push these sorts, we will have to enforce null order, since nulls are equal here List leftExpressions = (List) node.getFirstChild().getProperty(NodeConstants.Info.OUTPUT_COLS); List rightExpressions = (List) node.getLastChild().getProperty(NodeConstants.Info.OUTPUT_COLS); joinAsSet.setJoinType( setOp == Operation.EXCEPT ? JoinType.JOIN_ANTI_SEMI : JoinType.JOIN_SEMI); joinAsSet.setJoinExpressions(leftExpressions, rightExpressions); processNode = joinAsSet; } break; case NodeConstants.Types.TUPLE_LIMIT: Expression rowLimit = (Expression) node.getProperty(NodeConstants.Info.MAX_TUPLE_LIMIT); Expression offset = (Expression) node.getProperty(NodeConstants.Info.OFFSET_TUPLE_COUNT); LimitNode ln = new LimitNode(getID(), rowLimit, offset); ln.setImplicit(node.hasBooleanProperty(Info.IS_IMPLICIT_LIMIT)); processNode = ln; break; case NodeConstants.Types.NULL: processNode = new NullNode(getID()); break; default: throw new QueryPlannerException( QueryPlugin.Event.TEIID30250, QueryPlugin.Util.gs( QueryPlugin.Event.TEIID30250, NodeConstants.getNodeTypeString(node.getType()))); } if (processNode != null) { processNode = prepareToAdd(node, processNode); } return processNode; }