private boolean mergeSortWithDupRemoval(PlanNode node) {
if (node.getFirstChild() == null) {
return false;
}
switch (node.getFirstChild().getType()) {
case NodeConstants.Types.SET_OP:
if (node.getFirstChild().getProperty(NodeConstants.Info.SET_OPERATION)
== SetQuery.Operation.UNION
&& !node.getFirstChild().hasBooleanProperty(NodeConstants.Info.USE_ALL)) {
node.getFirstChild().setProperty(NodeConstants.Info.USE_ALL, true);
return true;
}
break;
case NodeConstants.Types.DUP_REMOVE:
NodeEditor.removeChildNode(node, node.getFirstChild());
return true;
}
if (node.hasBooleanProperty(Info.UNRELATED_SORT)) {
PlanNode source = NodeEditor.findNodePreOrder(node, NodeConstants.Types.SOURCE);
if (source != null) {
PlanNode parentProject = NodeEditor.findParent(source, NodeConstants.Types.PROJECT);
if (parentProject != null
&& parentProject
.getProperty(Info.PROJECT_COLS)
.equals(source.getProperty(Info.OUTPUT_COLS))) {
// can't sort on a derived expression
return mergeSortWithDupRemoval(source);
}
}
}
return false;
}
/**
* Walk the tree pre-order, finding all access nodes that are candidates and adding them to the
* matches list.
*
* @param metadata Metadata implementation
* @param node Root node to search
* @param matches Collection to accumulate matches in
* @throws TeiidComponentException
* @throws QueryMetadataException
*/
List<CandidateJoin> findCandidate(
PlanNode root, QueryMetadataInterface metadata, AnalysisRecord analysisRecord)
throws QueryMetadataException, TeiidComponentException {
List<CandidateJoin> candidates = new ArrayList<CandidateJoin>();
for (PlanNode joinNode :
NodeEditor.findAllNodes(root, NodeConstants.Types.JOIN, NodeConstants.Types.ACCESS)) {
CandidateJoin candidate = null;
for (Iterator<PlanNode> j = joinNode.getChildren().iterator(); j.hasNext(); ) {
PlanNode child = j.next();
child = FrameUtil.findJoinSourceNode(child);
if (child.hasBooleanProperty(NodeConstants.Info.MAKE_NOT_DEP)
|| !isValidJoin(joinNode, child, analysisRecord)) {
continue;
}
if (candidate == null) {
candidate = new CandidateJoin();
candidate.joinNode = joinNode;
candidates.add(candidate);
}
if (j.hasNext()) {
candidate.leftCandidate = true;
} else {
candidate.rightCandidate = true;
}
}
}
return candidates;
}
private boolean mergeSortWithDupRemovalAcrossSource(PlanNode toTest) {
PlanNode source =
NodeEditor.findNodePreOrder(
toTest,
NodeConstants.Types.SOURCE,
NodeConstants.Types.ACCESS | NodeConstants.Types.JOIN);
return source != null && mergeSortWithDupRemoval(source);
}
private PlanNode optimizeSorts(
boolean parentBlocking,
PlanNode node,
PlanNode root,
QueryMetadataInterface metadata,
CapabilitiesFinder capFinder,
AnalysisRecord record,
CommandContext context)
throws QueryMetadataException, TeiidComponentException, QueryPlannerException {
node =
NodeEditor.findNodePreOrder(
node,
NodeConstants.Types.SORT
| NodeConstants.Types.DUP_REMOVE
| NodeConstants.Types.GROUP
| NodeConstants.Types.JOIN
| NodeConstants.Types.SET_OP,
NodeConstants.Types.ACCESS);
if (node == null) {
return root;
}
switch (node.getType()) {
case NodeConstants.Types.SORT:
parentBlocking = true;
if (node.hasBooleanProperty(NodeConstants.Info.IS_DUP_REMOVAL)) {
break;
}
if (mergeSortWithDupRemoval(node)) {
node.setProperty(NodeConstants.Info.IS_DUP_REMOVAL, true);
} else {
root = checkForProjectOptimization(node, root, metadata, capFinder, record, context);
if (NodeEditor.findParent(node, NodeConstants.Types.ACCESS) != null) {
return root;
}
}
OrderBy orderBy = (OrderBy) node.getProperty(NodeConstants.Info.SORT_ORDER);
List<Expression> orderColumns = orderBy.getSortKeys();
List<Expression> sortExpressions = new ArrayList<Expression>(orderColumns.size());
PlanNode possibleSort =
NodeEditor.findNodePreOrder(
node,
NodeConstants.Types.GROUP,
NodeConstants.Types.SOURCE | NodeConstants.Types.ACCESS);
if (possibleSort != null && !possibleSort.hasBooleanProperty(Info.ROLLUP)) {
boolean otherExpression = false;
SymbolMap groupMap = (SymbolMap) possibleSort.getProperty(Info.SYMBOL_MAP);
for (Expression singleElementSymbol : orderColumns) {
Expression ex = SymbolMap.getExpression(singleElementSymbol);
if (ex instanceof ElementSymbol) {
sortExpressions.add(groupMap.getMappedExpression((ElementSymbol) ex));
} else {
otherExpression = true;
break;
}
}
List<Expression> exprs = (List<Expression>) possibleSort.getProperty(Info.GROUP_COLS);
if (!otherExpression && exprs != null && exprs.containsAll(sortExpressions)) {
exprs.removeAll(sortExpressions);
exprs.addAll(0, sortExpressions);
if (node.getParent() == null) {
root = node.getFirstChild();
root.removeFromParent();
node = root;
} else {
PlanNode nextNode = node.getFirstChild();
NodeEditor.removeChildNode(node.getParent(), node);
node = nextNode;
}
possibleSort.setProperty(Info.SORT_ORDER, orderBy);
}
}
break;
case NodeConstants.Types.DUP_REMOVE:
if (parentBlocking) {
node.setType(NodeConstants.Types.SORT);
node.setProperty(NodeConstants.Info.IS_DUP_REMOVAL, true);
}
break;
case NodeConstants.Types.GROUP:
if (!node.hasCollectionProperty(NodeConstants.Info.GROUP_COLS)) {
break;
}
SymbolMap map = (SymbolMap) node.getProperty(Info.SYMBOL_MAP);
boolean cardinalityDependent = false;
boolean canOptimize = true;
for (Expression ex : map.asMap().values()) {
if (ex instanceof AggregateSymbol) {
AggregateSymbol agg = (AggregateSymbol) ex;
if (agg.isCardinalityDependent()) {
cardinalityDependent = true;
break;
}
} else if (!(ex instanceof ElementSymbol)) {
// there is an expression in the grouping columns
canOptimize = false;
break;
}
}
if (canOptimize && mergeSortWithDupRemovalAcrossSource(node)) {
node.setProperty(NodeConstants.Info.IS_DUP_REMOVAL, true);
if (cardinalityDependent) {
PlanNode source = NodeEditor.findNodePreOrder(node, NodeConstants.Types.SOURCE);
List<Expression> sourceOutput = (List<Expression>) source.getProperty(Info.OUTPUT_COLS);
PlanNode child = node.getFirstChild();
while (child != source) {
child.setProperty(Info.OUTPUT_COLS, sourceOutput);
child = child.getFirstChild();
}
}
}
// TODO: check the join interesting order
parentBlocking = true;
break;
case NodeConstants.Types.JOIN:
if (node.getProperty(NodeConstants.Info.JOIN_STRATEGY) == JoinStrategyType.NESTED_LOOP
|| node.getProperty(NodeConstants.Info.JOIN_STRATEGY)
== JoinStrategyType.NESTED_TABLE) {
break;
}
/*
* Look under the left and the right sources for a dup removal operation
* join
* [project]
* source
* dup remove | union not all
*/
parentBlocking = true;
PlanNode toTest = node.getFirstChild();
if (mergeSortWithDupRemovalAcrossSource(toTest)) {
node.setProperty(NodeConstants.Info.SORT_LEFT, SortOption.SORT_DISTINCT);
if (node.getProperty(NodeConstants.Info.SORT_RIGHT) != SortOption.SORT) {
node.setProperty(NodeConstants.Info.JOIN_STRATEGY, JoinStrategyType.MERGE);
}
}
toTest = node.getLastChild();
if (mergeSortWithDupRemovalAcrossSource(toTest)) {
node.setProperty(NodeConstants.Info.SORT_RIGHT, SortOption.SORT_DISTINCT);
if (node.getProperty(NodeConstants.Info.SORT_LEFT) != SortOption.SORT) {
node.setProperty(NodeConstants.Info.JOIN_STRATEGY, JoinStrategyType.MERGE);
}
}
break;
case NodeConstants.Types.SET_OP:
// assumes the use of the merge algorithm
if (node.getProperty(NodeConstants.Info.SET_OPERATION) != SetQuery.Operation.UNION) {
parentBlocking = true;
} else if (!node.hasBooleanProperty(NodeConstants.Info.USE_ALL) && !parentBlocking) {
// do the incremental dup removal for lower latency
node.setProperty(NodeConstants.Info.IS_DUP_REMOVAL, true);
}
break;
}
for (PlanNode child : node.getChildren()) {
root = optimizeSorts(parentBlocking, child, root, metadata, capFinder, record, context);
}
return root;
}
private PlanNode checkForProjectOptimization(
PlanNode node,
PlanNode root,
QueryMetadataInterface metadata,
CapabilitiesFinder capFinder,
AnalysisRecord record,
CommandContext context)
throws QueryMetadataException, TeiidComponentException, QueryPlannerException {
PlanNode projectNode = node.getFirstChild();
PlanNode parent = node.getParent();
boolean raiseAccess = false;
// special check for unrelated order by compensation
if (projectNode.getType() == NodeConstants.Types.ACCESS
&& RuleRaiseAccess.canRaiseOverSort(
projectNode, metadata, capFinder, node, record, true, context)) {
projectNode =
NodeEditor.findNodePreOrder(
projectNode,
NodeConstants.Types.PROJECT,
NodeConstants.Types.SOURCE | NodeConstants.Types.SET_OP);
if (projectNode == null) {
return root; // no interviening project
}
raiseAccess = true;
} else if (projectNode.getType() == NodeConstants.Types.PROJECT
&& projectNode.getFirstChild() != null) {
raiseAccess =
projectNode.getFirstChild().getType() == NodeConstants.Types.ACCESS
&& RuleRaiseAccess.canRaiseOverSort(
projectNode.getFirstChild(), metadata, capFinder, node, record, false, context);
// if we can't raise the access node and this doesn't have a limit, there's no point in
// optimizing
if (!raiseAccess && (parent == null || parent.getType() != NodeConstants.Types.TUPLE_LIMIT)) {
return root;
}
} else {
return root;
}
List<Expression> childOutputCols =
(List<Expression>) projectNode.getFirstChild().getProperty(Info.OUTPUT_COLS);
OrderBy orderBy = (OrderBy) node.getProperty(Info.SORT_ORDER);
List<Expression> orderByKeys = orderBy.getSortKeys();
LinkedHashSet<Expression> toProject = new LinkedHashSet();
for (Expression ss : orderByKeys) {
Expression original = ss;
if (ss instanceof AliasSymbol) {
ss = ((AliasSymbol) ss).getSymbol();
}
if (ss instanceof ExpressionSymbol) {
if (!raiseAccess) {
return root; // TODO: insert a new project node to handle this case
}
}
if (!childOutputCols.contains(ss)) {
if (!raiseAccess) {
return root;
}
toProject.add(original);
}
}
PlanNode toRepair = projectNode.getParent();
if (!toProject.isEmpty()) {
PlanNode intermediateProject = NodeFactory.getNewNode(NodeConstants.Types.PROJECT);
toProject.addAll(childOutputCols);
List<Expression> projectCols = new ArrayList<Expression>(toProject);
childOutputCols = projectCols;
intermediateProject.setProperty(NodeConstants.Info.PROJECT_COLS, projectCols);
intermediateProject.setProperty(
NodeConstants.Info.OUTPUT_COLS, new ArrayList<Expression>(projectCols));
toRepair.getFirstChild().addAsParent(intermediateProject);
}
NodeEditor.removeChildNode(projectNode.getParent(), projectNode);
if (parent != null
&& parent.getType() == NodeConstants.Types.TUPLE_LIMIT
&& parent.getParent() != null) {
parent.addAsParent(projectNode);
} else {
if (parent == null) {
root = projectNode;
}
if (parent != null && parent.getType() == NodeConstants.Types.TUPLE_LIMIT) {
if (root == parent) {
root = projectNode;
}
projectNode.addFirstChild(parent);
} else {
projectNode.addFirstChild(node);
}
}
List<Expression> orderByOutputSymbols = (List<Expression>) node.getProperty(Info.OUTPUT_COLS);
boolean unrelated = false;
if (node.hasBooleanProperty(Info.UNRELATED_SORT)) {
node.setProperty(Info.UNRELATED_SORT, false);
unrelated = true;
}
for (OrderByItem item : orderBy.getOrderByItems()) {
if (unrelated || !toProject.isEmpty()) {
// update sort order
int index = childOutputCols.indexOf(item.getSymbol());
item.setExpressionPosition(index);
}
if (toProject.isEmpty()) {
// strip alias as project was raised
if (item.getSymbol() instanceof AliasSymbol) {
item.setSymbol(((AliasSymbol) item.getSymbol()).getSymbol());
}
}
}
while (toRepair != node) {
toRepair.setProperty(Info.OUTPUT_COLS, childOutputCols);
toRepair = toRepair.getParent();
}
projectNode.setProperty(Info.OUTPUT_COLS, orderByOutputSymbols);
projectNode.setProperty(Info.PROJECT_COLS, orderByOutputSymbols);
node.setProperty(Info.OUTPUT_COLS, childOutputCols);
if (parent != null) {
parent.setProperty(Info.OUTPUT_COLS, childOutputCols);
}
if (raiseAccess) {
PlanNode accessNode = NodeEditor.findNodePreOrder(node, NodeConstants.Types.ACCESS);
// instead of just calling ruleraiseaccess, we're more selective
// we do not want to raise the access node over a project that is handling an unrelated sort
PlanNode newRoot =
RuleRaiseAccess.raiseAccessNode(
root, accessNode, metadata, capFinder, true, record, context);
if (newRoot != null) {
accessNode.setProperty(NodeConstants.Info.OUTPUT_COLS, childOutputCols);
root = newRoot;
if (!toProject.isEmpty()) {
newRoot =
RuleRaiseAccess.raiseAccessNode(
root, accessNode, metadata, capFinder, true, record, context);
}
if (newRoot != null) {
root = newRoot;
if (accessNode.getParent().getType() == NodeConstants.Types.TUPLE_LIMIT) {
newRoot =
RulePushLimit.raiseAccessOverLimit(
root, accessNode, metadata, capFinder, accessNode.getParent(), record);
}
if (newRoot != null) {
root = newRoot;
}
}
}
}
return root;
}
/**
* Mark the specified access node to be made dependent
*
* @param sourceNode Node to make dependent
* @param dca
* @throws TeiidComponentException
* @throws QueryMetadataException
*/
boolean markDependent(
PlanNode sourceNode,
PlanNode joinNode,
QueryMetadataInterface metadata,
DependentCostAnalysis dca,
Boolean bound)
throws QueryMetadataException, TeiidComponentException {
boolean isLeft = joinNode.getFirstChild() == sourceNode;
// Get new access join node properties based on join criteria
List independentExpressions =
(List)
(isLeft
? joinNode.getProperty(NodeConstants.Info.RIGHT_EXPRESSIONS)
: joinNode.getProperty(NodeConstants.Info.LEFT_EXPRESSIONS));
List dependentExpressions =
(List)
(isLeft
? joinNode.getProperty(NodeConstants.Info.LEFT_EXPRESSIONS)
: joinNode.getProperty(NodeConstants.Info.RIGHT_EXPRESSIONS));
if (independentExpressions == null || independentExpressions.isEmpty()) {
return false;
}
String id = "$dsc/id" + ID.getAndIncrement(); // $NON-NLS-1$
// Create DependentValueSource and set on the independent side as this will feed the values
joinNode.setProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE, id);
PlanNode indNode = isLeft ? joinNode.getLastChild() : joinNode.getFirstChild();
if (bound == null) {
List<PlanNode> sources = NodeEditor.findAllNodes(indNode, NodeConstants.Types.SOURCE);
for (PlanNode planNode : sources) {
for (GroupSymbol gs : planNode.getGroups()) {
if (gs.isTempTable()
&& metadata.getCardinality(gs.getMetadataID())
== QueryMetadataInterface.UNKNOWN_CARDINALITY) {
bound = true;
break;
}
}
}
if (bound == null) {
bound = false;
}
}
PlanNode crit =
getDependentCriteriaNode(
id,
independentExpressions,
dependentExpressions,
indNode,
metadata,
dca,
bound,
(MakeDep) sourceNode.getProperty(Info.MAKE_DEP));
sourceNode.addAsParent(crit);
if (isLeft) {
JoinUtil.swapJoinChildren(joinNode);
}
return true;
}