private LanguageNode handleInFunction(FunctionCall fc) {
ExpressionNode lhs = fc.getParameters().get(0);
if (EngineConstant.COLUMN.has(lhs) && parent.isQualifyingColumn((ColumnInstance) lhs)) {
// only matches if all the rhs are constant
for (ExpressionNode en : fc.getParameters(1)) {
if (!EngineConstant.CONSTANT.has(en)) return fc;
}
ColumnInstance ci = (ColumnInstance) lhs;
if (!parent.isQualifyingColumn(ci.getPEColumn())) return fc;
ArrayList<ExpressionNode> subexprs = new ArrayList<ExpressionNode>();
ArrayList<Part> parts = new ArrayList<Part>();
for (ExpressionNode en : fc.getParameters(1)) {
ColumnInstance tci = (ColumnInstance) ci.copy(null);
ConstantExpression litex = (ConstantExpression) en;
ExpressionNode subeq = new FunctionCall(FunctionName.makeEquals(), tci, litex);
Part p = buildPart(subeq, tci, litex);
parts.add(p);
subexprs.add((ExpressionNode) p.getParent());
}
if (subexprs.size() > 1) {
FunctionCall orcall = new FunctionCall(FunctionName.makeOr(), subexprs);
OredParts pc = parent.buildOredParts(orcall, parts);
if (pc.isComplete()) setComplete(pc);
orcall.setGrouped();
state.put(orcall, pc);
return orcall;
} else {
Part p = parts.get(0);
return p.getParent();
}
}
return fc;
}
protected SelectStatement buildAggCommand(SelectStatement in) {
SelectStatement out = CopyVisitor.copy(in);
// we're going to build
// select Name, max(Engine), max(Version), max(Row_format),
// sum(Rows), avg(Avg_row_length), sum(Data_length),
// max(Max_data_length), sum(Index_length),
// sum(Data_free), max(Auto_increment), min(Create_time),
// max(Update_time), max(Check_time),
// max(Collation), null /*Checksum*/, max(Create_options), max(Comment) from temp1 group by Name
// eventually we will build another temp table of rewrite info (visible name, auto_inc values)
// and join
// but for now this is good enough
List<ExpressionNode> proj = new ArrayList<ExpressionNode>();
ExpressionAlias nameColumn = null;
for (ExpressionNode en : out.getProjection()) {
ExpressionAlias ea = (ExpressionAlias) en;
ColumnInstance ci = (ColumnInstance) ea.getTarget();
String cn = ci.getColumn().getName().getUnquotedName().get();
FunctionCall fc = null;
if (maxedColumns.contains(cn)) {
fc = new FunctionCall(FunctionName.makeMax(), ci);
} else if (summedColumns.contains(cn)) {
fc = new FunctionCall(FunctionName.makeSum(), ci);
} else if ("Name".equals(cn)) {
fc = null;
nameColumn = ea;
} else if ("Avg_row_length".equals(cn)) {
fc =
new FunctionCall(
FunctionName.makeRound(), new FunctionCall(FunctionName.makeAvg(), ci));
} else if ("Create_time".equals(cn)) {
fc = new FunctionCall(FunctionName.makeMin(), ci);
} else if ("Checksum".equals(cn)) {
fc = null;
ea =
new ExpressionAlias(
LiteralExpression.makeNullLiteral(),
new NameAlias(ci.getColumn().getName().getUnqualified()),
false);
} else {
throw new SchemaException(Pass.PLANNER, "Unknown show status column: " + cn);
}
if (fc != null) {
ea =
new ExpressionAlias(
fc, new NameAlias(ci.getColumn().getName().getUnqualified()), false);
}
proj.add(ea);
}
out.setProjection(proj);
SortingSpecification ngb = new SortingSpecification(nameColumn.buildAliasInstance(), true);
ngb.setOrdering(Boolean.FALSE);
out.getGroupBysEdge().add(ngb);
return out;
}
private EqualityPart makeNewEqualityPart(
EqualityPart existing, PEColumn c, ConstantExpression litex) {
TableKey tk = existing.getColumn().getColumnKey().getTableKey();
ColumnInstance nc = new ColumnInstance(c, tk.toInstance());
FunctionCall eq = new FunctionCall(FunctionName.makeEquals(), nc, litex);
EqualityPart eqp = buildEqualityPart(eq, nc, litex);
return eqp;
}
protected AndedParts completeKey(AndedParts existing, PEColumn c, ConstantExpression litex) {
EqualityPart ep = (EqualityPart) existing.getParts().get(0);
EqualityPart np = makeNewEqualityPart(ep, c, litex);
ArrayList<Part> subp = new ArrayList<Part>();
subp.addAll(existing.getParts());
subp.add(np);
List<ExpressionNode> subexprs = Functional.apply(subp, Part.castToExpression);
FunctionCall andEx = new FunctionCall(FunctionName.makeAnd(), subexprs);
return buildAndedParts(andEx, subp);
}
protected AndedParts completeKey(EqualityPart existing, PEColumn c, ConstantExpression litex) {
EqualityPart np = makeNewEqualityPart(existing, c, litex);
FunctionCall andEx =
new FunctionCall(
FunctionName.makeAnd(),
(ExpressionNode) existing.getParent(),
(ExpressionNode) np.getParent());
ArrayList<Part> subp = new ArrayList<Part>();
subp.add(existing);
subp.add(np);
return buildAndedParts(andEx, subp);
}
private LanguageNode handleAndFunction(FunctionCall fc) {
// and functions take incomplete simple parts and turn them into complete parts, if so desired
ArrayList<Part> incompletes = new ArrayList<Part>();
ArrayList<ExpressionNode> ok = new ArrayList<ExpressionNode>();
ArrayList<Part> subparts = new ArrayList<Part>();
for (ExpressionNode en : fc.getParameters()) {
Part p = state.get(en);
if (p == null || p.isComplete()) {
ok.add(en);
if (p != null) subparts.add(p);
continue;
}
incompletes.add(p);
}
if (incompletes.isEmpty()) return fc;
// now we have the problem of a mishmash of incompletes. some may be complex, some may be
// simple
// some may be collections. we need to handle cases like the following:
// (a = 1) and (b = 2) {a,b} (1 key)
// (a = 1) and (b = 2 or b = 3) {a,b} (2 keys)
// (a = 1 or a = 2) and (b = 3) {a,b} (2 keys)
// (a = 1 or a = 2) and (b = 3 or b = 4) {a,b} (4 keys here)
// all of the above, where the result is still not complete due to missing tenant column
MultiMap<Part, ColumnKey> needed = new MultiMap<Part, ColumnKey>();
MultiMap<ColumnKey, Part> classified = new MultiMap<ColumnKey, Part>();
for (Part p : incompletes) {
ListSet<ColumnKey> has = new ListSet<ColumnKey>();
has.addAll(p.getColumns());
needed.putAll(p, parent.getNeeded(has));
for (ColumnKey ck : has) {
classified.put(ck, p);
}
}
// so let's say we have a part that is (a = 1 and b = 2), needs c and tenant, and we have a
// part
// that is c in (1,2,3). The needed for (a = 1 and b = 2) is {c,tenant}. we'll pull (c in
// (1,2,3))
// so we'll get at least (a = 1 and b = 2 and c = 3) or (a = 1 and b =2 and c = 3) ...
// these we can then individually try to complete.
while (!needed.isEmpty()) {
combineParts(needed, classified, ok, subparts);
}
for (Part p : subparts) state.put(p.getParent(), p);
if (ok.size() == 1) return ok.get(0);
else {
// what's left is a mix of unrelated and complete or incomplete subexprs. unrelated nodes
// would come in from above, as would previously complete.
return new FunctionCall(FunctionName.makeAnd(), ok);
}
}
private Part buildNewMultiMultiPart(OredParts lp, OredParts rp) {
ArrayList<Part> newParts = new ArrayList<Part>();
for (Part lpc : lp.getParts()) {
for (Part rpc : rp.getParts()) {
newParts.add(combineParts(lpc.copy(), rpc.copy()));
}
}
FunctionCall orcall =
new FunctionCall(
FunctionName.makeOr(), Functional.apply(newParts, Part.castToExpression));
OredParts op = parent.buildOredParts(orcall, newParts);
if (op.isComplete()) setComplete(op);
return op;
}
private Part buildNewPartCollection(OredParts lp, Part rp) {
// rp could be simple or complex, lp has either simple or complex elements.
// the strategy here is to build a new complex part for each item in the collection, and
// return a new collection.
ArrayList<Part> newParts = new ArrayList<Part>();
for (Part sp : lp.getParts()) {
newParts.add(buildNewComplexPart(sp, rp.copy()));
}
FunctionCall orcall =
new FunctionCall(
FunctionName.makeOr(), Functional.apply(newParts, Part.castToExpression));
OredParts op = parent.buildOredParts(orcall, newParts);
if (op.isComplete()) setComplete(op);
return op;
}
@Override
public Part copy() {
List<Part> newparts = new ArrayList<Part>();
for (Part p : parts) newparts.add(p.copy());
FunctionCall newp =
new FunctionCall(
FunctionName.makeAnd(),
Functional.apply(
newparts,
new UnaryFunction<ExpressionNode, Part>() {
@Override
public ExpressionNode evaluate(Part object) {
return (ExpressionNode) object.getParent();
}
}));
return new AndedParts(newp, getTableKey(), newparts, complete);
}
private Part buildNewComplexPart(Part lp, Part rp) {
ListSet<Part> allParts = new ListSet<Part>();
allParts.addAll(lp.getParts());
allParts.addAll(rp.getParts());
FunctionCall andCall =
new FunctionCall(
FunctionName.makeAnd(), Functional.apply(allParts, Part.castToExpression));
AndedParts cp = parent.buildAndedParts(andCall, allParts);
if (parent.isComplete(cp)) {
setComplete(cp);
} else if (!parent.isComplete(cp)) {
AndedParts ncp = parent.maybeMakeComplete(cp);
if (ncp != null) {
setComplete(ncp);
return ncp;
}
}
return cp;
}
public static RedistFeatureStep buildLookupJoinRedist(
PlannerContext pc,
RedistFeatureStep lookupTable,
List<ExpressionNode> lookupJoinColumns,
PEStorageGroup targetGroup,
JoinEntry origEntry,
List<ExpressionNode> origJoinColumns,
PartitionEntry nonLookupSide)
throws PEException {
// we will modify the existing non lookup side projecting feature step, and add the lookup table
// as a requirement to it.
ProjectingFeatureStep nonLookupStep = (ProjectingFeatureStep) nonLookupSide.getStep(null);
nonLookupSide.maybeForceDoublePrecision(nonLookupStep);
SelectStatement lookupSelect = lookupTable.getTargetTempTable().buildSelect(pc.getContext());
SelectStatement nonLookupSelect = (SelectStatement) nonLookupStep.getPlannedStatement();
SelectStatement actualJoinStatement =
DMLStatementUtils.compose(pc.getContext(), nonLookupSelect, lookupSelect);
List<ExpressionNode> ands =
ExpressionUtils.decomposeAndClause(actualJoinStatement.getWhereClause());
// instead of building the join spec directly, map forward the original join condition from the
// joined table if it's available
IndexCollector ic = new IndexCollector();
if (origEntry.getJoin().getJoin() != null) {
FunctionCall mapped =
(FunctionCall)
actualJoinStatement
.getMapper()
.copyForward(origEntry.getJoin().getJoin().getJoinOn());
ands.add(mapped);
ListSet<ColumnInstance> cols = ColumnInstanceCollector.getColumnInstances(mapped);
for (ColumnInstance ci : cols) ic.addColumnInstance(ci);
} else {
Map<RewriteKey, ExpressionNode> projEntries = null;
for (int i = 0; i < origJoinColumns.size(); i++) {
ColumnKey mck =
actualJoinStatement.getMapper().mapExpressionToColumn(origJoinColumns.get(i));
ColumnKey muck =
actualJoinStatement.getMapper().mapExpressionToColumn(lookupJoinColumns.get(i));
ExpressionNode mc = null;
ExpressionNode muc = null;
if (mck == null || muck == null) {
if (projEntries == null) {
projEntries = new HashMap<RewriteKey, ExpressionNode>();
for (ExpressionNode en : actualJoinStatement.getProjectionEdge()) {
ExpressionNode actual = ExpressionUtils.getTarget(en);
if (actual instanceof ColumnInstance) {
projEntries.put(((ColumnInstance) actual).getColumnKey(), actual);
} else {
projEntries.put(new ExpressionKey(actual), actual);
}
}
}
if (mck == null)
mc =
(ExpressionNode)
projEntries.get(new ExpressionKey(origJoinColumns.get(i))).copy(null);
if (muck == null)
mc =
(ExpressionNode)
projEntries.get(new ExpressionKey(lookupJoinColumns.get(i))).copy(null);
}
if (mc == null) mc = mck.toInstance();
if (muc == null) muc = muck.toInstance();
if (mc instanceof ColumnInstance) ic.addColumnInstance((ColumnInstance) mc);
if (muc instanceof ColumnInstance) ic.addColumnInstance((ColumnInstance) muc);
FunctionCall eq = new FunctionCall(FunctionName.makeEquals(), mc, muc);
ands.add(eq);
}
}
ic.setIndexes(origEntry.getSchemaContext());
TempTable lookupEntryTarget = lookupTable.getTargetTempTable();
// everything from the lhs that's in the projection should be cleared - it's invisible
// note that we do it after building the where clause so that the original join condition can be
// mapped
for (Iterator<ExpressionNode> iter = actualJoinStatement.getProjectionEdge().iterator();
iter.hasNext(); ) {
ExpressionNode en = iter.next();
ExpressionNode targ = ExpressionUtils.getTarget(en);
if (targ instanceof ColumnInstance) {
ColumnInstance ci = (ColumnInstance) targ;
if (ci.getTableInstance().getAbstractTable() == lookupEntryTarget) iter.remove();
}
}
actualJoinStatement.setWhereClause(ExpressionUtils.safeBuildAnd(ands));
actualJoinStatement.normalize(origEntry.getSchemaContext());
// build a new projecting feature step
ProjectingFeatureStep lookupJoinStep =
DefaultFeatureStepBuilder.INSTANCE.buildProjectingStep(
origEntry.getPlannerContext(),
origEntry.getFeaturePlanner(),
actualJoinStatement,
new ExecutionCost(false, false, null, -1),
nonLookupStep.getSourceGroup(),
actualJoinStatement.getDatabase(origEntry.getSchemaContext()),
nonLookupStep.getDistributionVector(),
null,
DMLExplainReason.LOOKUP_JOIN.makeRecord());
// arrange for the children of the nonlookup side to become my children
// and add the lookup table step as well
lookupJoinStep.getSelfChildren().addAll(nonLookupStep.getAllChildren());
lookupJoinStep.getSelfChildren().add(lookupTable);
// children must be sequential - no need to modify here
List<Integer> mappedRedistOn =
nonLookupSide.mapDistributedOn(origJoinColumns, actualJoinStatement);
// now remove the lookup table from the mapper - have to do this pretty late - at this point
// the query won't be manipulated any more
actualJoinStatement.getMapper().remove(lookupEntryTarget);
RedistFeatureStep out =
lookupJoinStep.redist(
origEntry.getPlannerContext(),
origEntry.getFeaturePlanner(),
new TempTableCreateOptions(Model.STATIC, targetGroup)
.distributeOn(mappedRedistOn)
.withRowCount(origEntry.getScore().getRowCount()),
null,
DMLExplainReason.LOOKUP_JOIN.makeRecord());
return out;
}
/**
* The redundant column is eliminated and the column order is correct according to standard SQL:
*
* <p>First, coalesced common columns of the two joined tables, in the order in which they occur
* in the first table.
*
* <p>Second, columns unique to the first table, in order in which they occur in that table.
*
* <p>Third, columns unique to the second table, in order in which they occur in that table.
*/
private Collection<ExpressionNode> buildProjection(
final SchemaContext sc, final Map<TableInstance, Set<ColumnKey>> tablesAndJoinColumns) {
final ListSetMap<ColumnKey, ExpressionNode> projection =
new ListSetMap<ColumnKey, ExpressionNode>();
/*
* We need to coalesce the join columns by their unqualified names.
* Build a lookup map.
*/
final Map<ColumnKey, Set<ColumnInstance>> toCoalesceName =
new TreeMap<ColumnKey, Set<ColumnInstance>>(
new Comparator<ColumnKey>() {
@Override
public int compare(ColumnKey c1, ColumnKey c2) {
final String n1 = getColumnName(c1).get();
final String n2 = getColumnName(c2).get();
return n1.compareTo(n2);
}
});
/*
* Here we the projection. Join columns are inserted in order in which
* they occur in the first table and coalesced by name in set
* containers.
*/
for (final Map.Entry<TableInstance, Set<ColumnKey>> te : tablesAndJoinColumns.entrySet()) {
final Set<ColumnKey> entryJoinColumns = te.getValue();
for (final ColumnKey ck : entryJoinColumns) {
final ColumnInstance ci = ck.toInstance();
if (!toCoalesceName.containsKey(ck)) {
projection.put(ck, ci);
final Set<ColumnInstance> coalescedColumns = new HashSet<ColumnInstance>();
coalescedColumns.add(ci);
toCoalesceName.put(ck, coalescedColumns);
} else {
toCoalesceName.get(ck).add(ci);
}
}
addColumns(sc, projection, te.getKey(), entryJoinColumns);
}
/*
* We already have the right projection.
*/
if (toCoalesceName.isEmpty()) {
return projection.values();
}
/*
* It turns out that in case of multi-join statements MySQL also sorts
* adjacent join columns by the number of coalesced columns (by the
* frequency with which they appear in the USING clauses).
*
* i.e.
* "SELECT * FROM pe251C c RIGHT JOIN pe251D d USING (id1,id2,id3) LEFT OUTER JOIN pe251E e USING (id1,id3) ORDER BY c.id1;"
*
* Although the first-table order would be (id2, id1, id3):
* "COALESCE(id2, id2), COALESCE(id1, id1, id1), COALESCE(id3, id3, id3)"
*
* MySQL sorts the columns as (id1, id3, id2):
* "COALESCE(id1, id1, id1), COALESCE(id3, id3, id3), COALESCE(id2, id2)"
*
* Sort the adjacent coalesced column sets by their size using a stable
* algorithm.
*/
final Map<ColumnKey, Set<ColumnInstance>> toCoalesceByEntry =
new HashMap<ColumnKey, Set<ColumnInstance>>(toCoalesceName);
final int endIndex = projection.size();
for (int i = 0; i < endIndex; ++i) {
final ColumnKey ik = projection.getEntryAt(i).getKey();
if (toCoalesceByEntry.containsKey(ik)) {
int j = i;
while (j > 0) {
final ColumnKey jk1 = projection.getEntryAt(j).getKey();
final ColumnKey jk2 = projection.getEntryAt(j - 1).getKey();
if (toCoalesceByEntry.containsKey(jk1) && toCoalesceByEntry.containsKey(jk2)) {
final Set<ColumnInstance> jv1 = toCoalesceByEntry.get(jk1);
final Set<ColumnInstance> jv2 = toCoalesceByEntry.get(jk2);
if (jv1.size() > jv2.size()) {
projection.swap(jk1, jk2);
} else {
break;
}
} else {
break;
}
--j;
}
}
}
/*
* Finally, convert the coalesced column sets into COALESCE() function
* calls.
*/
for (final Map.Entry<ColumnKey, Set<ColumnInstance>> coalesceEntry :
toCoalesceName.entrySet()) {
final ColumnKey key = coalesceEntry.getKey();
final FunctionCall coalesce =
new FunctionCall(
FunctionName.makeCoalesce(), new ArrayList<ExpressionNode>(coalesceEntry.getValue()));
final ExpressionAlias coalesceAlias =
new ExpressionAlias(coalesce, new NameAlias(getColumnName(key)), false);
projection.put(key, coalesceAlias);
}
return projection.values();
}