private LanguageNode handleAndFunction(FunctionCall fc) {
// and functions take incomplete simple parts and turn them into complete parts, if so desired
ArrayList<Part> incompletes = new ArrayList<Part>();
ArrayList<ExpressionNode> ok = new ArrayList<ExpressionNode>();
ArrayList<Part> subparts = new ArrayList<Part>();
for (ExpressionNode en : fc.getParameters()) {
Part p = state.get(en);
if (p == null || p.isComplete()) {
ok.add(en);
if (p != null) subparts.add(p);
continue;
}
incompletes.add(p);
}
if (incompletes.isEmpty()) return fc;
// now we have the problem of a mishmash of incompletes. some may be complex, some may be
// simple
// some may be collections. we need to handle cases like the following:
// (a = 1) and (b = 2) {a,b} (1 key)
// (a = 1) and (b = 2 or b = 3) {a,b} (2 keys)
// (a = 1 or a = 2) and (b = 3) {a,b} (2 keys)
// (a = 1 or a = 2) and (b = 3 or b = 4) {a,b} (4 keys here)
// all of the above, where the result is still not complete due to missing tenant column
MultiMap<Part, ColumnKey> needed = new MultiMap<Part, ColumnKey>();
MultiMap<ColumnKey, Part> classified = new MultiMap<ColumnKey, Part>();
for (Part p : incompletes) {
ListSet<ColumnKey> has = new ListSet<ColumnKey>();
has.addAll(p.getColumns());
needed.putAll(p, parent.getNeeded(has));
for (ColumnKey ck : has) {
classified.put(ck, p);
}
}
// so let's say we have a part that is (a = 1 and b = 2), needs c and tenant, and we have a
// part
// that is c in (1,2,3). The needed for (a = 1 and b = 2) is {c,tenant}. we'll pull (c in
// (1,2,3))
// so we'll get at least (a = 1 and b = 2 and c = 3) or (a = 1 and b =2 and c = 3) ...
// these we can then individually try to complete.
while (!needed.isEmpty()) {
combineParts(needed, classified, ok, subparts);
}
for (Part p : subparts) state.put(p.getParent(), p);
if (ok.size() == 1) return ok.get(0);
else {
// what's left is a mix of unrelated and complete or incomplete subexprs. unrelated nodes
// would come in from above, as would previously complete.
return new FunctionCall(FunctionName.makeAnd(), ok);
}
}
protected ListSet<Part> getCompletedParts() {
if (stopped) return null;
ListSet<Part> cols = new ListSet<Part>();
for (Part p : completedParts) {
if (p instanceof OredParts) cols.add(p);
}
completedParts.removeAll(cols);
for (Part p : cols) {
for (Part sp : p.getParts()) {
completedParts.remove(sp);
}
}
completedParts.addAll(cols);
return completedParts;
}
private Part buildNewComplexPart(Part lp, Part rp) {
ListSet<Part> allParts = new ListSet<Part>();
allParts.addAll(lp.getParts());
allParts.addAll(rp.getParts());
FunctionCall andCall =
new FunctionCall(
FunctionName.makeAnd(), Functional.apply(allParts, Part.castToExpression));
AndedParts cp = parent.buildAndedParts(andCall, allParts);
if (parent.isComplete(cp)) {
setComplete(cp);
} else if (!parent.isComplete(cp)) {
AndedParts ncp = parent.maybeMakeComplete(cp);
if (ncp != null) {
setComplete(ncp);
return ncp;
}
}
return cp;
}
private Part choosePartForAndCombo(ListSet<Part> in) {
Part out = null;
for (Part p : in) {
if (p instanceof OredParts) {
out = p;
break;
}
}
if (out == null) out = in.get(0);
return out;
}
public static SelectStatement filterEntryProjection(SelectStatement in, PartitionEntry jre)
throws PEException {
PartitionEntry actual = jre.getActualEntry();
SelectStatement expecting = null;
if (actual instanceof OriginalPartitionEntry)
expecting = ((OriginalPartitionEntry) actual).getChildCopy();
else expecting = actual.getJoinQuery(null);
ListSet<ColumnKey> ec = new ListSet<ColumnKey>();
for (ExpressionNode en : expecting.getProjection()) {
ExpressionNode targ = ExpressionUtils.getTarget(en);
if (targ instanceof ColumnInstance) {
ColumnKey was = ((ColumnInstance) targ).getColumnKey();
ColumnKey isnow = in.getMapper().copyColumnKeyForward(was);
if (isnow == null) {
throw new SchemaException(Pass.PLANNER, "Lost column during lookup table join");
}
ec.add(isnow);
} else if (targ instanceof FunctionCall) {
ExpressionNode exn = targ;
RewriteKey rk = in.getMapper().mapExpressionToColumn(exn);
while (rk == null && (exn.getParent() instanceof ExpressionNode)) {
exn = (ExpressionNode) exn.getParent();
rk = in.getMapper().mapExpressionToColumn(exn);
}
if (rk != null) {
ec.add((ColumnKey) rk);
}
}
}
for (Iterator<ExpressionNode> iter = in.getProjectionEdge().iterator(); iter.hasNext(); ) {
ExpressionNode en = ExpressionUtils.getTarget(iter.next());
if (en instanceof ColumnInstance) {
ColumnKey ck = ((ColumnInstance) en).getColumnKey();
if (!ec.contains(ck)) iter.remove();
}
}
return in;
}
@Override
public void plan(SchemaContext sc, ExecutionSequence es, BehaviorConfiguration config)
throws PEException {
MultiMap<PEDatabase, PETable> plain = new MultiMap<PEDatabase, PETable>();
MultiMap<PEDatabase, PETable> fks = new MultiMap<PEDatabase, PETable>();
MultiMap<PEDatabase, PEViewTable> views = new MultiMap<PEDatabase, PEViewTable>();
AddStorageSiteStatement.sortTablesOnGroup(sc, onGroup, plain, fks, views);
ListSet<PEDatabase> dbs = new ListSet<PEDatabase>();
dbs.addAll(views.keySet());
dbs.addAll(plain.keySet());
dbs.addAll(fks.keySet());
Pair<List<PEAbstractTable<?>>, Boolean> tableDeclOrder =
AddStorageSiteStatement.buildTableDeclOrder(sc, plain, fks);
MultiMap<RangeDistribution, PETable> tablesByRange = new MultiMap<RangeDistribution, PETable>();
for (PEAbstractTable<?> abst : tableDeclOrder.getFirst()) {
if (abst.isView()) continue;
PETable pet = abst.asTable();
if (pet.getDistributionVector(sc).isRange()) {
RangeDistributionVector rdv = (RangeDistributionVector) pet.getDistributionVector(sc);
tablesByRange.put(rdv.getRangeDistribution().getDistribution(sc), pet);
}
}
List<AddStorageGenRangeInfo> rangeInfo = buildRangeInfo(sc, tablesByRange);
/*
for(AddStorageGenRangeInfo info : rangeInfo) {
info.display(System.out);
}
*/
es.append(
new ComplexDDLExecutionStep(
null,
onGroup,
null,
Action.ALTER,
new RebalanceCallback(onGroup, rangeInfo, tableDeclOrder.getSecond())));
}
// we have the temp table on the temp group for the constrained side
// we need to build the bcast temp table on the pers group
public static RedistFeatureStep buildLookupTableRedist(
PlannerContext pc,
PartitionEntry srcEntry,
RedistFeatureStep constrainedOnTempGroup,
JoinEntry rje,
PEStorageGroup targetGroup,
boolean indexJoinColumns)
throws PEException {
ProjectingFeatureStep selectDistinct =
constrainedOnTempGroup.buildNewProjectingStep(
pc,
rje.getFeaturePlanner(),
null,
DMLExplainReason.LOOKUP_JOIN_LOOKUP_TABLE.makeRecord());
SelectStatement ss = (SelectStatement) selectDistinct.getPlannedStatement();
DistributionVector distVect =
constrainedOnTempGroup.getTargetTempTable().getDistributionVector(pc.getContext());
ListSet<ColumnKey> mappedColumnsInJoin = null;
if (rje.getJoin().getJoin() != null) {
ListSet<ColumnKey> columnsInJoin =
ColumnInstanceCollector.getColumnKeys(
ColumnInstanceCollector.getColumnInstances(rje.getJoin().getJoin().getJoinOn()));
// build the set of columns in the src entry projection
ListSet<ColumnKey> srcColumns = new ListSet<ColumnKey>();
for (BufferEntry be : srcEntry.getBufferEntries()) {
ExpressionNode targ = ExpressionUtils.getTarget(be.getTarget());
if (targ instanceof ColumnInstance) {
ColumnInstance ci = (ColumnInstance) targ;
srcColumns.add(ci.getColumnKey());
}
}
columnsInJoin.retainAll(srcColumns);
mappedColumnsInJoin = new ListSet<ColumnKey>();
for (ColumnKey ck : columnsInJoin) {
mappedColumnsInJoin.add(ss.getMapper().copyColumnKeyForward(ck));
}
}
for (Iterator<ExpressionNode> iter = ss.getProjectionEdge().iterator(); iter.hasNext(); ) {
ExpressionNode en = iter.next();
if (en instanceof ColumnInstance) {
ColumnInstance ci = (ColumnInstance) en;
PEColumn pec = ci.getPEColumn();
if (!distVect.contains(srcEntry.getSchemaContext(), pec)
&& (mappedColumnsInJoin == null || !mappedColumnsInJoin.contains(ci.getColumnKey())))
iter.remove();
}
}
ss.setSetQuantifier(SetQuantifier.DISTINCT);
ss.normalize(srcEntry.getSchemaContext());
// now that we have the select distinct set up
// redistribute it bcast back onto the pers group
RedistFeatureStep out =
selectDistinct.redist(
pc,
rje.getFeaturePlanner(),
new TempTableCreateOptions(Model.BROADCAST, targetGroup),
null,
DMLExplainReason.LOOKUP_JOIN_LOOKUP_TABLE.makeRecord());
if (indexJoinColumns)
out.getTargetTempTable()
.noteJoinedColumns(pc.getContext(), out.getTargetTempTable().getColumns(pc.getContext()));
return out;
}
@Override
public ListSet<Part> getParts() {
ListSet<Part> me = new ListSet<Part>();
me.add(this);
return me;
}
public OredParts(LanguageNode parent, TableKey tk, Collection<Part> p) {
super(parent, tk);
parts = new ListSet<Part>();
parts.addAll(p);
((ExpressionNode) parent).setGrouped();
}
private void combineParts(
MultiMap<Part, ColumnKey> needed,
MultiMap<ColumnKey, Part> classified,
List<ExpressionNode> andexprs,
List<Part> andparts) {
Part p = needed.keySet().iterator().next();
Collection<ColumnKey> missing = needed.get(p);
if (missing == null || missing.isEmpty()) {
andexprs.add((ExpressionNode) p.getParent());
needed.remove(p);
andparts.add(p);
return;
}
ListSet<Part> containingMissing = new ListSet<Part>();
boolean first = true;
for (ColumnKey ck : missing) {
Collection<Part> matching = classified.get(ck);
if (matching == null || matching.isEmpty()) continue;
if (first) containingMissing.addAll(matching);
else {
containingMissing.retainAll(matching);
}
}
if (containingMissing.isEmpty()) {
andexprs.add((ExpressionNode) p.getParent());
andparts.add(p);
needed.remove(p);
} else {
ListSet<Part> toCombine = new ListSet<Part>();
toCombine.add(p);
toCombine.addAll(containingMissing);
ListSet<Part> toRemove = new ListSet<Part>();
toRemove.addAll(toCombine);
Part clhs = choosePartForAndCombo(toCombine);
toCombine.remove(clhs);
while (!toCombine.isEmpty()) {
Part crhs = choosePartForAndCombo(toCombine);
toCombine.remove(crhs);
clhs = combineParts(clhs, crhs);
}
for (Part rp : toRemove) {
needed.remove(rp);
ArrayList<ColumnKey> classKeys = new ArrayList<ColumnKey>(classified.keySet());
for (ColumnKey ck : classKeys) {
Collection<Part> sub = classified.get(ck);
if (sub == null || sub.isEmpty()) continue;
if (sub.contains(rp)) classified.remove(ck, rp);
}
}
andexprs.add((ExpressionNode) clhs.getParent());
andparts.add(clhs);
}
}
private void setComplete(Part p) {
p.setComplete();
completedParts.add(p);
}
protected ListSet<ColumnKey> buildKeyColumns(TableKey tk, Collection<PEColumn> cols) {
ListSet<ColumnKey> out = new ListSet<ColumnKey>();
for (PEColumn c : cols) out.add(new ColumnKey(tk, c));
return out;
}