/**
* Pipeline joins
*
* @return
*/
public Operator pipeline() {
EqualJoin firstJoin = joinList.get(0);
TimeCalc.begin(firstJoin.toString());
OperatorHashJoin hashJoin = decideWhichToCreate(firstJoin, operMap);
TimeCalc.end(firstJoin.toString() + " Finish create the first hash join.");
joinedHistory.add(firstJoin.getLeftTableName());
joinedHistory.add(firstJoin.getRightTableName());
for (int i = 1; i < joinList.size(); i++) {
EqualJoin ej = joinList.get(i);
Operator leftOper = operMap.get(ej.getLeftTableName());
Operator rightOper = operMap.get(ej.getRightTableName());
Operator neverJoinedOper;
if (joinedHistory.contains(ej.getLeftTableName())) neverJoinedOper = rightOper;
else neverJoinedOper = leftOper;
TimeCalc.begin(ej.toString());
hashJoin = decideWhichToCreate(ej, hashJoin, neverJoinedOper);
TimeCalc.end(ej.toString() + " Finish pipeline one hash join " + ej);
joinedHistory.add(ej.getLeftTableName());
joinedHistory.add(ej.getRightTableName());
}
return hashJoin;
}
/**
* put the length of the largest difference of join to the first based on the max difference, the
* first join in the list, reorder it by the finding the one which overlap the previous join, each
* join should have one table overlapped in the previous join in order to pipeline them
*
* @param joins
*/
private void reorder(List<EqualJoin> joins) {
//
int n = joins.size();
int maxIndex = 0;
long min = Integer.MIN_VALUE;
// find the index of the max difference between left and right
for (int i = 0; i < n; i++) {
EqualJoin ej = joins.get(i);
long diff = ej.getLeftSize() - ej.getRightSize();
if (diff < 0) diff = diff * (-1);
if (diff > min) {
maxIndex = i;
min = diff;
}
}
// swap the max diff value to the first
swap(joins, 0, maxIndex);
// reorder based one the first
Set<String> previous = new HashSet<String>(n + 1);
previous.add(joins.get(0).getLeftTableName());
previous.add(joins.get(0).getRightTableName());
for (int i = 1; i < n; i++) {
EqualJoin current = joins.get(i);
boolean left = previous.contains(current.getLeftTableName());
boolean right = previous.contains(current.getRightTableName());
if (!left && !right) {
// both not contains, then switch
for (int j = i + 1; j < n; j++) {
EqualJoin after = joins.get(j);
boolean leftAfter = previous.contains(after.getLeftTableName());
boolean rightAfter = previous.contains(after.getRightTableName());
if (leftAfter || rightAfter) {
swap(joins, i, j);
previous.add(after.getLeftTableName());
previous.add(after.getRightTableName());
break;
}
} // end for j
} else {
previous.add(current.getLeftTableName());
previous.add(current.getRightTableName());
}
} // end for i
}
/**
* Decide which join to create the first join
*
* @param ej
* @param operMap
* @return
*/
private OperatorHashJoin decideWhichToCreate(EqualJoin ej, Map<String, Operator> operMap) {
Operator left = operMap.get(ej.getLeftTableName());
Operator right = operMap.get(ej.getRightTableName());
if (left.getLength() > Config.FileThreshold_MB && right.getLength() > Config.FileThreshold_MB) {
return new OperatorHashJoinExternal(ej, left, right);
} else {
return new OperatorHashJoinMem(ej, left, right);
}
}
public JoinManager(List<EqualJoin> joins, Map<String, Operator> operMapIn, File swapDirIn) {
joinedHistory = new HashSet<String>();
joinList = joins;
operMap = operMapIn;
// find and set size
for (EqualJoin ej : joins) {
ej.setLeftSize(operMap.get(ej.getLeftTableName()).getLength());
ej.setRightSize(operMap.get(ej.getRightTableName()).getLength());
}
// move the biggest last, and
// make each of them has an overlap,
// so that easy to pipeline
reorder(joinList);
}