@Test
public final void testComparatorsFromJoinQual() {
Schema outerSchema = new Schema();
outerSchema.addColumn("employee.id1", CatalogUtil.newSimpleDataType(Type.INT4));
outerSchema.addColumn("employee.id2", CatalogUtil.newSimpleDataType(Type.INT4));
Schema innerSchema = new Schema();
innerSchema.addColumn("people.fid1", CatalogUtil.newSimpleDataType(Type.INT4));
innerSchema.addColumn("people.fid2", CatalogUtil.newSimpleDataType(Type.INT4));
FieldEval f1 = new FieldEval("employee.id1", CatalogUtil.newSimpleDataType(Type.INT4));
FieldEval f2 = new FieldEval("people.fid1", CatalogUtil.newSimpleDataType(Type.INT4));
FieldEval f3 = new FieldEval("employee.id2", CatalogUtil.newSimpleDataType(Type.INT4));
FieldEval f4 = new FieldEval("people.fid2", CatalogUtil.newSimpleDataType(Type.INT4));
EvalNode joinQual = new BinaryEval(EvalType.EQUAL, f1, f2);
TupleComparator[] comparators =
PlannerUtil.getComparatorsFromJoinQual(joinQual, outerSchema, innerSchema);
Tuple t1 = new VTuple(2);
t1.put(0, DatumFactory.createInt4(1));
t1.put(1, DatumFactory.createInt4(2));
Tuple t2 = new VTuple(2);
t2.put(0, DatumFactory.createInt4(2));
t2.put(1, DatumFactory.createInt4(3));
TupleComparator outerComparator = comparators[0];
assertTrue(outerComparator.compare(t1, t2) < 0);
assertTrue(outerComparator.compare(t2, t1) > 0);
TupleComparator innerComparator = comparators[1];
assertTrue(innerComparator.compare(t1, t2) < 0);
assertTrue(innerComparator.compare(t2, t1) > 0);
// tests for composited join key
EvalNode joinQual2 = new BinaryEval(EvalType.EQUAL, f3, f4);
EvalNode compositedJoinQual = new BinaryEval(EvalType.AND, joinQual, joinQual2);
comparators =
PlannerUtil.getComparatorsFromJoinQual(compositedJoinQual, outerSchema, innerSchema);
outerComparator = comparators[0];
assertTrue(outerComparator.compare(t1, t2) < 0);
assertTrue(outerComparator.compare(t2, t1) > 0);
innerComparator = comparators[1];
assertTrue(innerComparator.compare(t1, t2) < 0);
assertTrue(innerComparator.compare(t2, t1) > 0);
}
public FileChunk getFileCunks(Path outDir, String startKey, String endKey, boolean last)
throws IOException {
BSTIndex index = new BSTIndex(new TajoConf());
BSTIndex.BSTIndexReader idxReader = index.getIndexReader(new Path(outDir, "index"));
idxReader.open();
Schema keySchema = idxReader.getKeySchema();
TupleComparator comparator = idxReader.getComparator();
LOG.info(
"BSTIndex is loaded from disk (" + idxReader.getFirstKey() + ", " + idxReader.getLastKey());
File data = new File(URI.create(outDir.toUri() + "/output"));
byte[] startBytes = Base64.decodeBase64(startKey);
byte[] endBytes = Base64.decodeBase64(endKey);
Tuple start;
Tuple end;
try {
start = RowStoreUtil.RowStoreDecoder.toTuple(keySchema, startBytes);
} catch (Throwable t) {
throw new IllegalArgumentException(
"StartKey: " + startKey + ", decoded byte size: " + startBytes.length, t);
}
try {
end = RowStoreUtil.RowStoreDecoder.toTuple(keySchema, endBytes);
} catch (Throwable t) {
throw new IllegalArgumentException(
"EndKey: " + endKey + ", decoded byte size: " + endBytes.length, t);
}
if (!comparator.isAscendingFirstKey()) {
Tuple tmpKey = start;
start = end;
end = tmpKey;
}
LOG.info(
"GET Request for "
+ data.getAbsolutePath()
+ " (start="
+ start
+ ", end="
+ end
+ (last ? ", last=true" : "")
+ ")");
if (idxReader.getFirstKey() == null && idxReader.getLastKey() == null) { // if # of rows is zero
LOG.info("There is no contents");
return null;
}
if (comparator.compare(end, idxReader.getFirstKey()) < 0
|| comparator.compare(idxReader.getLastKey(), start) < 0) {
LOG.info(
"Out of Scope (indexed data ["
+ idxReader.getFirstKey()
+ ", "
+ idxReader.getLastKey()
+ "], but request start:"
+ start
+ ", end: "
+ end);
return null;
}
long startOffset;
long endOffset;
try {
startOffset = idxReader.find(start);
} catch (IOException ioe) {
LOG.error(
"State Dump (the requested range: "
+ new TupleRange(keySchema, start, end)
+ ", idx min: "
+ idxReader.getFirstKey()
+ ", idx max: "
+ idxReader.getLastKey());
throw ioe;
}
try {
endOffset = idxReader.find(end);
if (endOffset == -1) {
endOffset = idxReader.find(end, true);
}
} catch (IOException ioe) {
LOG.error(
"State Dump (the requested range: "
+ new TupleRange(keySchema, start, end)
+ ", idx min: "
+ idxReader.getFirstKey()
+ ", idx max: "
+ idxReader.getLastKey());
throw ioe;
}
// if startOffset == -1 then case 2-1 or case 3
if (startOffset == -1) { // this is a hack
// if case 2-1 or case 3
try {
startOffset = idxReader.find(start, true);
} catch (IOException ioe) {
LOG.error(
"State Dump (the requested range: "
+ new TupleRange(keySchema, start, end)
+ ", idx min: "
+ idxReader.getFirstKey()
+ ", idx max: "
+ idxReader.getLastKey());
throw ioe;
}
}
if (startOffset == -1) {
throw new IllegalStateException(
"startOffset "
+ startOffset
+ " is negative \n"
+ "State Dump (the requested range: "
+ new TupleRange(keySchema, start, end)
+ ", idx min: "
+ idxReader.getFirstKey()
+ ", idx max: "
+ idxReader.getLastKey());
}
// if greater than indexed values
if (last || (endOffset == -1 && comparator.compare(idxReader.getLastKey(), end) < 0)) {
endOffset = data.length();
}
FileChunk chunk = new FileChunk(data, startOffset, endOffset - startOffset);
LOG.info("Retrieve File Chunk: " + chunk);
return chunk;
}
@Test
public final void testNext() throws IOException, PlanningException {
FileFragment[] frags =
StorageManager.splitNG(
conf, "employee", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE);
Path workDir = new Path(testDir, TestExternalSortExec.class.getName());
TaskAttemptContext ctx =
new TaskAttemptContext(
conf,
LocalTajoTestingUtility.newQueryUnitAttemptId(),
new FileFragment[] {frags[0]},
workDir);
ctx.setEnforcer(new Enforcer());
Expr expr = analyzer.parse(QUERIES[0]);
LogicalPlan plan = planner.createPlan(expr);
LogicalNode rootNode = plan.getRootBlock().getRoot();
PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf, sm);
PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
ProjectionExec proj = (ProjectionExec) exec;
// TODO - should be planed with user's optimization hint
if (!(proj.getChild() instanceof ExternalSortExec)) {
UnaryPhysicalExec sortExec = proj.getChild();
SeqScanExec scan = sortExec.getChild();
ExternalSortExec extSort =
new ExternalSortExec(ctx, sm, ((MemSortExec) sortExec).getPlan(), scan);
proj.setChild(extSort);
}
Tuple tuple;
Tuple preVal = null;
Tuple curVal;
int cnt = 0;
exec.init();
long start = System.currentTimeMillis();
TupleComparator comparator =
new TupleComparator(
proj.getSchema(),
new SortSpec[] {
new SortSpec(new Column("managerId", Type.INT4)),
new SortSpec(new Column("empId", Type.INT4))
});
while ((tuple = exec.next()) != null) {
curVal = tuple;
if (preVal != null) {
assertTrue(
"prev: " + preVal + ", but cur: " + curVal, comparator.compare(preVal, curVal) <= 0);
}
preVal = curVal;
cnt++;
}
long end = System.currentTimeMillis();
assertEquals(numTuple, cnt);
// for rescan test
preVal = null;
exec.rescan();
cnt = 0;
while ((tuple = exec.next()) != null) {
curVal = tuple;
if (preVal != null) {
assertTrue(
"prev: " + preVal + ", but cur: " + curVal, comparator.compare(preVal, curVal) <= 0);
}
preVal = curVal;
cnt++;
}
assertEquals(numTuple, cnt);
exec.close();
System.out.println("Sort Time: " + (end - start) + " msc");
}
