@Test
public void testStaticStr1() throws JAXBException, IOException {
JAXBElement<?> jaxb =
(JAXBElement<?>)
_ds.unmarshal(
JarTestResourceUtils.readDemoFileAsStream("test/static_test_str1.xml", _ds_cfg));
DataSetDescr dsd = (DataSetDescr) jaxb.getValue();
SortGroup sg = dsd.getSortByGrp();
assertNotNull(sg);
assertEquals(1, sg.getSortBy().size());
}
/**
* Order sort groups by increasing totalOutputTupleSize. This minimizes the total volume of data
* that needs to be sorted.
*/
public void orderSortGroups() {
Collections.sort(
sortGroups,
new Comparator<SortGroup>() {
public int compare(SortGroup sg1, SortGroup sg2) {
Preconditions.checkState(sg1.totalOutputTupleSize > 0);
Preconditions.checkState(sg2.totalOutputTupleSize > 0);
int diff = sg1.totalOutputTupleSize - sg2.totalOutputTupleSize;
return (diff < 0 ? -1 : (diff > 0 ? 1 : 0));
}
});
for (SortGroup sortGroup : sortGroups) {
sortGroup.orderWindowGroups();
}
}
/** Partitions the windowGroups into SortGroups based on compatible order by exprs. */
private List<SortGroup> collectSortGroups(List<WindowGroup> windowGroups) {
List<SortGroup> sortGroups = Lists.newArrayList();
for (WindowGroup windowGroup : windowGroups) {
boolean match = false;
for (SortGroup sortGroup : sortGroups) {
if (sortGroup.isCompatible(windowGroup)) {
sortGroup.add(windowGroup);
match = true;
break;
}
}
if (!match) sortGroups.add(new SortGroup(windowGroup));
}
return sortGroups;
}
/**
* Coalesce sort groups that have compatible partition-by exprs and have a prefix relationship.
*/
private void mergeSortGroups(List<SortGroup> sortGroups) {
boolean hasMerged = false;
do {
hasMerged = false;
for (SortGroup sg1 : sortGroups) {
for (SortGroup sg2 : sortGroups) {
if (sg1 != sg2 && sg1.isPrefixOf(sg2)) {
sg1.absorb(sg2);
sortGroups.remove(sg2);
hasMerged = true;
break;
}
}
if (hasMerged) break;
}
} while (hasMerged);
}
/**
* Return plan tree that augments 'root' with plan nodes that implement single-node evaluation of
* the AnalyticExprs in analyticInfo. This plan takes into account a possible hash partition of
* its input on 'groupingExprs'; if this is non-null, it returns in 'inputPartitionExprs' a subset
* of the grouping exprs which should be used for the aggregate hash partitioning during the
* parallelization of 'root'. TODO: when generating sort orders for the sort groups, optimize the
* ordering of the partition exprs (so that subsequent sort operations see the input sorted on a
* prefix of their required sort exprs) TODO: when merging sort groups, recognize equivalent exprs
* (using the equivalence classes) rather than looking for expr equality
*/
public PlanNode createSingleNodePlan(
PlanNode root, List<Expr> groupingExprs, List<Expr> inputPartitionExprs)
throws ImpalaException {
List<WindowGroup> windowGroups = collectWindowGroups();
for (int i = 0; i < windowGroups.size(); ++i) {
windowGroups.get(i).init(analyzer_, "wg-" + i);
}
List<SortGroup> sortGroups = collectSortGroups(windowGroups);
mergeSortGroups(sortGroups);
for (SortGroup g : sortGroups) {
g.init();
}
List<PartitionGroup> partitionGroups = collectPartitionGroups(sortGroups);
mergePartitionGroups(partitionGroups, root.getNumNodes());
orderGroups(partitionGroups);
if (groupingExprs != null) {
Preconditions.checkNotNull(inputPartitionExprs);
computeInputPartitionExprs(
partitionGroups, groupingExprs, root.getNumNodes(), inputPartitionExprs);
}
for (PartitionGroup partitionGroup : partitionGroups) {
for (int i = 0; i < partitionGroup.sortGroups.size(); ++i) {
root =
createSortGroupPlan(
root,
partitionGroup.sortGroups.get(i),
i == 0 ? partitionGroup.partitionByExprs : null);
}
}
// create equiv classes for newly added slots
analyzer_.createIdentityEquivClasses();
return root;
}
public static void checkDataSetDescrFull(DataSetDescr ds) {
assertEquals("Description", "Test DataSet", ds.getDescr());
assertEquals("Enabled", true, ds.isEnabled());
LangMap lmap = ds.getLangMap();
assertNotNull("lang_map tag check", lmap);
List<LangColumn> lc = lmap.getColumn();
assertNotNull("lang_map->column(s) tag check", lc);
assertEquals("Lang Map Size", 2, lc.size());
assertEquals("COL1 Lang Column not found", "COL1", lc.get(0).getName());
assertEquals("COL2 Lang Column not found", "COL2", lc.get(1).getName());
ExColumns ec = ds.getExColumns();
assertNotNull("ex_columns tag check", ec);
AutoIncColumns lai = ec.getAutoInc();
assertNotNull("ex_columns->auto_inc tag check", lai);
List<AutoIncColumn> ai = lai.getColumn();
assertNotNull("ex_columns->auto_inc->column(s) tag check", ai);
assertEquals("Auto Inc Column Size", 2, ai.size());
assertEquals("Auto Inc Column #1 Name", "A11", ai.get(0).getName());
assertEquals("Auto Inc Column #2 Name", "B22", ai.get(1).getName());
CalcColumns lcc = ec.getCalc();
assertNotNull("ex_columns->calc tag check", lcc);
List<CalcColumn> cc = lcc.getColumn();
assertNotNull("ex_columns->calc->column(s) tag check", lc);
assertEquals("Calculated Column Size", 2, cc.size());
assertEquals("Calculated Column #1 Name", "CALC1", cc.get(0).getName());
assertEquals("Calculated Column #1 Formula", "A + B", cc.get(0).getValue());
assertEquals("Calculated Column #2 Name", "CALC2", cc.get(1).getName());
assertEquals("Calculated Column #2 Formula", "C + D", cc.get(1).getValue());
SortGroup fs = ds.getSortByGrp();
ConditionFilter cf = ds.getFilter();
assertEquals("Condition", "A < B", cf.getValue());
List<SortCond> sff = fs.getSortBy();
checkFilter(sff.get(0), 1, "COL1", SortTypes.ASC, false);
checkFilter(sff.get(1), 2, "COL2", SortTypes.DESC, false);
assertNull("Non-existing static_ds", ds.getStaticData());
// Check recursion
GroupData dsg = ds.getGroupData();
assertNotNull("group_ds tag check", dsg);
List<DataSetExt> ldsd1 = dsg.getDsList().getGroupDsOrStaticDsOrCsvDs();
assertEquals("DsGroup #1 size", 2, ldsd1.size());
// Check nested group
DataSetExt dse = ldsd1.get(0);
assertEquals(dse.getClass(), GroupDataSetDescr.class);
List<DataSetExt> ldsd2 =
((GroupDataSetDescr) dse).getGroupData().getDsList().getGroupDsOrStaticDsOrCsvDs();
assertEquals("DsGroup #2 size", 4, ldsd2.size());
// Check just class of each element
assertEquals(ldsd2.get(0).getClass(), GroupDataSetDescr.class);
assertEquals(ldsd2.get(1).getClass(), StaticDataSetDescr.class);
assertEquals(ldsd2.get(2).getClass(), CsvDataSetDescr.class);
assertEquals(ldsd2.get(3).getClass(), SqlDataSetDescr.class);
dse = ldsd1.get(1);
assertEquals(dse.getClass(), StaticDataSetDescr.class);
StaticDataSetDescr dsd = (StaticDataSetDescr) dse;
StaticDataSetDescr sds = dsd;
assertNotNull("group_ds->static_ds tag check", sds);
StaticData sdf = sds.getStaticData();
assertNotNull("static_ds->static_set tag check", sdf);
Columns cs = sdf.getColumns();
assertNotNull("static_ds->columns tag check", cs);
List<ColumnHeader> ch = cs.getColumn();
assertNotNull("columns->column tag check", ch);
assertEquals(2, ch.size());
checkColumnHeader(ch.get(0), "COL1", "java.lang.String", "ERROR GRP 2 !!!");
checkColumnHeader(ch.get(1), "COL2", "java.lang.String", "ERROR GRP 2 !!!");
StaticRecords sr = sdf.getStaticRows();
assertNotNull("static_ds->static tag check", dsg);
List<RowDef> lrd = sr.getRow();
assertNotNull("static tag->row(s) check", dsg);
assertEquals("Static Rows size", 2, lrd.size());
RowDef srd = lrd.get(0);
List<RowCell> lcd = srd.getCell();
assertNotNull("row#1>column(s) check", lcd);
assertEquals("Static Columns size", 2, lcd.size());
checkRowColumn(lcd.get(0), 1, 1, "COL1", "bBb");
checkRowColumn(lcd.get(1), 1, 2, "COL2", "УуУ");
srd = lrd.get(1);
lcd = srd.getCell();
assertNotNull("row#2>column(s) check", lcd);
checkRowColumn(lcd.get(0), 2, 1, "COL1", "AaA");
checkRowColumn(lcd.get(1), 2, 2, "COL2", "пПп");
}
