/**
* Obtains all the uids from the schema
*
* @param schema
* @return Set of uids from this schema. Its a recursive call
*/
private Set<Long> getAllUids(LogicalSchema schema) {
Set<Long> uids = new HashSet<Long>();
if (schema != null) {
for (LogicalFieldSchema fieldSchema : schema.getFields()) {
if ((fieldSchema.type == DataType.BAG || fieldSchema.type == DataType.TUPLE)
&& fieldSchema.schema != null) {
uids.addAll(getAllUids(fieldSchema.schema));
} else {
uids.add(fieldSchema.uid);
}
}
}
return uids;
}
public void testCoGroup4() throws Exception {
LogicalPlanTester lpt = new LogicalPlanTester();
lpt.buildPlan("a = load '/test/d.txt' as (name:chararray, age:int, gpa:float);");
lpt.buildPlan("b = load '/test/e.txt' as (name:chararray, age:int, blah:chararray );");
lpt.buildPlan("c = group a by ( name, age ), b by ( name, age );");
LogicalPlan plan = lpt.buildPlan("store c into '/test/empty';");
// check basics
org.apache.pig.experimental.logical.relational.LogicalPlan newPlan = migratePlan(plan);
assertEquals(
LOCogroup.class, newPlan.getSuccessors(newPlan.getSources().get(0)).get(0).getClass());
LOCogroup cogroup = (LOCogroup) newPlan.getSuccessors(newPlan.getSources().get(0)).get(0);
// Reason for this strange way of getting the load schema is to maintain the sequence correctly
LogicalSchema loadSchema =
((LogicalRelationalOperator) newPlan.getPredecessors(cogroup).get(0)).getSchema();
LogicalSchema load2Schema =
((LogicalRelationalOperator) newPlan.getPredecessors(cogroup).get(1)).getSchema();
Set<Long> uids = getAllUids(loadSchema);
uids.addAll(getAllUids(load2Schema));
LogicalRelationalOperator op =
(LogicalRelationalOperator) newPlan.getSuccessors(newPlan.getSources().get(0)).get(0);
assertEquals(LOCogroup.class, op.getClass());
LogicalSchema schema = op.getSchema();
assertEquals(3, schema.size());
assertEquals(DataType.TUPLE, schema.getField(0).type);
assertEquals(false, uids.contains(schema.getField(0).uid));
assertEquals(0, schema.getField(0).alias.compareTo("group"));
assertEquals(DataType.CHARARRAY, schema.getField(0).schema.getField(0).type);
assertEquals(0, schema.getField(0).schema.getField(0).alias.compareTo("name"));
assertEquals(DataType.INTEGER, schema.getField(0).schema.getField(1).type);
assertEquals(0, schema.getField(0).schema.getField(1).alias.compareTo("age"));
assertEquals(DataType.BAG, schema.getField(1).type);
assertEquals(DataType.CHARARRAY, schema.getField(1).schema.getField(0).type);
assertEquals(0, schema.getField(1).schema.getField(0).alias.compareTo("name"));
assertEquals(loadSchema.getField(0).uid, schema.getField(1).schema.getField(0).uid);
assertEquals(DataType.INTEGER, schema.getField(1).schema.getField(1).type);
assertEquals(0, schema.getField(1).schema.getField(1).alias.compareTo("age"));
assertEquals(loadSchema.getField(1).uid, schema.getField(1).schema.getField(1).uid);
assertEquals(DataType.FLOAT, schema.getField(1).schema.getField(2).type);
assertEquals(0, schema.getField(1).schema.getField(2).alias.compareTo("gpa"));
assertEquals(loadSchema.getField(2).uid, schema.getField(1).schema.getField(2).uid);
assertEquals(DataType.BAG, schema.getField(2).type);
assertEquals(DataType.CHARARRAY, schema.getField(2).schema.getField(0).type);
assertEquals(0, schema.getField(2).schema.getField(0).alias.compareTo("name"));
assertEquals(load2Schema.getField(0).uid, schema.getField(2).schema.getField(0).uid);
assertEquals(DataType.INTEGER, schema.getField(2).schema.getField(1).type);
assertEquals(0, schema.getField(2).schema.getField(1).alias.compareTo("age"));
assertEquals(load2Schema.getField(1).uid, schema.getField(2).schema.getField(1).uid);
assertEquals(DataType.CHARARRAY, schema.getField(2).schema.getField(2).type);
assertEquals(0, schema.getField(2).schema.getField(2).alias.compareTo("blah"));
assertEquals(load2Schema.getField(2).uid, schema.getField(2).schema.getField(2).uid);
// We are doing Uid tests at the end as the uids should not repeat
assertEquals(false, uids.contains(schema.getField(0).schema.getField(0).uid));
assertEquals(false, uids.contains(schema.getField(0).schema.getField(1).uid));
assertEquals(false, uids.contains(schema.getField(1).uid));
uids.add(schema.getField(1).uid);
assertEquals(false, uids.contains(schema.getField(2).uid));
MultiMap<Integer, LogicalExpressionPlan> expressionPlans = cogroup.getExpressionPlans();
assertEquals(2, expressionPlans.size());
List<LogicalExpressionPlan> plans =
(List<LogicalExpressionPlan>) expressionPlans.get(Integer.valueOf(0));
assertEquals(2, plans.size());
List<LogicalExpressionPlan> plans2 =
(List<LogicalExpressionPlan>) expressionPlans.get(Integer.valueOf(1));
assertEquals(2, plans2.size());
LogicalExpressionPlan exprPlan = plans.get(0);
assertEquals(1, exprPlan.getSinks().size());
assertEquals(ProjectExpression.class, exprPlan.getSinks().get(0).getClass());
ProjectExpression prj = (ProjectExpression) exprPlan.getSinks().get(0);
assertEquals(loadSchema.getField(0).uid, prj.getUid());
assertEquals(0, prj.getColNum());
assertEquals(0, prj.getInputNum());
LogicalExpressionPlan exprPlan2 = plans.get(1);
assertEquals(1, exprPlan2.getSinks().size());
assertEquals(ProjectExpression.class, exprPlan2.getSinks().get(0).getClass());
ProjectExpression prj2 = (ProjectExpression) exprPlan2.getSinks().get(0);
assertEquals(loadSchema.getField(1).uid, prj2.getUid());
assertEquals(1, prj2.getColNum());
assertEquals(0, prj2.getInputNum());
LogicalExpressionPlan exprPlan3 = plans2.get(0);
assertEquals(1, exprPlan3.getSinks().size());
assertEquals(ProjectExpression.class, exprPlan3.getSinks().get(0).getClass());
ProjectExpression prj3 = (ProjectExpression) exprPlan3.getSinks().get(0);
assertEquals(load2Schema.getField(0).uid, prj3.getUid());
assertEquals(0, prj3.getColNum());
assertEquals(1, prj3.getInputNum());
LogicalExpressionPlan exprPlan4 = plans2.get(1);
assertEquals(1, exprPlan4.getSinks().size());
assertEquals(ProjectExpression.class, exprPlan4.getSinks().get(0).getClass());
ProjectExpression prj4 = (ProjectExpression) exprPlan4.getSinks().get(0);
assertEquals(load2Schema.getField(1).uid, prj4.getUid());
assertEquals(1, prj4.getColNum());
assertEquals(1, prj4.getInputNum());
}
public void testForeachPlan2() throws Exception {
LogicalPlanTester lpt = new LogicalPlanTester();
lpt.buildPlan("a = load '/test/d.txt' as (id, d:bag{t:(id:int, s)});");
lpt.buildPlan("b = foreach a generate id, FLATTEN(d);");
LogicalPlan plan = lpt.buildPlan("store b into '/test/empty';");
// check basics
org.apache.pig.experimental.logical.relational.LogicalPlan newPlan = migratePlan(plan);
org.apache.pig.experimental.logical.relational.LogicalPlan expected =
new org.apache.pig.experimental.logical.relational.LogicalPlan();
LogicalSchema aschema = new LogicalSchema();
aschema.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.BYTEARRAY));
LogicalSchema aschema2 = new LogicalSchema();
LogicalSchema aschema3 = new LogicalSchema();
aschema3.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.INTEGER));
aschema3.addField(new LogicalSchema.LogicalFieldSchema("s", null, DataType.BYTEARRAY));
aschema2.addField(new LogicalSchema.LogicalFieldSchema("t", aschema3, DataType.TUPLE));
aschema.addField(new LogicalSchema.LogicalFieldSchema("d", aschema2, DataType.BAG));
LOLoad load =
new LOLoad(
new FileSpec("file:///test/d.txt", new FuncSpec("org.apache.pig.builtin.PigStorage")),
aschema,
expected);
expected.add(load);
LOForEach foreach2 = new LOForEach(expected);
org.apache.pig.experimental.logical.relational.LogicalPlan innerPlan =
new org.apache.pig.experimental.logical.relational.LogicalPlan();
LOInnerLoad l1 = new LOInnerLoad(innerPlan, foreach2, 0);
innerPlan.add(l1);
LOInnerLoad l2 = new LOInnerLoad(innerPlan, foreach2, 1);
List<LogicalExpressionPlan> eps = new ArrayList<LogicalExpressionPlan>();
LogicalExpressionPlan p1 = new LogicalExpressionPlan();
new ProjectExpression(p1, DataType.BYTEARRAY, 0, 0);
LogicalExpressionPlan p2 = new LogicalExpressionPlan();
new ProjectExpression(p2, DataType.BAG, 1, 0);
eps.add(p1);
eps.add(p2);
LOGenerate gen = new LOGenerate(innerPlan, eps, new boolean[] {false, true});
innerPlan.add(gen);
innerPlan.connect(l1, gen);
innerPlan.connect(l2, gen);
foreach2.setInnerPlan(innerPlan);
expected.add(foreach2);
LOStore s =
new LOStore(
expected,
new FileSpec("file:///test/empty", new FuncSpec("org.apache.pig.builtin.PigStorage")));
expected.add(s);
expected.connect(load, foreach2);
expected.connect(foreach2, s);
try {
UidStamper stamper = new UidStamper(expected);
stamper.visit();
} catch (Exception e) {
throw new VisitorException(e);
}
assertTrue(expected.isEqual(newPlan));
LogicalSchema schema = foreach2.getSchema();
aschema = new LogicalSchema();
aschema.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.BYTEARRAY));
aschema.addField(new LogicalSchema.LogicalFieldSchema("d::id", null, DataType.INTEGER));
aschema.addField(new LogicalSchema.LogicalFieldSchema("s", null, DataType.BYTEARRAY));
assertTrue(schema.isEqual(aschema));
assertTrue(schema.getField("id") == schema.getField(0));
assertTrue(schema.getField("d::id") == schema.getField(1));
}
public void testCoGroup() throws Exception {
LogicalPlanTester lpt = new LogicalPlanTester();
lpt.buildPlan("a = load '/test/d.txt' as (name:chararray, age:int, gpa:float);");
lpt.buildPlan("b = group a by name;");
LogicalPlan plan = lpt.buildPlan("store b into '/test/empty';");
// check basics
org.apache.pig.experimental.logical.relational.LogicalPlan newPlan = migratePlan(plan);
LogicalSchema loadSchema =
((LogicalRelationalOperator) newPlan.getSources().get(0)).getSchema();
Set<Long> uids = getAllUids(loadSchema);
LogicalRelationalOperator op =
(LogicalRelationalOperator) newPlan.getSuccessors(newPlan.getSources().get(0)).get(0);
assertEquals(LOCogroup.class, op.getClass());
LogicalSchema schema = op.getSchema();
assertEquals(2, schema.size());
assertEquals(DataType.CHARARRAY, schema.getField(0).type);
assertEquals(false, uids.contains(schema.getField(0).uid));
assertEquals(0, schema.getField(0).alias.compareTo("group"));
assertEquals(DataType.BAG, schema.getField(1).type);
assertEquals(DataType.CHARARRAY, schema.getField(1).schema.getField(0).type);
assertEquals(0, schema.getField(1).schema.getField(0).alias.compareTo("name"));
assertEquals(loadSchema.getField(0).uid, schema.getField(1).schema.getField(0).uid);
assertEquals(DataType.INTEGER, schema.getField(1).schema.getField(1).type);
assertEquals(0, schema.getField(1).schema.getField(1).alias.compareTo("age"));
assertEquals(loadSchema.getField(1).uid, schema.getField(1).schema.getField(1).uid);
assertEquals(DataType.FLOAT, schema.getField(1).schema.getField(2).type);
assertEquals(0, schema.getField(1).schema.getField(2).alias.compareTo("gpa"));
assertEquals(loadSchema.getField(2).uid, schema.getField(1).schema.getField(2).uid);
uids.add(Long.valueOf(schema.getField(0).uid));
assertEquals(false, uids.contains(schema.getField(1).uid));
assertEquals(
LOCogroup.class, newPlan.getSuccessors(newPlan.getSources().get(0)).get(0).getClass());
LOCogroup cogroup = (LOCogroup) newPlan.getSuccessors(newPlan.getSources().get(0)).get(0);
MultiMap<Integer, LogicalExpressionPlan> expressionPlans = cogroup.getExpressionPlans();
assertEquals(1, expressionPlans.size());
List<LogicalExpressionPlan> plans =
(List<LogicalExpressionPlan>) expressionPlans.get(Integer.valueOf(0));
assertEquals(1, plans.size());
LogicalExpressionPlan exprPlan = plans.get(0);
assertEquals(1, exprPlan.getSinks().size());
assertEquals(ProjectExpression.class, exprPlan.getSinks().get(0).getClass());
ProjectExpression prj = (ProjectExpression) exprPlan.getSinks().get(0);
assertEquals(loadSchema.getField(0).uid, prj.getUid());
assertEquals(0, prj.getColNum());
assertEquals(0, prj.getInputNum());
}
public void testForeachSchema() throws Exception {
// test flatten
LogicalPlanTester lpt = new LogicalPlanTester();
lpt.buildPlan("a = load '/test/d.txt' as (id, d:tuple(v, s));");
LogicalPlan plan = lpt.buildPlan("b = foreach a generate id, FLATTEN(d);");
org.apache.pig.experimental.logical.relational.LogicalPlan newPlan = migratePlan(plan);
LogicalRelationalOperator op = (LogicalRelationalOperator) newPlan.getSinks().get(0);
LogicalSchema s2 = new LogicalSchema();
s2.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.BYTEARRAY));
s2.addField(new LogicalSchema.LogicalFieldSchema("v", null, DataType.BYTEARRAY));
s2.addField(new LogicalSchema.LogicalFieldSchema("s", null, DataType.BYTEARRAY));
assertTrue(s2.isEqual(op.getSchema()));
// test no flatten
lpt = new LogicalPlanTester();
lpt.buildPlan("a = load '/test/d.txt' as (id, d:bag{t:(v, s)});");
plan = lpt.buildPlan("b = foreach a generate id, d;");
newPlan = migratePlan(plan);
op = (LogicalRelationalOperator) newPlan.getSinks().get(0);
LogicalSchema aschema = new LogicalSchema();
aschema.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.BYTEARRAY));
LogicalSchema aschema2 = new LogicalSchema();
LogicalSchema aschema3 = new LogicalSchema();
aschema3.addField(new LogicalSchema.LogicalFieldSchema("v", null, DataType.BYTEARRAY));
aschema3.addField(new LogicalSchema.LogicalFieldSchema("s", null, DataType.BYTEARRAY));
aschema2.addField(new LogicalSchema.LogicalFieldSchema("t", aschema3, DataType.TUPLE));
aschema.addField(new LogicalSchema.LogicalFieldSchema("d", aschema2, DataType.BAG));
assertTrue(aschema.isEqual(op.getSchema()));
// check with defined data type
lpt = new LogicalPlanTester();
lpt.buildPlan("a = load '/test/d.txt' as (id, d:bag{t:(v:int, s)});");
lpt.buildPlan("b = foreach a generate id, FLATTEN(d);");
plan = lpt.buildPlan("store b into '/test/empty';");
newPlan = migratePlan(plan);
op = (LogicalRelationalOperator) newPlan.getSinks().get(0);
op = (LogicalRelationalOperator) newPlan.getPredecessors(op).get(0);
LogicalSchema schema = op.getSchema();
aschema = new LogicalSchema();
aschema.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.BYTEARRAY));
aschema.addField(new LogicalSchema.LogicalFieldSchema("v", null, DataType.INTEGER));
aschema.addField(new LogicalSchema.LogicalFieldSchema("s", null, DataType.BYTEARRAY));
assertTrue(schema.isEqual(aschema));
// test with add
lpt = new LogicalPlanTester();
lpt.buildPlan("a = load '/test/d.txt' as (id, v:int, s:int);");
lpt.buildPlan("b = foreach a generate id, v+s;");
plan = lpt.buildPlan("store b into '/test/empty';");
newPlan = migratePlan(plan);
op = (LogicalRelationalOperator) newPlan.getSinks().get(0);
op = (LogicalRelationalOperator) newPlan.getPredecessors(op).get(0);
schema = op.getSchema();
aschema = new LogicalSchema();
aschema.addField(new LogicalSchema.LogicalFieldSchema("id", null, DataType.BYTEARRAY));
aschema.addField(new LogicalSchema.LogicalFieldSchema(null, null, DataType.INTEGER));
assertTrue(schema.isEqual(aschema));
}