@Test public void testLimitAdjusterFuncShipped() throws Exception { String query = "a = load 'input';" + "b = order a by $0 parallel 2;" + "c = limit b 7;" + "store c into 'output' using " + PigStorageNoDefCtor.class.getName() + "('\t');"; PhysicalPlan pp = Util.buildPp(pigServerMR, query); MROperPlan mrPlan = Util.buildMRPlan(pp, pc); LimitAdjuster la = new LimitAdjuster(mrPlan, pc); la.visit(); la.adjust(); MapReduceOper mrOper = mrPlan.getRoots().get(0); int count = 1; while (mrPlan.getSuccessors(mrOper) != null) { mrOper = mrPlan.getSuccessors(mrOper).get(0); ++count; } assertEquals(4, count); MapReduceOper op = mrPlan.getLeaves().get(0); assertTrue(op.UDFs.contains(new FuncSpec(PigStorageNoDefCtor.class.getName()) + "('\t')")); }
/** * Test to ensure that the order by without parallel followed by a limit, i.e., top k always * produces the correct number of map reduce jobs. In the testcase below since we are running the * unit test locally, we will get reduce parallelism as 1. So we will NOT introduce the extra MR * job to do a final limit */ @Test public void testNumReducersInLimit() throws Exception { String query = "a = load 'input';" + "b = order a by $0;" + "c = limit b 10;" + "store c into 'output';"; PhysicalPlan pp = Util.buildPp(pigServer, query); MROperPlan mrPlan = Util.buildMRPlan(pp, pc); MapReduceOper mrOper = mrPlan.getRoots().get(0); int count = 1; while (mrPlan.getSuccessors(mrOper) != null) { mrOper = mrPlan.getSuccessors(mrOper).get(0); ++count; } assertEquals(3, count); }
/** * Test to ensure that the order by with parallel followed by a limit, i.e., top k always produces * the correct number of map reduce jobs */ @Test public void testNumReducersInLimitWithParallel() throws Exception { planTester.buildPlan("a = load 'input';"); planTester.buildPlan("b = order a by $0 parallel 2;"); planTester.buildPlan("c = limit b 10;"); LogicalPlan lp = planTester.buildPlan("store c into '/tmp';"); PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc); MROperPlan mrPlan = Util.buildMRPlan(pp, pc); MapReduceOper mrOper = mrPlan.getRoots().get(0); int count = 1; while (mrPlan.getSuccessors(mrOper) != null) { mrOper = mrPlan.getSuccessors(mrOper).get(0); ++count; } assertTrue(count == 4); }
@Test public void testMergeJoin() throws Exception { String query = "a = load '/tmp/input1';" + "b = load '/tmp/input2';" + "c = join a by $0, b by $0 using 'merge';" + "store c into '/tmp/output1';"; PhysicalPlan pp = Util.buildPp(pigServer, query); MRCompiler comp = new MRCompiler(pp, pc); comp.compile(); MROperPlan mrp = comp.getMRPlan(); assertTrue(mrp.size() == 2); MapReduceOper mrOp0 = mrp.getRoots().get(0); assertTrue(mrOp0.mapPlan.size() == 2); PhysicalOperator load0 = mrOp0.mapPlan.getRoots().get(0); MergeJoinIndexer func = (MergeJoinIndexer) PigContext.instantiateFuncFromSpec(((POLoad) load0).getLFile().getFuncSpec()); Field lrField = MergeJoinIndexer.class.getDeclaredField("lr"); lrField.setAccessible(true); POLocalRearrange lr = (POLocalRearrange) lrField.get(func); List<PhysicalPlan> innerPlans = lr.getPlans(); PhysicalOperator localrearrange0 = mrOp0.mapPlan.getSuccessors(load0).get(0); assertTrue(localrearrange0 instanceof POLocalRearrange); assertTrue(mrOp0.reducePlan.size() == 3); PhysicalOperator pack0 = mrOp0.reducePlan.getRoots().get(0); assertTrue(pack0 instanceof POPackage); PhysicalOperator foreach0 = mrOp0.reducePlan.getSuccessors(pack0).get(0); assertTrue(foreach0 instanceof POForEach); PhysicalOperator store0 = mrOp0.reducePlan.getSuccessors(foreach0).get(0); assertTrue(store0 instanceof POStore); assertTrue(innerPlans.size() == 1); PhysicalPlan innerPlan = innerPlans.get(0); assertTrue(innerPlan.size() == 1); PhysicalOperator project = innerPlan.getRoots().get(0); assertTrue(project instanceof POProject); assertTrue(((POProject) project).getColumn() == 0); MapReduceOper mrOp1 = mrp.getSuccessors(mrOp0).get(0); assertTrue(mrOp1.mapPlan.size() == 3); PhysicalOperator load1 = mrOp1.mapPlan.getRoots().get(0); assertTrue(load1 instanceof POLoad); PhysicalOperator mergejoin1 = mrOp1.mapPlan.getSuccessors(load1).get(0); assertTrue(mergejoin1 instanceof POMergeJoin); PhysicalOperator store1 = mrOp1.mapPlan.getSuccessors(mergejoin1).get(0); assertTrue(store1 instanceof POStore); assertTrue(mrOp1.reducePlan.isEmpty()); }
@Test public void testUDFInMergedCoGroup() throws Exception { String query = "a = load 'input1' using " + TestCollectableLoadFunc.class.getName() + "();" + "b = load 'input2' using " + TestIndexableLoadFunc.class.getName() + "();" + "c = cogroup a by $0, b by $0 using 'merge';" + "store c into 'output';"; PhysicalPlan pp = Util.buildPp(pigServer, query); MROperPlan mrPlan = Util.buildMRPlan(pp, pc); MapReduceOper mrOper = mrPlan.getRoots().get(0); assertTrue(mrOper.UDFs.contains(TestCollectableLoadFunc.class.getName())); mrOper = mrPlan.getSuccessors(mrOper).get(0); assertTrue(mrOper.UDFs.contains(TestCollectableLoadFunc.class.getName())); assertTrue(mrOper.UDFs.contains(TestIndexableLoadFunc.class.getName())); }