@Test public void testSim4() throws Exception { PhysicalPlan php = new PhysicalPlan(); PhysicalPlan ldGrpChain1 = GenPhyOp.loadedGrpChain(); PhysicalPlan ldGrpChain2 = GenPhyOp.loadedGrpChain(); php.merge(ldGrpChain1); php.merge(ldGrpChain2); POUnion un = GenPhyOp.topUnionOp(); php.addAsLeaf(un); PhysicalPlan ldFil1 = GenPhyOp.loadedFilter(); PhysicalPlan ldFil2 = GenPhyOp.loadedFilter(); php.merge(ldFil1); php.connect(ldFil1.getLeaves().get(0), un); php.merge(ldFil2); php.connect(ldFil2.getLeaves().get(0), un); POStore st = GenPhyOp.topStoreOp(); php.add(st); php.connect(un, st); run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC4.gld"); }
@Test public void testSim8() throws Exception { PhysicalPlan php = new PhysicalPlan(); PhysicalPlan ldGrpChain1 = GenPhyOp.loadedGrpChain(); PhysicalPlan ldGrpChain2 = GenPhyOp.loadedGrpChain(); POLocalRearrange lr1 = GenPhyOp.topLocalRearrangeOp(); POLocalRearrange lr2 = GenPhyOp.topLocalRearrangeOp(); ldGrpChain1.addAsLeaf(lr1); ldGrpChain2.addAsLeaf(lr2); php.merge(ldGrpChain1); php.merge(ldGrpChain2); POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp(); php.addAsLeaf(gr); PhysicalPlan ldFil1 = GenPhyOp.loadedFilter(); PhysicalPlan ldFil2 = GenPhyOp.loadedFilter(); php.merge(ldFil1); php.connect(ldFil1.getLeaves().get(0), gr); php.merge(ldFil2); php.connect(ldFil2.getLeaves().get(0), gr); POPackage pk = GenPhyOp.topPackageOp(); php.addAsLeaf(pk); POStore st = GenPhyOp.topStoreOp(); php.addAsLeaf(st); run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC8.gld"); }
// Tests Single input case for both blocking and non-blocking // with both map and reduce phases @Test public void testSim1() throws Exception { PhysicalPlan php = new PhysicalPlan(); POLoad ld = GenPhyOp.topLoadOp(); php.add(ld); PhysicalPlan grpChain1 = GenPhyOp.grpChain(); php.merge(grpChain1); php.connect(ld, grpChain1.getRoots().get(0)); PhysicalOperator leaf = php.getLeaves().get(0); PhysicalPlan grpChain2 = GenPhyOp.grpChain(); php.merge(grpChain2); php.connect(leaf, grpChain2.getRoots().get(0)); leaf = php.getLeaves().get(0); POFilter fl = GenPhyOp.topFilterOp(); php.add(fl); php.connect(leaf, fl); POStore st = GenPhyOp.topStoreOp(); php.add(st); php.connect(fl, st); run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC1.gld"); }
public void testDistinct1() throws Exception { PhysicalPlan php = new PhysicalPlan(); PhysicalPlan ldFil1 = GenPhyOp.loadedFilter(); php.merge(ldFil1); PODistinct op = new PODistinct(new OperatorKey("", r.nextLong()), -1, null); php.addAsLeaf(op); PhysicalPlan grpChain1 = GenPhyOp.grpChain(); php.merge(grpChain1); php.connect(op, grpChain1.getRoots().get(0)); PODistinct op1 = new PODistinct(new OperatorKey("", r.nextLong()), -1, null); php.addAsLeaf(op1); POStore st = GenPhyOp.topStoreOp(); php.addAsLeaf(st); run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC16.gld"); }
@Test(expected = MRCompilerException.class) public void testMRCompilerErr1() throws Exception { PhysicalPlan pp = new PhysicalPlan(); PhysicalPlan ldFil1 = GenPhyOp.loadedFilter(); pp.merge(ldFil1); POSplit op = GenPhyOp.topSplitOp(); pp.addAsLeaf(op); try { Util.buildMRPlan(pp, pc); } catch (MRCompilerException mrce) { assertEquals(2025, mrce.getErrorCode()); throw mrce; } }
@Test public void testMRCompilerErr1() throws Exception { PhysicalPlan pp = new PhysicalPlan(); PhysicalPlan ldFil1 = GenPhyOp.loadedFilter(); pp.merge(ldFil1); POSplit op = GenPhyOp.topSplitOp(); pp.addAsLeaf(op); try { Util.buildMRPlan(pp, pc); fail("Expected failure."); } catch (MRCompilerException mrce) { assertTrue(mrce.getErrorCode() == 2025); } }
/** * Test that POSortedDistinct gets printed as POSortedDistinct * * @throws Exception */ @Test public void testSortedDistinctInForeach() throws Exception { PhysicalPlan php = new PhysicalPlan(); PhysicalPlan grpChain1 = GenPhyOp.loadedGrpChain(); php.merge(grpChain1); List<PhysicalPlan> inputs = new LinkedList<PhysicalPlan>(); PhysicalPlan inplan = new PhysicalPlan(); PODistinct op1 = new POSortedDistinct(new OperatorKey("", r.nextLong()), -1, null); inplan.addAsLeaf(op1); inputs.add(inplan); List<Boolean> toFlattens = new ArrayList<Boolean>(); toFlattens.add(false); POForEach pofe = new POForEach(new OperatorKey("", r.nextLong()), 1, inputs, toFlattens); php.addAsLeaf(pofe); POStore st = GenPhyOp.topStoreOp(); php.addAsLeaf(st); run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC19.gld"); }
@Test public void testSortUDF1() throws Exception { PhysicalPlan php = new PhysicalPlan(); PhysicalPlan ldFil1 = GenPhyOp.loadedFilter(); php.merge(ldFil1); // set up order by * String funcName = WeirdComparator.class.getName(); POUserComparisonFunc comparator = new POUserComparisonFunc( new OperatorKey("", r.nextLong()), -1, null, new FuncSpec(funcName)); POSort sort = new POSort( new OperatorKey("", r.nextLong()), -1, ldFil1.getLeaves(), null, new ArrayList<Boolean>(), comparator); sort.setRequestedParallelism(20); PhysicalPlan nesSortPlan = new PhysicalPlan(); POProject topPrj = new POProject(new OperatorKey("", r.nextLong())); topPrj.setColumn(1); topPrj.setOverloaded(true); topPrj.setResultType(DataType.TUPLE); nesSortPlan.add(topPrj); POProject prjStar2 = new POProject(new OperatorKey("", r.nextLong())); prjStar2.setResultType(DataType.TUPLE); prjStar2.setStar(true); nesSortPlan.add(prjStar2); nesSortPlan.connect(topPrj, prjStar2); List<PhysicalPlan> nesSortPlanLst = new ArrayList<PhysicalPlan>(); nesSortPlanLst.add(nesSortPlan); sort.setSortPlans(nesSortPlanLst); php.add(sort); php.connect(ldFil1.getLeaves().get(0), sort); // have a foreach which takes the sort output // and send it two two udfs List<String> udfs = new ArrayList<String>(); udfs.add(COUNT.class.getName()); udfs.add(SUM.class.getName()); POForEach fe3 = GenPhyOp.topForEachOPWithUDF(udfs); php.add(fe3); php.connect(sort, fe3); // add a group above the foreach PhysicalPlan grpChain1 = GenPhyOp.grpChain(); php.merge(grpChain1); php.connect(fe3, grpChain1.getRoots().get(0)); udfs.clear(); udfs.add(AVG.class.getName()); POForEach fe4 = GenPhyOp.topForEachOPWithUDF(udfs); php.addAsLeaf(fe4); PhysicalPlan grpChain2 = GenPhyOp.grpChain(); php.merge(grpChain2); php.connect(fe4, grpChain2.getRoots().get(0)); udfs.clear(); udfs.add(GFCross.class.getName() + "('1')"); POForEach fe5 = GenPhyOp.topForEachOPWithUDF(udfs); php.addAsLeaf(fe5); POStore st = GenPhyOp.topStoreOp(); php.addAsLeaf(st); run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC15.gld"); }
@Test public void testRun2() throws Exception { PhysicalPlan php = new PhysicalPlan(); PhysicalPlan part1 = new PhysicalPlan(); POLoad lC = GenPhyOp.topLoadOp(); POFilter fC = GenPhyOp.topFilterOp(); POLocalRearrange lrC = GenPhyOp.topLocalRearrangeOp(); POGlobalRearrange grC = GenPhyOp.topGlobalRearrangeOp(); POPackage pkC = GenPhyOp.topPackageOp(); part1.add(lC); part1.add(fC); part1.connect(lC, fC); part1.add(lrC); part1.connect(fC, lrC); part1.add(grC); part1.connect(lrC, grC); part1.add(pkC); part1.connect(grC, pkC); POPackage pkD = GenPhyOp.topPackageOp(); POLocalRearrange lrD = GenPhyOp.topLocalRearrangeOp(); POGlobalRearrange grD = GenPhyOp.topGlobalRearrangeOp(); POLoad lD = GenPhyOp.topLoadOp(); part1.add(lD); part1.add(lrD); part1.connect(lD, lrD); part1.add(grD); part1.connect(lrD, grD); part1.add(pkD); part1.connect(grD, pkD); part1.connect(pkD, grC); POLoad lA = GenPhyOp.topLoadOp(); POLoad lB = GenPhyOp.topLoadOp(); // POLoad lC = lA; POFilter fA = GenPhyOp.topFilterOp(); POLocalRearrange lrA = GenPhyOp.topLocalRearrangeOp(); POLocalRearrange lrB = GenPhyOp.topLocalRearrangeOp(); POGlobalRearrange grAB = GenPhyOp.topGlobalRearrangeOp(); POPackage pkAB = GenPhyOp.topPackageOp(); POFilter fAB = GenPhyOp.topFilterOp(); POUnion unABC = GenPhyOp.topUnionOp(); php.add(lA); php.add(lB); php.add(fA); php.connect(lA, fA); php.add(lrA); php.add(lrB); php.connect(fA, lrA); php.connect(lB, lrB); php.add(grAB); php.connect(lrA, grAB); php.connect(lrB, grAB); php.add(pkAB); php.connect(grAB, pkAB); php.add(fAB); php.connect(pkAB, fAB); php.merge(part1); List<PhysicalOperator> leaves = new ArrayList<PhysicalOperator>(); for (PhysicalOperator phyOp : php.getLeaves()) { leaves.add(phyOp); } php.add(unABC); for (PhysicalOperator physicalOperator : leaves) { php.connect(physicalOperator, unABC); } POStore st = GenPhyOp.topStoreOp(); php.add(st); php.connect(unABC, st); run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC11.gld"); }