/** * @param rearrange * @return */ private static POPreCombinerLocalRearrange getPreCombinerLR(POLocalRearrange rearrange) { String scope = rearrange.getOperatorKey().scope; POPreCombinerLocalRearrange pclr = new POPreCombinerLocalRearrange( createOperatorKey(scope), rearrange.getRequestedParallelism(), rearrange.getInputs()); pclr.setPlans(rearrange.getPlans()); return pclr; }
@Test public void testMergeJoin() throws Exception { String query = "a = load '/tmp/input1';" + "b = load '/tmp/input2';" + "c = join a by $0, b by $0 using 'merge';" + "store c into '/tmp/output1';"; PhysicalPlan pp = Util.buildPp(pigServer, query); MRCompiler comp = new MRCompiler(pp, pc); comp.compile(); MROperPlan mrp = comp.getMRPlan(); assertTrue(mrp.size() == 2); MapReduceOper mrOp0 = mrp.getRoots().get(0); assertTrue(mrOp0.mapPlan.size() == 2); PhysicalOperator load0 = mrOp0.mapPlan.getRoots().get(0); MergeJoinIndexer func = (MergeJoinIndexer) PigContext.instantiateFuncFromSpec(((POLoad) load0).getLFile().getFuncSpec()); Field lrField = MergeJoinIndexer.class.getDeclaredField("lr"); lrField.setAccessible(true); POLocalRearrange lr = (POLocalRearrange) lrField.get(func); List<PhysicalPlan> innerPlans = lr.getPlans(); PhysicalOperator localrearrange0 = mrOp0.mapPlan.getSuccessors(load0).get(0); assertTrue(localrearrange0 instanceof POLocalRearrange); assertTrue(mrOp0.reducePlan.size() == 3); PhysicalOperator pack0 = mrOp0.reducePlan.getRoots().get(0); assertTrue(pack0 instanceof POPackage); PhysicalOperator foreach0 = mrOp0.reducePlan.getSuccessors(pack0).get(0); assertTrue(foreach0 instanceof POForEach); PhysicalOperator store0 = mrOp0.reducePlan.getSuccessors(foreach0).get(0); assertTrue(store0 instanceof POStore); assertTrue(innerPlans.size() == 1); PhysicalPlan innerPlan = innerPlans.get(0); assertTrue(innerPlan.size() == 1); PhysicalOperator project = innerPlan.getRoots().get(0); assertTrue(project instanceof POProject); assertTrue(((POProject) project).getColumn() == 0); MapReduceOper mrOp1 = mrp.getSuccessors(mrOp0).get(0); assertTrue(mrOp1.mapPlan.size() == 3); PhysicalOperator load1 = mrOp1.mapPlan.getRoots().get(0); assertTrue(load1 instanceof POLoad); PhysicalOperator mergejoin1 = mrOp1.mapPlan.getSuccessors(load1).get(0); assertTrue(mergejoin1 instanceof POMergeJoin); PhysicalOperator store1 = mrOp1.mapPlan.getSuccessors(mergejoin1).get(0); assertTrue(store1 instanceof POStore); assertTrue(mrOp1.reducePlan.isEmpty()); }
/** * create new Local rearrange by cloning existing rearrange and add plan for projecting the key * * @param rearrange * @return * @throws PlanException * @throws CloneNotSupportedException */ private static POLocalRearrange getNewRearrange(POLocalRearrange rearrange) throws PlanException, CloneNotSupportedException { POLocalRearrange newRearrange = rearrange.clone(); // Set the projection to be the key PhysicalPlan newPlan = new PhysicalPlan(); String scope = newRearrange.getOperatorKey().scope; POProject proj = new POProject( new OperatorKey(scope, NodeIdGenerator.getGenerator().getNextNodeId(scope)), -1, 0); proj.setResultType(newRearrange.getKeyType()); newPlan.add(proj); List<PhysicalPlan> plans = new ArrayList<PhysicalPlan>(1); plans.add(newPlan); newRearrange.setPlansFromCombiner(plans); return newRearrange; }
@Override public void visitLocalRearrange(POLocalRearrange lrearrange) throws VisitorException { POLocalRearrangeTez lr = (POLocalRearrangeTez) lrearrange; if (!(lr.isConnectedToPackage() && lr.getOutputKey().equals(pkgTezOp.getOperatorKey().toString()))) { return; } loRearrangeFound++; Map<Integer, Pair<Boolean, Map<Integer, Integer>>> keyInfo; if (pkg.getPkgr() instanceof LitePackager) { if (lrearrange.getIndex() != 0) { // Throw some exception here throw new RuntimeException( "POLocalRearrange for POPackageLite cannot have index other than 0, but has index - " + lrearrange.getIndex()); } } // annotate the package with information from the LORearrange // update the keyInfo information if already present in the POPackage keyInfo = pkg.getPkgr().getKeyInfo(); if (keyInfo == null) keyInfo = new HashMap<Integer, Pair<Boolean, Map<Integer, Integer>>>(); if (keyInfo.get(Integer.valueOf(lrearrange.getIndex())) != null) { // something is wrong - we should not be getting key info // for the same index from two different Local Rearranges int errCode = 2087; String msg = "Unexpected problem during optimization." + " Found index:" + lrearrange.getIndex() + " in multiple LocalRearrange operators."; throw new OptimizerException(msg, errCode, PigException.BUG); } keyInfo.put( Integer.valueOf(lrearrange.getIndex()), new Pair<Boolean, Map<Integer, Integer>>( lrearrange.isProjectStar(), lrearrange.getProjectedColsMap())); pkg.getPkgr().setKeyInfo(keyInfo); pkg.getPkgr().setKeyTuple(lrearrange.isKeyTuple()); pkg.getPkgr().setKeyCompound(lrearrange.isKeyCompound()); }
@Test public void testSpl3() throws Exception { PhysicalPlan php = new PhysicalPlan(); POLoad lA = GenPhyOp.topLoadOp(); POSplit spl = GenPhyOp.topSplitOp(); php.add(lA); php.add(spl); php.connect(lA, spl); POFilter fl1 = GenPhyOp.topFilterOp(); fl1.setRequestedParallelism(10); POFilter fl2 = GenPhyOp.topFilterOp(); fl2.setRequestedParallelism(20); php.add(fl1); php.add(fl2); php.connect(spl, fl1); php.connect(spl, fl2); POSplit sp11 = GenPhyOp.topSplitOp(); POSplit sp21 = GenPhyOp.topSplitOp(); php.add(sp11); php.add(sp21); php.connect(fl1, sp11); php.connect(fl2, sp21); POFilter fl11 = GenPhyOp.topFilterOp(); fl11.setRequestedParallelism(10); POFilter fl21 = GenPhyOp.topFilterOp(); fl21.setRequestedParallelism(20); POFilter fl22 = GenPhyOp.topFilterOp(); fl22.setRequestedParallelism(30); php.add(fl11); php.add(fl21); php.add(fl22); php.connect(sp11, fl11); php.connect(sp21, fl21); php.connect(sp21, fl22); POLocalRearrange lr1 = GenPhyOp.topLocalRearrangeOp(); lr1.setRequestedParallelism(40); POLocalRearrange lr21 = GenPhyOp.topLocalRearrangeOp(); lr21.setRequestedParallelism(15); POLocalRearrange lr22 = GenPhyOp.topLocalRearrangeOp(); lr22.setRequestedParallelism(35); php.add(lr1); php.add(lr21); php.add(lr22); php.connect(fl11, lr1); php.connect(fl21, lr21); php.connect(fl22, lr22); POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp(); php.addAsLeaf(gr); POPackage pk = GenPhyOp.topPackageOp(); pk.setRequestedParallelism(25); php.addAsLeaf(pk); POSplit sp2 = GenPhyOp.topSplitOp(); php.addAsLeaf(sp2); POFilter fl3 = GenPhyOp.topFilterOp(); fl3.setRequestedParallelism(100); POFilter fl4 = GenPhyOp.topFilterOp(); fl4.setRequestedParallelism(80); php.add(fl3); php.add(fl4); php.connect(sp2, fl3); php.connect(sp2, fl4); POUnion un = GenPhyOp.topUnionOp(); php.addAsLeaf(un); POStore st = GenPhyOp.topStoreOp(); php.addAsLeaf(st); run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC14.gld"); }
@Test public void testRun1() throws Exception { PhysicalPlan php = new PhysicalPlan(); PhysicalPlan part1 = new PhysicalPlan(); POLoad lC = GenPhyOp.topLoadOp(); POFilter fC = GenPhyOp.topFilterOp(); fC.setRequestedParallelism(20); POLocalRearrange lrC = GenPhyOp.topLocalRearrangeOp(); lrC.setRequestedParallelism(10); POGlobalRearrange grC = GenPhyOp.topGlobalRearrangeOp(); POPackage pkC = GenPhyOp.topPackageOp(); part1.add(lC); part1.add(fC); part1.connect(lC, fC); part1.add(lrC); part1.connect(fC, lrC); part1.add(grC); part1.connect(lrC, grC); part1.add(pkC); part1.connect(grC, pkC); POPackage pkD = GenPhyOp.topPackageOp(); pkD.setRequestedParallelism(20); POLocalRearrange lrD = GenPhyOp.topLocalRearrangeOp(); lrD.setRequestedParallelism(30); POGlobalRearrange grD = GenPhyOp.topGlobalRearrangeOp(); POLoad lD = GenPhyOp.topLoadOp(); part1.add(lD); part1.add(lrD); part1.connect(lD, lrD); part1.add(grD); part1.connect(lrD, grD); part1.add(pkD); part1.connect(grD, pkD); POPackage pkCD = GenPhyOp.topPackageOp(); POLocalRearrange lrCD1 = GenPhyOp.topLocalRearrangeOp(); POLocalRearrange lrCD2 = GenPhyOp.topLocalRearrangeOp(); POGlobalRearrange grCD = GenPhyOp.topGlobalRearrangeOp(); part1.add(lrCD1); part1.add(lrCD2); part1.connect(pkC, lrCD1); part1.connect(pkD, lrCD2); part1.add(grCD); part1.connect(lrCD1, grCD); part1.connect(lrCD2, grCD); part1.add(pkCD); part1.connect(grCD, pkCD); POLoad lA = GenPhyOp.topLoadOp(); POLoad lB = GenPhyOp.topLoadOp(); // POLoad lC = lA; POFilter fA = GenPhyOp.topFilterOp(); POLocalRearrange lrA = GenPhyOp.topLocalRearrangeOp(); POLocalRearrange lrB = GenPhyOp.topLocalRearrangeOp(); POGlobalRearrange grAB = GenPhyOp.topGlobalRearrangeOp(); POPackage pkAB = GenPhyOp.topPackageOp(); POFilter fAB = GenPhyOp.topFilterOp(); POUnion unABC = GenPhyOp.topUnionOp(); php.add(lA); php.add(lB); php.add(fA); php.connect(lA, fA); php.add(lrA); php.add(lrB); php.connect(fA, lrA); php.connect(lB, lrB); php.add(grAB); php.connect(lrA, grAB); php.connect(lrB, grAB); php.add(pkAB); php.connect(grAB, pkAB); php.add(fAB); php.connect(pkAB, fAB); php.merge(part1); List<PhysicalOperator> leaves = new ArrayList<PhysicalOperator>(); for (PhysicalOperator phyOp : php.getLeaves()) { leaves.add(phyOp); } php.add(unABC); for (PhysicalOperator physicalOperator : leaves) { php.connect(physicalOperator, unABC); } POStore st = GenPhyOp.topStoreOp(); php.add(st); php.connect(unABC, st); run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC10.gld"); }
/** * Algebraic functions and distinct in nested plan of a foreach are partially computed in the map * and combine phase. A new foreach statement with initial and intermediate forms of algebraic * functions are added to map and combine plans respectively. * * <p>If bag portion of group-by result is projected or a non algebraic expression/udf has bag as * input, combiner will not be used. This is because the use of combiner in such case is likely to * degrade performance as there will not be much reduction in data size in combine stage to offset * the cost of the additional number of times (de)serialization is done. * * <p>Major areas for enhancement: 1. use of combiner in cogroup 2. queries with order-by, limit * or sort in a nested foreach after group-by 3. case where group-by is followed by filter that * has algebraic expression */ public static void addCombiner( PhysicalPlan mapPlan, PhysicalPlan reducePlan, PhysicalPlan combinePlan, CompilationMessageCollector messageCollector, boolean doMapAgg) throws VisitorException { // part one - check if this MR job represents a group-by + foreach. Find // the POLocalRearrange in the map. I'll need it later. List<PhysicalOperator> mapLeaves = mapPlan.getLeaves(); if (mapLeaves == null || mapLeaves.size() != 1) { messageCollector.collect( "Expected map to have single leaf", MessageType.Warning, PigWarning.MULTI_LEAF_MAP); return; } PhysicalOperator mapLeaf = mapLeaves.get(0); if (!(mapLeaf instanceof POLocalRearrange)) { return; } POLocalRearrange rearrange = (POLocalRearrange) mapLeaf; List<PhysicalOperator> reduceRoots = reducePlan.getRoots(); if (reduceRoots.size() != 1) { messageCollector.collect( "Expected reduce to have single root", MessageType.Warning, PigWarning.MULTI_ROOT_REDUCE); return; } // I expect that the first root should always be a POPackage. If not, I // don't know what's going on, so I'm out of here. PhysicalOperator root = reduceRoots.get(0); if (!(root instanceof POPackage)) { messageCollector.collect( "Expected reduce root to be a POPackage", MessageType.Warning, PigWarning.NON_PACKAGE_REDUCE_PLAN_ROOT); return; } POPackage pack = (POPackage) root; List<PhysicalOperator> packSuccessors = reducePlan.getSuccessors(root); if (packSuccessors == null || packSuccessors.size() != 1) { return; } PhysicalOperator successor = packSuccessors.get(0); if (successor instanceof POLimit) { // POLimit is acceptable, as long has it has a single foreach as // successor List<PhysicalOperator> limitSucs = reducePlan.getSuccessors(successor); if (limitSucs != null && limitSucs.size() == 1 && limitSucs.get(0) instanceof POForEach) { // the code below will now further examine the foreach successor = limitSucs.get(0); } } if (successor instanceof POForEach) { POForEach foreach = (POForEach) successor; List<PhysicalPlan> feInners = foreach.getInputPlans(); // find algebraic operators and also check if the foreach statement // is suitable for combiner use List<Pair<PhysicalOperator, PhysicalPlan>> algebraicOps = findAlgebraicOps(feInners); if (algebraicOps == null || algebraicOps.size() == 0) { // the plan is not combinable or there is nothing to combine // we're done return; } if (combinePlan != null && combinePlan.getRoots().size() != 0) { messageCollector.collect( "Wasn't expecting to find anything already " + "in the combiner!", MessageType.Warning, PigWarning.NON_EMPTY_COMBINE_PLAN); return; } LOG.info("Choosing to move algebraic foreach to combiner"); try { // replace PODistinct->Project[*] with distinct udf (which is Algebraic) for (Pair<PhysicalOperator, PhysicalPlan> op2plan : algebraicOps) { if (!(op2plan.first instanceof PODistinct)) { continue; } DistinctPatcher distinctPatcher = new DistinctPatcher(op2plan.second); distinctPatcher.visit(); if (distinctPatcher.getDistinct() == null) { int errCode = 2073; String msg = "Problem with replacing distinct operator with distinct built-in function."; throw new PlanException(msg, errCode, PigException.BUG); } op2plan.first = distinctPatcher.getDistinct(); } // create new map foreach POForEach mfe = createForEachWithGrpProj(foreach, rearrange.getKeyType()); Map<PhysicalOperator, Integer> op2newpos = Maps.newHashMap(); Integer pos = 1; // create plan for each algebraic udf and add as inner plan in map-foreach for (Pair<PhysicalOperator, PhysicalPlan> op2plan : algebraicOps) { PhysicalPlan udfPlan = createPlanWithPredecessors(op2plan.first, op2plan.second); mfe.addInputPlan(udfPlan, false); op2newpos.put(op2plan.first, pos++); } changeFunc(mfe, POUserFunc.INITIAL); // since we will only be creating SingleTupleBag as input to // the map foreach, we should flag the POProjects in the map // foreach inner plans to also use SingleTupleBag for (PhysicalPlan mpl : mfe.getInputPlans()) { try { new fixMapProjects(mpl).visit(); } catch (VisitorException e) { int errCode = 2089; String msg = "Unable to flag project operator to use single tuple bag."; throw new PlanException(msg, errCode, PigException.BUG, e); } } // create new combine foreach POForEach cfe = createForEachWithGrpProj(foreach, rearrange.getKeyType()); // add algebraic functions with appropriate projection addAlgebraicFuncToCombineFE(cfe, op2newpos); changeFunc(cfe, POUserFunc.INTERMEDIATE); // fix projection and function time for algebraic functions in reduce foreach for (Pair<PhysicalOperator, PhysicalPlan> op2plan : algebraicOps) { setProjectInput(op2plan.first, op2plan.second, op2newpos.get(op2plan.first)); byte resultType = op2plan.first.getResultType(); ((POUserFunc) op2plan.first).setAlgebraicFunction(POUserFunc.FINAL); op2plan.first.setResultType(resultType); } // we have modified the foreach inner plans - so set them again // for the foreach so that foreach can do any re-initialization // around them. // FIXME - this is a necessary evil right now because the leaves // are explicitly stored in the POForeach as a list rather than // computed each time at run time from the plans for // optimization. Do we want to have the Foreach compute the // leaves each time and have Java optimize it (will Java // optimize?)? mfe.setInputPlans(mfe.getInputPlans()); cfe.setInputPlans(cfe.getInputPlans()); foreach.setInputPlans(foreach.getInputPlans()); // tell POCombinerPackage which fields need projected and which // placed in bags. First field is simple project rest need to go // into bags int numFields = algebraicOps.size() + 1; // algebraic funcs + group key boolean[] bags = new boolean[numFields]; bags[0] = false; for (int i = 1; i < numFields; i++) { bags[i] = true; } // Use the POCombiner package in the combine plan // as it needs to act differently than the regular // package operator. CombinerPackager pkgr = new CombinerPackager(pack.getPkgr(), bags); POPackage combinePack = pack.clone(); combinePack.setPkgr(pkgr); combinePack.setParentPlan(null); combinePlan.add(combinePack); combinePlan.add(cfe); combinePlan.connect(combinePack, cfe); // No need to connect projections in cfe to cp, because // PigCombiner directly attaches output from package to // root of remaining plan. POLocalRearrange mlr = getNewRearrange(rearrange); POPartialAgg mapAgg = null; if (doMapAgg) { mapAgg = createPartialAgg(cfe); } // A specialized local rearrange operator will replace // the normal local rearrange in the map plan. This behaves // like the regular local rearrange in the getNext() // as far as getting its input and constructing the // "key" out of the input. It then returns a tuple with // two fields - the key in the first position and the // "value" inside a bag in the second position. This output // format resembles the format out of a Package. This output // will feed to the map foreach which expects this format. // If the key field isn't in the project of the combiner or map foreach, // it is added to the end (This is required so that we can // set up the inner plan of the new Local Rearrange leaf in the map // and combine plan to contain just the project of the key). patchUpMap(mapPlan, getPreCombinerLR(rearrange), mfe, mapAgg, mlr); POLocalRearrange clr = getNewRearrange(rearrange); clr.setParentPlan(null); combinePlan.add(clr); combinePlan.connect(cfe, clr); // Change the package operator in the reduce plan to // be the POCombiner package, as it needs to act // differently than the regular package operator. pack.setPkgr(pkgr.clone()); } catch (Exception e) { int errCode = 2018; String msg = "Internal error. Unable to introduce the combiner for optimization."; throw new OptimizerException(msg, errCode, PigException.BUG, e); } } }
@Override public void visitLocalRearrange(POLocalRearrange lr) throws VisitorException { this.mro.mapKeyType = lr.getKeyType(); foundKeyType = true; }