예제 #1
0
파일: TestMRCompiler.java 프로젝트: scr/pig
  @Test
  public void testSim4() throws Exception {
    PhysicalPlan php = new PhysicalPlan();

    PhysicalPlan ldGrpChain1 = GenPhyOp.loadedGrpChain();
    PhysicalPlan ldGrpChain2 = GenPhyOp.loadedGrpChain();

    php.merge(ldGrpChain1);
    php.merge(ldGrpChain2);

    POUnion un = GenPhyOp.topUnionOp();
    php.addAsLeaf(un);

    PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
    PhysicalPlan ldFil2 = GenPhyOp.loadedFilter();

    php.merge(ldFil1);
    php.connect(ldFil1.getLeaves().get(0), un);

    php.merge(ldFil2);
    php.connect(ldFil2.getLeaves().get(0), un);

    POStore st = GenPhyOp.topStoreOp();
    php.add(st);

    php.connect(un, st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC4.gld");
  }
예제 #2
0
파일: TestMRCompiler.java 프로젝트: scr/pig
  @Test
  public void testSim8() throws Exception {
    PhysicalPlan php = new PhysicalPlan();

    PhysicalPlan ldGrpChain1 = GenPhyOp.loadedGrpChain();
    PhysicalPlan ldGrpChain2 = GenPhyOp.loadedGrpChain();

    POLocalRearrange lr1 = GenPhyOp.topLocalRearrangeOp();
    POLocalRearrange lr2 = GenPhyOp.topLocalRearrangeOp();

    ldGrpChain1.addAsLeaf(lr1);
    ldGrpChain2.addAsLeaf(lr2);

    php.merge(ldGrpChain1);
    php.merge(ldGrpChain2);

    POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
    php.addAsLeaf(gr);

    PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
    PhysicalPlan ldFil2 = GenPhyOp.loadedFilter();

    php.merge(ldFil1);
    php.connect(ldFil1.getLeaves().get(0), gr);

    php.merge(ldFil2);
    php.connect(ldFil2.getLeaves().get(0), gr);

    POPackage pk = GenPhyOp.topPackageOp();
    php.addAsLeaf(pk);

    POStore st = GenPhyOp.topStoreOp();
    php.addAsLeaf(st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC8.gld");
  }
예제 #3
0
파일: TestMRCompiler.java 프로젝트: scr/pig
  // Tests Single input case for both blocking and non-blocking
  // with both map and reduce phases
  @Test
  public void testSim1() throws Exception {
    PhysicalPlan php = new PhysicalPlan();
    POLoad ld = GenPhyOp.topLoadOp();
    php.add(ld);
    PhysicalPlan grpChain1 = GenPhyOp.grpChain();
    php.merge(grpChain1);

    php.connect(ld, grpChain1.getRoots().get(0));

    PhysicalOperator leaf = php.getLeaves().get(0);

    PhysicalPlan grpChain2 = GenPhyOp.grpChain();
    php.merge(grpChain2);

    php.connect(leaf, grpChain2.getRoots().get(0));

    leaf = php.getLeaves().get(0);
    POFilter fl = GenPhyOp.topFilterOp();
    php.add(fl);

    php.connect(leaf, fl);

    POStore st = GenPhyOp.topStoreOp();
    php.add(st);

    php.connect(fl, st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC1.gld");
  }
예제 #4
0
  public void testDistinct1() throws Exception {
    PhysicalPlan php = new PhysicalPlan();
    PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
    php.merge(ldFil1);

    PODistinct op = new PODistinct(new OperatorKey("", r.nextLong()), -1, null);

    php.addAsLeaf(op);

    PhysicalPlan grpChain1 = GenPhyOp.grpChain();
    php.merge(grpChain1);
    php.connect(op, grpChain1.getRoots().get(0));

    PODistinct op1 = new PODistinct(new OperatorKey("", r.nextLong()), -1, null);

    php.addAsLeaf(op1);
    POStore st = GenPhyOp.topStoreOp();
    php.addAsLeaf(st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC16.gld");
  }
예제 #5
0
파일: TestMRCompiler.java 프로젝트: scr/pig
  @Test(expected = MRCompilerException.class)
  public void testMRCompilerErr1() throws Exception {
    PhysicalPlan pp = new PhysicalPlan();
    PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
    pp.merge(ldFil1);

    POSplit op = GenPhyOp.topSplitOp();
    pp.addAsLeaf(op);

    try {
      Util.buildMRPlan(pp, pc);
    } catch (MRCompilerException mrce) {
      assertEquals(2025, mrce.getErrorCode());
      throw mrce;
    }
  }
예제 #6
0
  @Test
  public void testMRCompilerErr1() throws Exception {
    PhysicalPlan pp = new PhysicalPlan();
    PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
    pp.merge(ldFil1);

    POSplit op = GenPhyOp.topSplitOp();
    pp.addAsLeaf(op);

    try {
      Util.buildMRPlan(pp, pc);
      fail("Expected failure.");
    } catch (MRCompilerException mrce) {
      assertTrue(mrce.getErrorCode() == 2025);
    }
  }
예제 #7
0
파일: TestMRCompiler.java 프로젝트: scr/pig
  /**
   * Test that POSortedDistinct gets printed as POSortedDistinct
   *
   * @throws Exception
   */
  @Test
  public void testSortedDistinctInForeach() throws Exception {
    PhysicalPlan php = new PhysicalPlan();
    PhysicalPlan grpChain1 = GenPhyOp.loadedGrpChain();
    php.merge(grpChain1);

    List<PhysicalPlan> inputs = new LinkedList<PhysicalPlan>();
    PhysicalPlan inplan = new PhysicalPlan();
    PODistinct op1 = new POSortedDistinct(new OperatorKey("", r.nextLong()), -1, null);
    inplan.addAsLeaf(op1);
    inputs.add(inplan);
    List<Boolean> toFlattens = new ArrayList<Boolean>();
    toFlattens.add(false);
    POForEach pofe = new POForEach(new OperatorKey("", r.nextLong()), 1, inputs, toFlattens);

    php.addAsLeaf(pofe);
    POStore st = GenPhyOp.topStoreOp();
    php.addAsLeaf(st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC19.gld");
  }
예제 #8
0
파일: TestMRCompiler.java 프로젝트: scr/pig
  @Test
  public void testSortUDF1() throws Exception {
    PhysicalPlan php = new PhysicalPlan();
    PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
    php.merge(ldFil1);

    // set up order by *
    String funcName = WeirdComparator.class.getName();
    POUserComparisonFunc comparator =
        new POUserComparisonFunc(
            new OperatorKey("", r.nextLong()), -1, null, new FuncSpec(funcName));
    POSort sort =
        new POSort(
            new OperatorKey("", r.nextLong()),
            -1,
            ldFil1.getLeaves(),
            null,
            new ArrayList<Boolean>(),
            comparator);
    sort.setRequestedParallelism(20);
    PhysicalPlan nesSortPlan = new PhysicalPlan();
    POProject topPrj = new POProject(new OperatorKey("", r.nextLong()));
    topPrj.setColumn(1);
    topPrj.setOverloaded(true);
    topPrj.setResultType(DataType.TUPLE);
    nesSortPlan.add(topPrj);

    POProject prjStar2 = new POProject(new OperatorKey("", r.nextLong()));
    prjStar2.setResultType(DataType.TUPLE);
    prjStar2.setStar(true);
    nesSortPlan.add(prjStar2);

    nesSortPlan.connect(topPrj, prjStar2);
    List<PhysicalPlan> nesSortPlanLst = new ArrayList<PhysicalPlan>();
    nesSortPlanLst.add(nesSortPlan);

    sort.setSortPlans(nesSortPlanLst);

    php.add(sort);
    php.connect(ldFil1.getLeaves().get(0), sort);
    // have a foreach which takes the sort output
    // and send it two two udfs
    List<String> udfs = new ArrayList<String>();
    udfs.add(COUNT.class.getName());
    udfs.add(SUM.class.getName());
    POForEach fe3 = GenPhyOp.topForEachOPWithUDF(udfs);
    php.add(fe3);
    php.connect(sort, fe3);

    // add a group above the foreach
    PhysicalPlan grpChain1 = GenPhyOp.grpChain();
    php.merge(grpChain1);
    php.connect(fe3, grpChain1.getRoots().get(0));

    udfs.clear();
    udfs.add(AVG.class.getName());
    POForEach fe4 = GenPhyOp.topForEachOPWithUDF(udfs);
    php.addAsLeaf(fe4);

    PhysicalPlan grpChain2 = GenPhyOp.grpChain();
    php.merge(grpChain2);
    php.connect(fe4, grpChain2.getRoots().get(0));

    udfs.clear();
    udfs.add(GFCross.class.getName() + "('1')");
    POForEach fe5 = GenPhyOp.topForEachOPWithUDF(udfs);
    php.addAsLeaf(fe5);

    POStore st = GenPhyOp.topStoreOp();
    php.addAsLeaf(st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC15.gld");
  }
예제 #9
0
파일: TestMRCompiler.java 프로젝트: scr/pig
  @Test
  public void testRun2() throws Exception {
    PhysicalPlan php = new PhysicalPlan();

    PhysicalPlan part1 = new PhysicalPlan();
    POLoad lC = GenPhyOp.topLoadOp();
    POFilter fC = GenPhyOp.topFilterOp();
    POLocalRearrange lrC = GenPhyOp.topLocalRearrangeOp();
    POGlobalRearrange grC = GenPhyOp.topGlobalRearrangeOp();
    POPackage pkC = GenPhyOp.topPackageOp();
    part1.add(lC);
    part1.add(fC);
    part1.connect(lC, fC);
    part1.add(lrC);
    part1.connect(fC, lrC);
    part1.add(grC);
    part1.connect(lrC, grC);
    part1.add(pkC);
    part1.connect(grC, pkC);

    POPackage pkD = GenPhyOp.topPackageOp();
    POLocalRearrange lrD = GenPhyOp.topLocalRearrangeOp();
    POGlobalRearrange grD = GenPhyOp.topGlobalRearrangeOp();
    POLoad lD = GenPhyOp.topLoadOp();
    part1.add(lD);
    part1.add(lrD);
    part1.connect(lD, lrD);

    part1.add(grD);
    part1.connect(lrD, grD);
    part1.add(pkD);
    part1.connect(grD, pkD);
    part1.connect(pkD, grC);

    POLoad lA = GenPhyOp.topLoadOp();
    POLoad lB = GenPhyOp.topLoadOp();

    // POLoad lC = lA;
    POFilter fA = GenPhyOp.topFilterOp();

    POLocalRearrange lrA = GenPhyOp.topLocalRearrangeOp();
    POLocalRearrange lrB = GenPhyOp.topLocalRearrangeOp();

    POGlobalRearrange grAB = GenPhyOp.topGlobalRearrangeOp();

    POPackage pkAB = GenPhyOp.topPackageOp();

    POFilter fAB = GenPhyOp.topFilterOp();
    POUnion unABC = GenPhyOp.topUnionOp();

    php.add(lA);
    php.add(lB);

    php.add(fA);

    php.connect(lA, fA);

    php.add(lrA);
    php.add(lrB);

    php.connect(fA, lrA);
    php.connect(lB, lrB);

    php.add(grAB);
    php.connect(lrA, grAB);
    php.connect(lrB, grAB);

    php.add(pkAB);
    php.connect(grAB, pkAB);

    php.add(fAB);
    php.connect(pkAB, fAB);

    php.merge(part1);

    List<PhysicalOperator> leaves = new ArrayList<PhysicalOperator>();
    for (PhysicalOperator phyOp : php.getLeaves()) {
      leaves.add(phyOp);
    }

    php.add(unABC);
    for (PhysicalOperator physicalOperator : leaves) {
      php.connect(physicalOperator, unABC);
    }

    POStore st = GenPhyOp.topStoreOp();

    php.add(st);
    php.connect(unABC, st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC11.gld");
  }