Exemplo n.º 1
0
  // Tests Single input case for both blocking and non-blocking
  // with both map and reduce phases
  @Test
  public void testSim1() throws Exception {
    PhysicalPlan php = new PhysicalPlan();
    POLoad ld = GenPhyOp.topLoadOp();
    php.add(ld);
    PhysicalPlan grpChain1 = GenPhyOp.grpChain();
    php.merge(grpChain1);

    php.connect(ld, grpChain1.getRoots().get(0));

    PhysicalOperator leaf = php.getLeaves().get(0);

    PhysicalPlan grpChain2 = GenPhyOp.grpChain();
    php.merge(grpChain2);

    php.connect(leaf, grpChain2.getRoots().get(0));

    leaf = php.getLeaves().get(0);
    POFilter fl = GenPhyOp.topFilterOp();
    php.add(fl);

    php.connect(leaf, fl);

    POStore st = GenPhyOp.topStoreOp();
    php.add(st);

    php.connect(fl, st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC1.gld");
  }
Exemplo n.º 2
0
  @Test
  public void testSpl2() throws Exception {
    PhysicalPlan php = new PhysicalPlan();

    POLoad lA = GenPhyOp.topLoadOp();
    POSplit spl = GenPhyOp.topSplitOp();
    php.add(lA);
    php.add(spl);
    php.connect(lA, spl);

    POFilter fl1 = GenPhyOp.topFilterOp();
    POFilter fl2 = GenPhyOp.topFilterOp();
    php.add(fl1);
    php.add(fl2);
    php.connect(spl, fl1);
    php.connect(spl, fl2);

    POLocalRearrange lr1 = GenPhyOp.topLocalRearrangeOp();
    POLocalRearrange lr2 = GenPhyOp.topLocalRearrangeOp();
    php.add(lr1);
    php.add(lr2);
    php.connect(fl1, lr1);
    php.connect(fl2, lr2);

    POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
    php.addAsLeaf(gr);

    POPackage pk = GenPhyOp.topPackageOp();
    php.addAsLeaf(pk);

    POSplit sp2 = GenPhyOp.topSplitOp();
    php.addAsLeaf(sp2);

    POFilter fl3 = GenPhyOp.topFilterOp();
    POFilter fl4 = GenPhyOp.topFilterOp();
    php.add(fl3);
    php.add(fl4);
    php.connect(sp2, fl3);
    php.connect(sp2, fl4);

    POUnion un = GenPhyOp.topUnionOp();
    php.addAsLeaf(un);

    POStore st = GenPhyOp.topStoreOp();
    php.addAsLeaf(st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC13.gld");
  }
Exemplo n.º 3
0
  public void testSpl1() throws Exception {
    PhysicalPlan php = new PhysicalPlan();

    POLoad lA = GenPhyOp.topLoadOp();
    POSplit spl = GenPhyOp.topSplitOp();
    php.add(lA);
    php.add(spl);
    php.connect(lA, spl);

    POFilter fl1 = GenPhyOp.topFilterOp();
    POFilter fl2 = GenPhyOp.topFilterOp();
    php.add(fl1);
    php.add(fl2);
    php.connect(spl, fl1);
    php.connect(spl, fl2);

    POLocalRearrange lr1 = GenPhyOp.topLocalRearrangeOp();
    POLocalRearrange lr2 = GenPhyOp.topLocalRearrangeOp();
    php.add(lr1);
    php.add(lr2);
    php.connect(fl1, lr1);
    php.connect(fl2, lr2);

    POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
    php.add(gr);
    php.connect(lr1, gr);
    php.connect(lr2, gr);

    POPackage pk = GenPhyOp.topPackageOp();
    php.add(pk);
    php.connect(gr, pk);

    POStore st = GenPhyOp.topStoreOp();
    php.add(st);
    php.connect(pk, st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC12.gld");
  }
Exemplo n.º 4
0
  @Test
  public void testSpl3() throws Exception {
    PhysicalPlan php = new PhysicalPlan();

    POLoad lA = GenPhyOp.topLoadOp();
    POSplit spl = GenPhyOp.topSplitOp();
    php.add(lA);
    php.add(spl);
    php.connect(lA, spl);

    POFilter fl1 = GenPhyOp.topFilterOp();
    fl1.setRequestedParallelism(10);
    POFilter fl2 = GenPhyOp.topFilterOp();
    fl2.setRequestedParallelism(20);
    php.add(fl1);
    php.add(fl2);
    php.connect(spl, fl1);
    php.connect(spl, fl2);

    POSplit sp11 = GenPhyOp.topSplitOp();
    POSplit sp21 = GenPhyOp.topSplitOp();
    php.add(sp11);
    php.add(sp21);
    php.connect(fl1, sp11);
    php.connect(fl2, sp21);

    POFilter fl11 = GenPhyOp.topFilterOp();
    fl11.setRequestedParallelism(10);
    POFilter fl21 = GenPhyOp.topFilterOp();
    fl21.setRequestedParallelism(20);
    POFilter fl22 = GenPhyOp.topFilterOp();
    fl22.setRequestedParallelism(30);
    php.add(fl11);
    php.add(fl21);
    php.add(fl22);
    php.connect(sp11, fl11);
    php.connect(sp21, fl21);
    php.connect(sp21, fl22);

    POLocalRearrange lr1 = GenPhyOp.topLocalRearrangeOp();
    lr1.setRequestedParallelism(40);
    POLocalRearrange lr21 = GenPhyOp.topLocalRearrangeOp();
    lr21.setRequestedParallelism(15);
    POLocalRearrange lr22 = GenPhyOp.topLocalRearrangeOp();
    lr22.setRequestedParallelism(35);
    php.add(lr1);
    php.add(lr21);
    php.add(lr22);
    php.connect(fl11, lr1);
    php.connect(fl21, lr21);
    php.connect(fl22, lr22);

    POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
    php.addAsLeaf(gr);

    POPackage pk = GenPhyOp.topPackageOp();
    pk.setRequestedParallelism(25);
    php.addAsLeaf(pk);

    POSplit sp2 = GenPhyOp.topSplitOp();
    php.addAsLeaf(sp2);

    POFilter fl3 = GenPhyOp.topFilterOp();
    fl3.setRequestedParallelism(100);
    POFilter fl4 = GenPhyOp.topFilterOp();
    fl4.setRequestedParallelism(80);
    php.add(fl3);
    php.add(fl4);
    php.connect(sp2, fl3);
    php.connect(sp2, fl4);

    POUnion un = GenPhyOp.topUnionOp();
    php.addAsLeaf(un);

    POStore st = GenPhyOp.topStoreOp();
    php.addAsLeaf(st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC14.gld");
  }
Exemplo n.º 5
0
  @Test
  public void testRun2() throws Exception {
    PhysicalPlan php = new PhysicalPlan();

    PhysicalPlan part1 = new PhysicalPlan();
    POLoad lC = GenPhyOp.topLoadOp();
    POFilter fC = GenPhyOp.topFilterOp();
    POLocalRearrange lrC = GenPhyOp.topLocalRearrangeOp();
    POGlobalRearrange grC = GenPhyOp.topGlobalRearrangeOp();
    POPackage pkC = GenPhyOp.topPackageOp();
    part1.add(lC);
    part1.add(fC);
    part1.connect(lC, fC);
    part1.add(lrC);
    part1.connect(fC, lrC);
    part1.add(grC);
    part1.connect(lrC, grC);
    part1.add(pkC);
    part1.connect(grC, pkC);

    POPackage pkD = GenPhyOp.topPackageOp();
    POLocalRearrange lrD = GenPhyOp.topLocalRearrangeOp();
    POGlobalRearrange grD = GenPhyOp.topGlobalRearrangeOp();
    POLoad lD = GenPhyOp.topLoadOp();
    part1.add(lD);
    part1.add(lrD);
    part1.connect(lD, lrD);

    part1.add(grD);
    part1.connect(lrD, grD);
    part1.add(pkD);
    part1.connect(grD, pkD);
    part1.connect(pkD, grC);

    POLoad lA = GenPhyOp.topLoadOp();
    POLoad lB = GenPhyOp.topLoadOp();

    // POLoad lC = lA;
    POFilter fA = GenPhyOp.topFilterOp();

    POLocalRearrange lrA = GenPhyOp.topLocalRearrangeOp();
    POLocalRearrange lrB = GenPhyOp.topLocalRearrangeOp();

    POGlobalRearrange grAB = GenPhyOp.topGlobalRearrangeOp();

    POPackage pkAB = GenPhyOp.topPackageOp();

    POFilter fAB = GenPhyOp.topFilterOp();
    POUnion unABC = GenPhyOp.topUnionOp();

    php.add(lA);
    php.add(lB);

    php.add(fA);

    php.connect(lA, fA);

    php.add(lrA);
    php.add(lrB);

    php.connect(fA, lrA);
    php.connect(lB, lrB);

    php.add(grAB);
    php.connect(lrA, grAB);
    php.connect(lrB, grAB);

    php.add(pkAB);
    php.connect(grAB, pkAB);

    php.add(fAB);
    php.connect(pkAB, fAB);

    php.merge(part1);

    List<PhysicalOperator> leaves = new ArrayList<PhysicalOperator>();
    for (PhysicalOperator phyOp : php.getLeaves()) {
      leaves.add(phyOp);
    }

    php.add(unABC);
    for (PhysicalOperator physicalOperator : leaves) {
      php.connect(physicalOperator, unABC);
    }

    POStore st = GenPhyOp.topStoreOp();

    php.add(st);
    php.connect(unABC, st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC11.gld");
  }