Exemple #1
0
  // Tests Single input case for both blocking and non-blocking
  // with both map and reduce phases
  @Test
  public void testSim1() throws Exception {
    PhysicalPlan php = new PhysicalPlan();
    POLoad ld = GenPhyOp.topLoadOp();
    php.add(ld);
    PhysicalPlan grpChain1 = GenPhyOp.grpChain();
    php.merge(grpChain1);

    php.connect(ld, grpChain1.getRoots().get(0));

    PhysicalOperator leaf = php.getLeaves().get(0);

    PhysicalPlan grpChain2 = GenPhyOp.grpChain();
    php.merge(grpChain2);

    php.connect(leaf, grpChain2.getRoots().get(0));

    leaf = php.getLeaves().get(0);
    POFilter fl = GenPhyOp.topFilterOp();
    php.add(fl);

    php.connect(leaf, fl);

    POStore st = GenPhyOp.topStoreOp();
    php.add(st);

    php.connect(fl, st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC1.gld");
  }
Exemple #2
0
 @Before
 public void setUp() throws ExecException {
   GenPhyOp.setR(r);
   GenPhyOp.setPc(pc);
   // Set random seed to generate deterministic temporary paths
   FileLocalizer.setR(new Random(1331L));
   NodeIdGenerator.reset("");
   pigServer = new PigServer(pc);
   pigServerMR = new PigServer(pcMR);
 }
Exemple #3
0
  @Test
  public void testSim8() throws Exception {
    PhysicalPlan php = new PhysicalPlan();

    PhysicalPlan ldGrpChain1 = GenPhyOp.loadedGrpChain();
    PhysicalPlan ldGrpChain2 = GenPhyOp.loadedGrpChain();

    POLocalRearrange lr1 = GenPhyOp.topLocalRearrangeOp();
    POLocalRearrange lr2 = GenPhyOp.topLocalRearrangeOp();

    ldGrpChain1.addAsLeaf(lr1);
    ldGrpChain2.addAsLeaf(lr2);

    php.merge(ldGrpChain1);
    php.merge(ldGrpChain2);

    POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
    php.addAsLeaf(gr);

    PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
    PhysicalPlan ldFil2 = GenPhyOp.loadedFilter();

    php.merge(ldFil1);
    php.connect(ldFil1.getLeaves().get(0), gr);

    php.merge(ldFil2);
    php.connect(ldFil2.getLeaves().get(0), gr);

    POPackage pk = GenPhyOp.topPackageOp();
    php.addAsLeaf(pk);

    POStore st = GenPhyOp.topStoreOp();
    php.addAsLeaf(st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC8.gld");
  }
 @Test
 public void testIntegerEq() throws Exception {
   ConstantExpression lt = GenPhyOp.exprConst();
   lt.setValue(new Integer(1));
   ConstantExpression rt = GenPhyOp.exprConst();
   rt.setValue(new Integer(1));
   NotEqualToExpr g = GenPhyOp.compNotEqualToExpr();
   g.setLhs(lt);
   g.setRhs(rt);
   g.setOperandType(DataType.INTEGER);
   Result r = g.getNextBoolean();
   assertEquals(POStatus.STATUS_OK, r.returnStatus);
   assertFalse((Boolean) r.result);
 }
 @Test
 public void testBooleanNe() throws Exception {
   ConstantExpression lt = GenPhyOp.exprConst();
   lt.setValue(Boolean.TRUE);
   ConstantExpression rt = GenPhyOp.exprConst();
   rt.setValue(Boolean.FALSE);
   NotEqualToExpr g = GenPhyOp.compNotEqualToExpr();
   g.setLhs(lt);
   g.setRhs(rt);
   g.setOperandType(DataType.BOOLEAN);
   Result r = g.getNextBoolean();
   assertEquals(POStatus.STATUS_OK, r.returnStatus);
   assertTrue((Boolean) r.result);
 }
 @Test
 public void testDataByteArrayNe() throws Exception {
   ConstantExpression lt = GenPhyOp.exprConst();
   lt.setValue(new DataByteArray("a"));
   ConstantExpression rt = GenPhyOp.exprConst();
   rt.setValue(new DataByteArray("b"));
   NotEqualToExpr g = GenPhyOp.compNotEqualToExpr();
   g.setLhs(lt);
   g.setRhs(rt);
   g.setOperandType(DataType.BYTEARRAY);
   Result r = g.getNextBoolean();
   assertEquals(POStatus.STATUS_OK, r.returnStatus);
   assertTrue((Boolean) r.result);
 }
  public void testLimit() throws Exception {
    PhysicalPlan php = new PhysicalPlan();

    POLoad lC = GenPhyOp.topLoadOp();
    php.add(lC);

    POLimit op = new POLimit(new OperatorKey("", r.nextLong()), -1, null);

    php.add(op);
    php.connect(lC, op);

    POStore st = GenPhyOp.topStoreOp();
    php.addAsLeaf(st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC17.gld");
  }
  @Test
  public void testMRCompilerErr1() throws Exception {
    PhysicalPlan pp = new PhysicalPlan();
    PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
    pp.merge(ldFil1);

    POSplit op = GenPhyOp.topSplitOp();
    pp.addAsLeaf(op);

    try {
      Util.buildMRPlan(pp, pc);
      fail("Expected failure.");
    } catch (MRCompilerException mrce) {
      assertTrue(mrce.getErrorCode() == 2025);
    }
  }
Exemple #9
0
  @Test(expected = MRCompilerException.class)
  public void testMRCompilerErr1() throws Exception {
    PhysicalPlan pp = new PhysicalPlan();
    PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
    pp.merge(ldFil1);

    POSplit op = GenPhyOp.topSplitOp();
    pp.addAsLeaf(op);

    try {
      Util.buildMRPlan(pp, pc);
    } catch (MRCompilerException mrce) {
      assertEquals(2025, mrce.getErrorCode());
      throw mrce;
    }
  }
 @Test
 public void testTupleNe() throws ExecException {
   Tuple tuple_1 = TupleFactory.getInstance().newTuple("item_1");
   Tuple tuple_2 = TupleFactory.getInstance().newTuple("item_2");
   ConstantExpression lt = GenPhyOp.exprConst();
   lt.setValue(tuple_1);
   ConstantExpression rt = GenPhyOp.exprConst();
   rt.setValue(tuple_2);
   NotEqualToExpr g = GenPhyOp.compNotEqualToExpr();
   g.setLhs(lt);
   g.setRhs(rt);
   g.setOperandType(DataType.TUPLE);
   Result r = g.getNextBoolean();
   assertEquals(POStatus.STATUS_OK, r.returnStatus);
   assertTrue((Boolean) r.result);
 }
  public void testSim5() throws Exception {
    PhysicalPlan php = new PhysicalPlan();
    PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
    PhysicalPlan ldFil2 = GenPhyOp.loadedFilter();
    php.merge(ldFil1);
    php.merge(ldFil2);

    POUnion un = GenPhyOp.topUnionOp();
    php.addAsLeaf(un);

    POStore st = GenPhyOp.topStoreOp();
    php.add(st);

    php.connect(un, st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC5.gld");
  }
  @Test
  public void testMapEq() throws ExecException {
    Map map_1 = new HashMap();
    map_1.put("key_1", "value_1");
    Map map_2 = new HashMap();
    map_2.put("key_1", "value_1");

    ConstantExpression lt = GenPhyOp.exprConst();
    lt.setValue(map_1);
    ConstantExpression rt = GenPhyOp.exprConst();
    rt.setValue(map_2);
    NotEqualToExpr g = GenPhyOp.compNotEqualToExpr();
    g.setLhs(lt);
    g.setRhs(rt);
    g.setOperandType(DataType.MAP);
    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_OK, r.returnStatus);
    assertFalse((Boolean) r.result);
  }
Exemple #13
0
  /**
   * Test that POSortedDistinct gets printed as POSortedDistinct
   *
   * @throws Exception
   */
  @Test
  public void testSortedDistinctInForeach() throws Exception {
    PhysicalPlan php = new PhysicalPlan();
    PhysicalPlan grpChain1 = GenPhyOp.loadedGrpChain();
    php.merge(grpChain1);

    List<PhysicalPlan> inputs = new LinkedList<PhysicalPlan>();
    PhysicalPlan inplan = new PhysicalPlan();
    PODistinct op1 = new POSortedDistinct(new OperatorKey("", r.nextLong()), -1, null);
    inplan.addAsLeaf(op1);
    inputs.add(inplan);
    List<Boolean> toFlattens = new ArrayList<Boolean>();
    toFlattens.add(false);
    POForEach pofe = new POForEach(new OperatorKey("", r.nextLong()), 1, inputs, toFlattens);

    php.addAsLeaf(pofe);
    POStore st = GenPhyOp.topStoreOp();
    php.addAsLeaf(st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC19.gld");
  }
  public void testDistinct1() throws Exception {
    PhysicalPlan php = new PhysicalPlan();
    PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
    php.merge(ldFil1);

    PODistinct op = new PODistinct(new OperatorKey("", r.nextLong()), -1, null);

    php.addAsLeaf(op);

    PhysicalPlan grpChain1 = GenPhyOp.grpChain();
    php.merge(grpChain1);
    php.connect(op, grpChain1.getRoots().get(0));

    PODistinct op1 = new PODistinct(new OperatorKey("", r.nextLong()), -1, null);

    php.addAsLeaf(op1);
    POStore st = GenPhyOp.topStoreOp();
    php.addAsLeaf(st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC16.gld");
  }
Exemple #15
0
 public static void oneTimeSetUp() throws Exception {
   cluster = MiniGenericCluster.buildCluster();
   pc = new PigContext(cluster.getExecType(), cluster.getProperties());
   try {
     pc.connect();
   } catch (ExecException e) {
     // TODO Auto-generated catch block
     e.printStackTrace();
   }
   GenPhyOp.setPc(pc);
   Util.copyFromLocalToCluster(cluster, "test/org/apache/pig/test/data/passwd", "/passwd");
 }
  public void intTestSim4() throws Exception {
    PhysicalPlan php = new PhysicalPlan();

    PhysicalPlan ldGrpChain1 = GenPhyOp.loadedGrpChain();
    PhysicalPlan ldGrpChain2 = GenPhyOp.loadedGrpChain();

    php.merge(ldGrpChain1);
    php.merge(ldGrpChain2);

    POUnion un = GenPhyOp.topUnionOp();
    php.addAsLeaf(un);

    PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
    PhysicalPlan ldFil2 = GenPhyOp.loadedFilter();

    php.merge(ldFil1);
    php.connect(ldFil1.getLeaves().get(0), un);

    php.merge(ldFil2);
    php.connect(ldFil2.getLeaves().get(0), un);

    POStore st = GenPhyOp.topStoreOp();
    php.add(st);

    php.connect(un, st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC4.gld");
  }
  public <U> void checkNullValues(byte operandType, U value) throws Exception {

    ConstantExpression lt = GenPhyOp.exprConst();
    ConstantExpression rt = GenPhyOp.exprConst();
    NotEqualToExpr g = GenPhyOp.compNotEqualToExpr();

    // test with null in lhs
    g.setOperandType(operandType);
    lt.setValue(null);
    rt.setValue(value);
    g.setLhs(lt);
    g.setRhs(rt);

    Result r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_NULL, r.returnStatus);
    assertNull(r.result);

    // test with null in rhs
    g.setOperandType(operandType);
    lt.setValue(value);
    rt.setValue(null);
    g.setLhs(lt);
    g.setRhs(rt);

    r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_NULL, r.returnStatus);
    assertNull(r.result);

    // test with null in lhs and rhs
    g.setOperandType(operandType);
    lt.setValue(null);
    rt.setValue(null);
    g.setLhs(lt);
    g.setRhs(rt);

    r = g.getNextBoolean();
    assertEquals(POStatus.STATUS_NULL, r.returnStatus);
    assertNull(r.result);
  }
Exemple #18
0
  @Test
  public void testSpl1() throws Exception {
    PhysicalPlan php = new PhysicalPlan();

    POLoad lA = GenPhyOp.topLoadOp();
    POSplit spl = GenPhyOp.topSplitOp();
    php.add(lA);
    php.add(spl);
    php.connect(lA, spl);

    POFilter fl1 = GenPhyOp.topFilterOp();
    POFilter fl2 = GenPhyOp.topFilterOp();
    php.add(fl1);
    php.add(fl2);
    php.connect(spl, fl1);
    php.connect(spl, fl2);

    POLocalRearrange lr1 = GenPhyOp.topLocalRearrangeOp();
    POLocalRearrange lr2 = GenPhyOp.topLocalRearrangeOp();
    php.add(lr1);
    php.add(lr2);
    php.connect(fl1, lr1);
    php.connect(fl2, lr2);

    POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
    php.add(gr);
    php.connect(lr1, gr);
    php.connect(lr2, gr);

    POPackage pk = GenPhyOp.topPackageOp();
    php.add(pk);
    php.connect(gr, pk);

    POStore st = GenPhyOp.topStoreOp();
    php.add(st);
    php.connect(pk, st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC12.gld");
  }
Exemple #19
0
  @Test
  public void testSortUDF1() throws Exception {
    PhysicalPlan php = new PhysicalPlan();
    PhysicalPlan ldFil1 = GenPhyOp.loadedFilter();
    php.merge(ldFil1);

    // set up order by *
    String funcName = WeirdComparator.class.getName();
    POUserComparisonFunc comparator =
        new POUserComparisonFunc(
            new OperatorKey("", r.nextLong()), -1, null, new FuncSpec(funcName));
    POSort sort =
        new POSort(
            new OperatorKey("", r.nextLong()),
            -1,
            ldFil1.getLeaves(),
            null,
            new ArrayList<Boolean>(),
            comparator);
    sort.setRequestedParallelism(20);
    PhysicalPlan nesSortPlan = new PhysicalPlan();
    POProject topPrj = new POProject(new OperatorKey("", r.nextLong()));
    topPrj.setColumn(1);
    topPrj.setOverloaded(true);
    topPrj.setResultType(DataType.TUPLE);
    nesSortPlan.add(topPrj);

    POProject prjStar2 = new POProject(new OperatorKey("", r.nextLong()));
    prjStar2.setResultType(DataType.TUPLE);
    prjStar2.setStar(true);
    nesSortPlan.add(prjStar2);

    nesSortPlan.connect(topPrj, prjStar2);
    List<PhysicalPlan> nesSortPlanLst = new ArrayList<PhysicalPlan>();
    nesSortPlanLst.add(nesSortPlan);

    sort.setSortPlans(nesSortPlanLst);

    php.add(sort);
    php.connect(ldFil1.getLeaves().get(0), sort);
    // have a foreach which takes the sort output
    // and send it two two udfs
    List<String> udfs = new ArrayList<String>();
    udfs.add(COUNT.class.getName());
    udfs.add(SUM.class.getName());
    POForEach fe3 = GenPhyOp.topForEachOPWithUDF(udfs);
    php.add(fe3);
    php.connect(sort, fe3);

    // add a group above the foreach
    PhysicalPlan grpChain1 = GenPhyOp.grpChain();
    php.merge(grpChain1);
    php.connect(fe3, grpChain1.getRoots().get(0));

    udfs.clear();
    udfs.add(AVG.class.getName());
    POForEach fe4 = GenPhyOp.topForEachOPWithUDF(udfs);
    php.addAsLeaf(fe4);

    PhysicalPlan grpChain2 = GenPhyOp.grpChain();
    php.merge(grpChain2);
    php.connect(fe4, grpChain2.getRoots().get(0));

    udfs.clear();
    udfs.add(GFCross.class.getName() + "('1')");
    POForEach fe5 = GenPhyOp.topForEachOPWithUDF(udfs);
    php.addAsLeaf(fe5);

    POStore st = GenPhyOp.topStoreOp();
    php.addAsLeaf(st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC15.gld");
  }
Exemple #20
0
  @Test
  public void testSpl3() throws Exception {
    PhysicalPlan php = new PhysicalPlan();

    POLoad lA = GenPhyOp.topLoadOp();
    POSplit spl = GenPhyOp.topSplitOp();
    php.add(lA);
    php.add(spl);
    php.connect(lA, spl);

    POFilter fl1 = GenPhyOp.topFilterOp();
    fl1.setRequestedParallelism(10);
    POFilter fl2 = GenPhyOp.topFilterOp();
    fl2.setRequestedParallelism(20);
    php.add(fl1);
    php.add(fl2);
    php.connect(spl, fl1);
    php.connect(spl, fl2);

    POSplit sp11 = GenPhyOp.topSplitOp();
    POSplit sp21 = GenPhyOp.topSplitOp();
    php.add(sp11);
    php.add(sp21);
    php.connect(fl1, sp11);
    php.connect(fl2, sp21);

    POFilter fl11 = GenPhyOp.topFilterOp();
    fl11.setRequestedParallelism(10);
    POFilter fl21 = GenPhyOp.topFilterOp();
    fl21.setRequestedParallelism(20);
    POFilter fl22 = GenPhyOp.topFilterOp();
    fl22.setRequestedParallelism(30);
    php.add(fl11);
    php.add(fl21);
    php.add(fl22);
    php.connect(sp11, fl11);
    php.connect(sp21, fl21);
    php.connect(sp21, fl22);

    POLocalRearrange lr1 = GenPhyOp.topLocalRearrangeOp();
    lr1.setRequestedParallelism(40);
    POLocalRearrange lr21 = GenPhyOp.topLocalRearrangeOp();
    lr21.setRequestedParallelism(15);
    POLocalRearrange lr22 = GenPhyOp.topLocalRearrangeOp();
    lr22.setRequestedParallelism(35);
    php.add(lr1);
    php.add(lr21);
    php.add(lr22);
    php.connect(fl11, lr1);
    php.connect(fl21, lr21);
    php.connect(fl22, lr22);

    POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
    php.addAsLeaf(gr);

    POPackage pk = GenPhyOp.topPackageOp();
    pk.setRequestedParallelism(25);
    php.addAsLeaf(pk);

    POSplit sp2 = GenPhyOp.topSplitOp();
    php.addAsLeaf(sp2);

    POFilter fl3 = GenPhyOp.topFilterOp();
    fl3.setRequestedParallelism(100);
    POFilter fl4 = GenPhyOp.topFilterOp();
    fl4.setRequestedParallelism(80);
    php.add(fl3);
    php.add(fl4);
    php.connect(sp2, fl3);
    php.connect(sp2, fl4);

    POUnion un = GenPhyOp.topUnionOp();
    php.addAsLeaf(un);

    POStore st = GenPhyOp.topStoreOp();
    php.addAsLeaf(st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC14.gld");
  }
Exemple #21
0
  @Test
  public void testRun2() throws Exception {
    PhysicalPlan php = new PhysicalPlan();

    PhysicalPlan part1 = new PhysicalPlan();
    POLoad lC = GenPhyOp.topLoadOp();
    POFilter fC = GenPhyOp.topFilterOp();
    POLocalRearrange lrC = GenPhyOp.topLocalRearrangeOp();
    POGlobalRearrange grC = GenPhyOp.topGlobalRearrangeOp();
    POPackage pkC = GenPhyOp.topPackageOp();
    part1.add(lC);
    part1.add(fC);
    part1.connect(lC, fC);
    part1.add(lrC);
    part1.connect(fC, lrC);
    part1.add(grC);
    part1.connect(lrC, grC);
    part1.add(pkC);
    part1.connect(grC, pkC);

    POPackage pkD = GenPhyOp.topPackageOp();
    POLocalRearrange lrD = GenPhyOp.topLocalRearrangeOp();
    POGlobalRearrange grD = GenPhyOp.topGlobalRearrangeOp();
    POLoad lD = GenPhyOp.topLoadOp();
    part1.add(lD);
    part1.add(lrD);
    part1.connect(lD, lrD);

    part1.add(grD);
    part1.connect(lrD, grD);
    part1.add(pkD);
    part1.connect(grD, pkD);
    part1.connect(pkD, grC);

    POLoad lA = GenPhyOp.topLoadOp();
    POLoad lB = GenPhyOp.topLoadOp();

    // POLoad lC = lA;
    POFilter fA = GenPhyOp.topFilterOp();

    POLocalRearrange lrA = GenPhyOp.topLocalRearrangeOp();
    POLocalRearrange lrB = GenPhyOp.topLocalRearrangeOp();

    POGlobalRearrange grAB = GenPhyOp.topGlobalRearrangeOp();

    POPackage pkAB = GenPhyOp.topPackageOp();

    POFilter fAB = GenPhyOp.topFilterOp();
    POUnion unABC = GenPhyOp.topUnionOp();

    php.add(lA);
    php.add(lB);

    php.add(fA);

    php.connect(lA, fA);

    php.add(lrA);
    php.add(lrB);

    php.connect(fA, lrA);
    php.connect(lB, lrB);

    php.add(grAB);
    php.connect(lrA, grAB);
    php.connect(lrB, grAB);

    php.add(pkAB);
    php.connect(grAB, pkAB);

    php.add(fAB);
    php.connect(pkAB, fAB);

    php.merge(part1);

    List<PhysicalOperator> leaves = new ArrayList<PhysicalOperator>();
    for (PhysicalOperator phyOp : php.getLeaves()) {
      leaves.add(phyOp);
    }

    php.add(unABC);
    for (PhysicalOperator physicalOperator : leaves) {
      php.connect(physicalOperator, unABC);
    }

    POStore st = GenPhyOp.topStoreOp();

    php.add(st);
    php.connect(unABC, st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC11.gld");
  }
  @Before
  public void setUp() throws ExecException {
    GenPhyOp.setR(r);

    GenPhyOp.setPc(pc);
  }
  public void testSpl2() throws Exception {
    PhysicalPlan php = new PhysicalPlan();

    POLoad lA = GenPhyOp.topLoadOp();
    POSplit spl = GenPhyOp.topSplitOp();
    php.add(lA);
    php.add(spl);
    php.connect(lA, spl);

    POFilter fl1 = GenPhyOp.topFilterOp();
    POFilter fl2 = GenPhyOp.topFilterOp();
    php.add(fl1);
    php.add(fl2);
    php.connect(spl, fl1);
    php.connect(spl, fl2);

    POLocalRearrange lr1 = GenPhyOp.topLocalRearrangeOp();
    POLocalRearrange lr2 = GenPhyOp.topLocalRearrangeOp();
    php.add(lr1);
    php.add(lr2);
    php.connect(fl1, lr1);
    php.connect(fl2, lr2);

    POGlobalRearrange gr = GenPhyOp.topGlobalRearrangeOp();
    php.addAsLeaf(gr);

    POPackage pk = GenPhyOp.topPackageOp();
    php.addAsLeaf(pk);

    POSplit sp2 = GenPhyOp.topSplitOp();
    php.addAsLeaf(sp2);

    POFilter fl3 = GenPhyOp.topFilterOp();
    POFilter fl4 = GenPhyOp.topFilterOp();
    php.add(fl3);
    php.add(fl4);
    php.connect(sp2, fl3);
    php.connect(sp2, fl4);

    POUnion un = GenPhyOp.topUnionOp();
    php.addAsLeaf(un);

    POStore st = GenPhyOp.topStoreOp();
    php.addAsLeaf(st);
    run(php, "test/org/apache/pig/test/data/GoldenFiles/MRC13.gld");
  }