Exemplo n.º 1
0
  @Test
  public void testLimitAdjusterFuncShipped() throws Exception {
    String query =
        "a = load 'input';"
            + "b = order a by $0 parallel 2;"
            + "c = limit b 7;"
            + "store c into 'output' using "
            + PigStorageNoDefCtor.class.getName()
            + "('\t');";

    PhysicalPlan pp = Util.buildPp(pigServerMR, query);
    MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

    LimitAdjuster la = new LimitAdjuster(mrPlan, pc);
    la.visit();
    la.adjust();

    MapReduceOper mrOper = mrPlan.getRoots().get(0);
    int count = 1;

    while (mrPlan.getSuccessors(mrOper) != null) {
      mrOper = mrPlan.getSuccessors(mrOper).get(0);
      ++count;
    }
    assertEquals(4, count);

    MapReduceOper op = mrPlan.getLeaves().get(0);
    assertTrue(op.UDFs.contains(new FuncSpec(PigStorageNoDefCtor.class.getName()) + "('\t')"));
  }
Exemplo n.º 2
0
  // PIG-2146
  @Test
  public void testStorerLimit() throws Exception {
    // test if the POStore in the 1st mr plan
    // use the right StoreFunc
    String query =
        "a = load 'input1';"
            + "b = limit a 10;"
            + "store b into 'output' using "
            + PigStorageNoDefCtor.class.getName()
            + "(',');";

    PhysicalPlan pp = Util.buildPp(pigServer, query);
    MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

    LimitAdjuster la = new LimitAdjuster(mrPlan, pc);
    la.visit();
    la.adjust();

    MapReduceOper firstMrOper = mrPlan.getRoots().get(0);
    POStore store = (POStore) firstMrOper.reducePlan.getLeaves().get(0);
    assertEquals(store.getStoreFunc().getClass().getName(), "org.apache.pig.impl.io.InterStorage");
  }
Exemplo n.º 3
0
  /**
   * Test to ensure that the order by with parallel followed by a limit, i.e., top k always produces
   * the correct number of map reduce jobs
   */
  @Test
  public void testNumReducersInLimitWithParallel() throws Exception {
    String query =
        "a = load 'input';"
            + "b = order a by $0 parallel 2;"
            + "c = limit b 10;"
            + "store c into 'output';";

    PhysicalPlan pp = Util.buildPp(pigServerMR, query);
    MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

    LimitAdjuster la = new LimitAdjuster(mrPlan, pc);
    la.visit();
    la.adjust();

    MapReduceOper mrOper = mrPlan.getRoots().get(0);
    int count = 1;

    while (mrPlan.getSuccessors(mrOper) != null) {
      mrOper = mrPlan.getSuccessors(mrOper).get(0);
      ++count;
    }
    assertEquals(4, count);
  }