Beispiel #1
0
 public void addInput(MutableObjectIterator<PactRecord> input, int groupId) {
   this.mockEnv.addInput(input);
   TaskConfig conf = new TaskConfig(this.mockEnv.getTaskConfiguration());
   conf.addInputToGroup(groupId);
   conf.setInputSerializer(PactRecordSerializerFactory.get(), groupId);
 }
Beispiel #2
0
 public void addOutput(List<PactRecord> output) {
   this.mockEnv.addOutput(output);
   TaskConfig conf = new TaskConfig(this.mockEnv.getTaskConfiguration());
   conf.addOutputShipStrategy(ShipStrategyType.FORWARD);
   conf.setOutputSerializer(PactRecordSerializerFactory.get());
 }
public class ChainTaskTest extends TaskTestBase {

  private final List<PactRecord> outList = new ArrayList<PactRecord>();

  @SuppressWarnings("unchecked")
  private final PactRecordComparatorFactory compFact =
      new PactRecordComparatorFactory(
          new int[] {0}, new Class[] {PactInteger.class}, new boolean[] {true});

  private final PactRecordSerializerFactory serFact = PactRecordSerializerFactory.get();

  @Test
  public void testMapTask() {
    final int keyCnt = 100;
    final int valCnt = 20;

    try {

      // environment
      initEnvironment(3 * 1024 * 1024);
      addInput(new UniformPactRecordGenerator(keyCnt, valCnt, false), 0);
      addOutput(this.outList);

      // chained combine config
      {
        final TaskConfig combineConfig = new TaskConfig(new Configuration());

        // input
        combineConfig.addInputToGroup(0);
        combineConfig.setInputSerializer(serFact, 0);

        // output
        combineConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
        combineConfig.setOutputSerializer(serFact);

        // driver
        combineConfig.setDriverStrategy(DriverStrategy.PARTIAL_GROUP);
        combineConfig.setDriverComparator(compFact, 0);
        combineConfig.setMemoryDriver(3 * 1024 * 1024);

        // udf
        combineConfig.setStubClass(MockReduceStub.class);

        getTaskConfig().addChainedTask(ChainedCombineDriver.class, combineConfig, "combine");
      }

      // chained map+combine
      {
        RegularPactTask<GenericMapper<PactRecord, PactRecord>, PactRecord> testTask =
            new RegularPactTask<GenericMapper<PactRecord, PactRecord>, PactRecord>();
        registerTask(testTask, MapDriver.class, MockMapStub.class);

        try {
          testTask.invoke();
        } catch (Exception e) {
          e.printStackTrace();
          Assert.fail("Invoke method caused exception.");
        }
      }

      Assert.assertEquals(keyCnt, this.outList.size());
    } catch (Exception e) {
      e.printStackTrace();
      Assert.fail(e.getMessage());
    }
  }

  @Test
  public void testFailingMapTask() {
    int keyCnt = 100;
    int valCnt = 20;

    try {
      // environment
      initEnvironment(3 * 1024 * 1024);
      addInput(new UniformPactRecordGenerator(keyCnt, valCnt, false), 0);
      addOutput(this.outList);

      // chained combine config
      {
        final TaskConfig combineConfig = new TaskConfig(new Configuration());

        // input
        combineConfig.addInputToGroup(0);
        combineConfig.setInputSerializer(serFact, 0);

        // output
        combineConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
        combineConfig.setOutputSerializer(serFact);

        // driver
        combineConfig.setDriverStrategy(DriverStrategy.PARTIAL_GROUP);
        combineConfig.setDriverComparator(compFact, 0);
        combineConfig.setMemoryDriver(3 * 1024 * 1024);

        // udf
        combineConfig.setStubClass(MockFailingCombineStub.class);

        getTaskConfig().addChainedTask(ChainedCombineDriver.class, combineConfig, "combine");
      }

      // chained map+combine
      {
        final RegularPactTask<GenericMapper<PactRecord, PactRecord>, PactRecord> testTask =
            new RegularPactTask<GenericMapper<PactRecord, PactRecord>, PactRecord>();

        super.registerTask(testTask, MapDriver.class, MockMapStub.class);

        boolean stubFailed = false;

        try {
          testTask.invoke();
        } catch (Exception e) {
          stubFailed = true;
        }

        Assert.assertTrue("Stub exception was not forwarded.", stubFailed);
      }
    } catch (Exception e) {
      e.printStackTrace();
      Assert.fail(e.getMessage());
    }
  }

  public static final class MockFailingCombineStub extends ReduceStub {

    private int cnt = 0;

    @Override
    public void reduce(Iterator<PactRecord> records, Collector<PactRecord> out) throws Exception {
      if (++this.cnt >= 5) {
        throw new RuntimeException("Expected Test Exception");
      }
      while (records.hasNext()) out.collect(records.next());
    }
  }
}