Ejemplo n.º 1
0
  @Test
  public void testSerDe() throws IOException {
    Closer closer = Closer.create();
    try {
      ByteArrayOutputStream baos = closer.register(new ByteArrayOutputStream());
      DataOutputStream dos = closer.register(new DataOutputStream(baos));
      this.multiWorkUnit.write(dos);

      ByteArrayInputStream bais = closer.register((new ByteArrayInputStream(baos.toByteArray())));
      DataInputStream dis = closer.register((new DataInputStream(bais)));
      MultiWorkUnit copy = new MultiWorkUnit();
      copy.readFields(dis);

      List<WorkUnit> workUnitList = copy.getWorkUnits();
      Assert.assertEquals(workUnitList.size(), 2);

      Assert.assertEquals(workUnitList.get(0).getHighWaterMark(), 1000);
      Assert.assertEquals(workUnitList.get(0).getLowWaterMark(), 0);
      Assert.assertEquals(workUnitList.get(0).getProp("k1"), "v1");

      Assert.assertEquals(workUnitList.get(1).getHighWaterMark(), 2000);
      Assert.assertEquals(workUnitList.get(1).getLowWaterMark(), 1001);
      Assert.assertEquals(workUnitList.get(1).getProp("k2"), "v2");
    } catch (Throwable t) {
      throw closer.rethrow(t);
    } finally {
      closer.close();
    }
  }
  @BeforeClass
  public void setupWorkUnitFiles() throws IOException {
    this.conf = new Configuration();
    this.fs = FileSystem.getLocal(this.conf);
    this.stagingDirs = Lists.newArrayList();

    // Create a list of WorkUnits to serialize
    WorkUnit wu1 = createAndSetWorkUnit("wu1");
    WorkUnit wu2 = createAndSetWorkUnit("wu2");
    WorkUnit wu3 = createAndSetWorkUnit("wu3");
    WorkUnit wu4 = createAndSetWorkUnit("wu4");

    // Create a MultiWorkUnit to serialize
    MultiWorkUnit mwu1 = new MultiWorkUnit();
    mwu1.setProp(ConfigurationKeys.TASK_ID_KEY, System.nanoTime());
    mwu1.addWorkUnits(Arrays.asList(wu3, wu4));

    Path inputDir = new Path(new Path(OUTPUT_PATH, JOB_NAME), "input");

    // Writer each WorkUnit to a separate file under inputDir
    Closer closer = Closer.create();
    try {
      wu1.write(
          closer.register(
              this.fs.create(
                  new Path(
                          inputDir,
                          wu1.getProp(ConfigurationKeys.TASK_ID_KEY) + Path.SEPARATOR + "_")
                      .suffix("wu"))));
      wu2.write(
          closer.register(
              this.fs.create(
                  new Path(
                          inputDir,
                          wu2.getProp(ConfigurationKeys.TASK_ID_KEY) + Path.SEPARATOR + "_")
                      .suffix("wu"))));
      mwu1.write(
          closer.register(
              this.fs.create(
                  new Path(
                          inputDir,
                          mwu1.getProp(ConfigurationKeys.TASK_ID_KEY) + Path.SEPARATOR + "_")
                      .suffix("mwu"))));
    } finally {
      closer.close();
    }
  }
  @Override
  public List<WorkUnit> pack(Map<String, List<WorkUnit>> workUnitsByTopic, int numContainers) {
    setWorkUnitEstSizes(workUnitsByTopic);
    List<WorkUnit> workUnits = Lists.newArrayList();
    for (List<WorkUnit> workUnitsForTopic : workUnitsByTopic.values()) {

      // For each topic, merge all empty workunits into a single workunit, so that a single
      // empty task will be created instead of many.
      MultiWorkUnit zeroSizeWorkUnit = MultiWorkUnit.createEmpty();
      for (WorkUnit workUnit : workUnitsForTopic) {
        if (DoubleMath.fuzzyEquals(getWorkUnitEstSize(workUnit), 0.0, EPS)) {
          addWorkUnitToMultiWorkUnit(workUnit, zeroSizeWorkUnit);
        } else {
          workUnit.setWatermarkInterval(getWatermarkIntervalFromWorkUnit(workUnit));
          workUnits.add(workUnit);
        }
      }
      if (!zeroSizeWorkUnit.getWorkUnits().isEmpty()) {
        workUnits.add(squeezeMultiWorkUnit(zeroSizeWorkUnit));
      }
    }
    return worstFitDecreasingBinPacking(workUnits, numContainers);
  }