@Test
public void testSerDe() throws IOException {
Closer closer = Closer.create();
try {
ByteArrayOutputStream baos = closer.register(new ByteArrayOutputStream());
DataOutputStream dos = closer.register(new DataOutputStream(baos));
this.multiWorkUnit.write(dos);
ByteArrayInputStream bais = closer.register((new ByteArrayInputStream(baos.toByteArray())));
DataInputStream dis = closer.register((new DataInputStream(bais)));
MultiWorkUnit copy = new MultiWorkUnit();
copy.readFields(dis);
List<WorkUnit> workUnitList = copy.getWorkUnits();
Assert.assertEquals(workUnitList.size(), 2);
Assert.assertEquals(workUnitList.get(0).getHighWaterMark(), 1000);
Assert.assertEquals(workUnitList.get(0).getLowWaterMark(), 0);
Assert.assertEquals(workUnitList.get(0).getProp("k1"), "v1");
Assert.assertEquals(workUnitList.get(1).getHighWaterMark(), 2000);
Assert.assertEquals(workUnitList.get(1).getLowWaterMark(), 1001);
Assert.assertEquals(workUnitList.get(1).getProp("k2"), "v2");
} catch (Throwable t) {
throw closer.rethrow(t);
} finally {
closer.close();
}
}
@BeforeClass
public void setupWorkUnitFiles() throws IOException {
this.conf = new Configuration();
this.fs = FileSystem.getLocal(this.conf);
this.stagingDirs = Lists.newArrayList();
// Create a list of WorkUnits to serialize
WorkUnit wu1 = createAndSetWorkUnit("wu1");
WorkUnit wu2 = createAndSetWorkUnit("wu2");
WorkUnit wu3 = createAndSetWorkUnit("wu3");
WorkUnit wu4 = createAndSetWorkUnit("wu4");
// Create a MultiWorkUnit to serialize
MultiWorkUnit mwu1 = new MultiWorkUnit();
mwu1.setProp(ConfigurationKeys.TASK_ID_KEY, System.nanoTime());
mwu1.addWorkUnits(Arrays.asList(wu3, wu4));
Path inputDir = new Path(new Path(OUTPUT_PATH, JOB_NAME), "input");
// Writer each WorkUnit to a separate file under inputDir
Closer closer = Closer.create();
try {
wu1.write(
closer.register(
this.fs.create(
new Path(
inputDir,
wu1.getProp(ConfigurationKeys.TASK_ID_KEY) + Path.SEPARATOR + "_")
.suffix("wu"))));
wu2.write(
closer.register(
this.fs.create(
new Path(
inputDir,
wu2.getProp(ConfigurationKeys.TASK_ID_KEY) + Path.SEPARATOR + "_")
.suffix("wu"))));
mwu1.write(
closer.register(
this.fs.create(
new Path(
inputDir,
mwu1.getProp(ConfigurationKeys.TASK_ID_KEY) + Path.SEPARATOR + "_")
.suffix("mwu"))));
} finally {
closer.close();
}
}
@Override
public List<WorkUnit> pack(Map<String, List<WorkUnit>> workUnitsByTopic, int numContainers) {
setWorkUnitEstSizes(workUnitsByTopic);
List<WorkUnit> workUnits = Lists.newArrayList();
for (List<WorkUnit> workUnitsForTopic : workUnitsByTopic.values()) {
// For each topic, merge all empty workunits into a single workunit, so that a single
// empty task will be created instead of many.
MultiWorkUnit zeroSizeWorkUnit = MultiWorkUnit.createEmpty();
for (WorkUnit workUnit : workUnitsForTopic) {
if (DoubleMath.fuzzyEquals(getWorkUnitEstSize(workUnit), 0.0, EPS)) {
addWorkUnitToMultiWorkUnit(workUnit, zeroSizeWorkUnit);
} else {
workUnit.setWatermarkInterval(getWatermarkIntervalFromWorkUnit(workUnit));
workUnits.add(workUnit);
}
}
if (!zeroSizeWorkUnit.getWorkUnits().isEmpty()) {
workUnits.add(squeezeMultiWorkUnit(zeroSizeWorkUnit));
}
}
return worstFitDecreasingBinPacking(workUnits, numContainers);
}
