private void compile(Fragment fragment, StageModel stage) throws IOException {
if (fragment.isRendezvous()) {
CompiledType compiled =
new ReduceFragmentEmitter(environment)
.emit(fragment, stage.getShuffleModel(), stage.getStageBlock());
fragment.setCompiled(compiled);
} else {
CompiledType compiled =
new MapFragmentEmitter(environment).emit(fragment, stage.getStageBlock());
fragment.setCompiled(compiled);
}
}
/**
* ステージブロックを解析して断片までの解析が終わったステージの構造を返す。
*
* @param block ステージブロック
* @return 断片までの解析が終わったステージの構造
* @throws IOException 出力に失敗した場合
*/
protected StageModel compileFragments(StageBlock block) throws IOException {
ShuffleModel shuffle = compileShuffle(block);
StageModel stage = new StageAnalyzer(environment).analyze(block, shuffle);
for (MapUnit unit : stage.getMapUnits()) {
for (Fragment fragment : unit.getFragments()) {
compile(fragment, stage);
}
}
for (ReduceUnit unit : stage.getReduceUnits()) {
for (Fragment fragment : unit.getFragments()) {
compile(fragment, stage);
}
}
return stage;
}
/** グループごとの最小値を求めるテスト。 */
@Test
public void withParameter() {
List<StageModel> stages = compile(GroupSortFlowWithParameter.class);
StageModel stage = stages.get(0);
Assume.assumeThat(stage.getReduceUnits().size(), is(1));
ReduceUnit reduce = stage.getReduceUnits().get(0);
Fragment fragment = reduce.getFragments().get(0);
Name name = fragment.getCompiled().getQualifiedName();
ClassLoader loader = start();
PortMapper mapper = new PortMapper(fragment);
MockResult<Ex1> r1 = mapper.add("r1", new Ex1Copier());
MockResult<Ex1> r2 = mapper.add("r2", new Ex1Copier());
@SuppressWarnings("unchecked")
Rendezvous<Writable> f = (Rendezvous<Writable>) create(loader, name, mapper.toArguments());
Segment segment = stage.getShuffleModel().findSegment(fragment.getInputPorts().get(0));
SegmentedWritable value = createShuffleValue(loader, stage);
Ex1 ex1 = new Ex1();
f.begin();
ex1.setValue(50);
setShuffleValue(segment, value, ex1);
f.process(value);
ex1.setValue(100);
setShuffleValue(segment, value, ex1);
f.process(value);
ex1.setValue(101);
setShuffleValue(segment, value, ex1);
f.process(value);
ex1.setValue(150);
setShuffleValue(segment, value, ex1);
f.process(value);
f.end();
assertThat(r1.getResults().size(), is(2));
assertThat(r2.getResults().size(), is(2));
assertThat(r1.getResults().get(0).getValue(), is(50));
assertThat(r1.getResults().get(1).getValue(), is(100));
assertThat(r2.getResults().get(0).getValue(), is(101));
assertThat(r2.getResults().get(1).getValue(), is(150));
}
/**
* 指定のステージのシャッフル値を返す。
*
* @param loader 利用するローダー
* @param stage 対象のステージ
* @return 生成したシャッフル値
*/
protected SegmentedWritable createShuffleValue(ClassLoader loader, StageModel stage) {
assertThat(stage.getShuffleModel(), not(nullValue()));
Name name = stage.getShuffleModel().getCompiled().getValueTypeName();
return (SegmentedWritable) create(loader, name);
}