@SuppressWarnings("unchecked")
@Override
public void build() {
init();
LazyPlume plume = new LazyPlume();
PCollection input;
PCollection input2;
try {
// Read input
input = plume.readFile("/tmp/input-wordcount.txt", collectionOf(strings()));
input2 = plume.readFile("/tmp/input-moretext.txt", collectionOf(strings()));
// Add it as workflow's input
addInput(input);
} catch (IOException e) {
throw new RuntimeException();
}
PCollection transform =
input.map(
new DoFn() {
@Override
public void process(Object v, EmitFn emitter) {
Text t = (Text) v;
emitter.emit(new Text(t.toString() + "-bar"));
}
},
collectionOf(strings()));
addOutput(plume.flatten(input2, transform)); // flatten with another file
PCollection groupedTransform =
input
.map(
new DoFn() {
@Override
public void process(Object v, EmitFn emitter) {
Text t = (Text) v;
emitter.emit(Pair.create(t, new Text("foo")));
}
},
tableOf(strings(), strings()))
.groupByKey();
addOutput(groupedTransform);
}
