@Test
public void unionWriteShouldNotThrowNPE() throws IOException {
String outputPath1 = tmpDir.getFileName("output1");
String outputPath2 = tmpDir.getFileName("output2");
String outputPath3 = tmpDir.getFileName("output3");
if (typeFamily == AvroTypeFamily.getInstance()) {
union.write(To.avroFile(outputPath1));
pipeline.write(union, To.avroFile(outputPath2));
pipeline.run();
checkFileContents(outputPath1);
checkFileContents(outputPath2);
} else {
union.write(To.textFile(outputPath1));
pipeline.write(union, To.textFile(outputPath2));
pipeline.writeTextFile(union, outputPath3);
pipeline.run();
checkFileContents(outputPath1);
checkFileContents(outputPath2);
checkFileContents(outputPath3);
}
}
public static void main(String[] args) throws Exception {
Pipeline pipeline = new MRPipeline(WordCount.class);
PCollection<String> lines = pipeline.readTextFile(args[0]);
PCollection<String> words =
lines.parallelDo(
"my splitter",
new DoFn<String, String>() {
public void process(String line, Emitter<String> emitter) {
for (String word : line.split("\\s+")) {
emitter.emit(word);
}
}
},
Writables.strings());
PTable<String, Long> counts = Aggregate.count(words);
pipeline.writeTextFile(counts, args[1]);
pipeline.run();
}