@Before
@SuppressWarnings("unchecked")
public void setUp() throws IOException {
String inputFile1 = tmpDir.copyResourceFileName("set1.txt");
String inputFile2 = tmpDir.copyResourceFileName("set2.txt");
if (pipelineClass == null) {
pipeline = MemPipeline.getInstance();
} else {
pipeline = new MRPipeline(pipelineClass, tmpDir.getDefaultConfiguration());
}
PCollection<String> firstCollection =
pipeline.read(At.textFile(inputFile1, typeFamily.strings()));
PCollection<String> secondCollection =
pipeline.read(At.textFile(inputFile2, typeFamily.strings()));
LOG.info(
"Test fixture: ["
+ pipeline.getClass().getSimpleName()
+ " : "
+ typeFamily.getClass().getSimpleName()
+ "] First: "
+ Lists.newArrayList(firstCollection.materialize().iterator())
+ ", Second: "
+ Lists.newArrayList(secondCollection.materialize().iterator()));
union = secondCollection.union(firstCollection);
}
@Test
public void testQuantilesNines() {
PTable<String, Integer> testTable =
MemPipeline.typedTableOf(
tableOf(strings(), ints()),
"a",
10,
"a",
20,
"a",
30,
"a",
40,
"a",
50,
"a",
60,
"a",
70,
"a",
80,
"a",
90,
"a",
100);
Map<String, Result<Integer>> actualS =
Quantiles.distributed(testTable, 0.9, 0.99).materializeToMap();
Map<String, Result<Integer>> actualM =
Quantiles.inMemory(testTable, 0.9, 0.99).materializeToMap();
Map<String, Result<Integer>> expected =
ImmutableMap.of("a", result(10, Pair.of(0.9, 90), Pair.of(0.99, 100)));
assertEquals(expected, actualS);
assertEquals(expected, actualM);
}
@Test
public void testQuantilesExact() {
PTable<String, Integer> testTable =
MemPipeline.typedTableOf(
tableOf(strings(), ints()), "a", 5, "a", 2, "a", 3, "a", 4, "a", 1);
Map<String, Result<Integer>> actualS =
Quantiles.distributed(testTable, 0, 0.5, 1.0).materializeToMap();
Map<String, Result<Integer>> actualM =
Quantiles.inMemory(testTable, 0, 0.5, 1.0).materializeToMap();
Map<String, Result<Integer>> expected =
ImmutableMap.of("a", result(5, Pair.of(0.0, 1), Pair.of(0.5, 3), Pair.of(1.0, 5)));
assertEquals(expected, actualS);
assertEquals(expected, actualM);
}
@Test
public void testQuantilesBetween() {
PTable<String, Integer> testTable =
MemPipeline.typedTableOf(
tableOf(strings(), ints()),
"a",
5,
"a",
2, // We expect the 0.5 to correspond to this element, according to the "nearest rank"
// %ile definition.
"a",
4,
"a",
1);
Map<String, Result<Integer>> actualS = Quantiles.distributed(testTable, 0.5).materializeToMap();
Map<String, Result<Integer>> actualM = Quantiles.inMemory(testTable, 0.5).materializeToMap();
Map<String, Result<Integer>> expected = ImmutableMap.of("a", result(4, Pair.of(0.5, 2)));
assertEquals(expected, actualS);
assertEquals(expected, actualM);
}
@Test
public void testMapSideJoinLeftOuterJoin_MemPipeline() {
runMapsideLeftOuterJoin(MemPipeline.getInstance(), true, false);
}
@Test
public void testMapSideJoin_MemPipeline_Materialized() {
runMapsideJoin(MemPipeline.getInstance(), true, true);
}
@Test
public void testAsCollectionMemPipeline() throws IOException {
runAsCollection(MemPipeline.getInstance());
}
@Test
public void testPObjectMemPipeline() throws IOException {
runPObject(MemPipeline.getInstance());
}