Beispiel #1
0
  @Before
  @SuppressWarnings("unchecked")
  public void setUp() throws IOException {
    String inputFile1 = tmpDir.copyResourceFileName("set1.txt");
    String inputFile2 = tmpDir.copyResourceFileName("set2.txt");
    if (pipelineClass == null) {
      pipeline = MemPipeline.getInstance();
    } else {
      pipeline = new MRPipeline(pipelineClass, tmpDir.getDefaultConfiguration());
    }
    PCollection<String> firstCollection =
        pipeline.read(At.textFile(inputFile1, typeFamily.strings()));
    PCollection<String> secondCollection =
        pipeline.read(At.textFile(inputFile2, typeFamily.strings()));

    LOG.info(
        "Test fixture: ["
            + pipeline.getClass().getSimpleName()
            + " : "
            + typeFamily.getClass().getSimpleName()
            + "]  First: "
            + Lists.newArrayList(firstCollection.materialize().iterator())
            + ", Second: "
            + Lists.newArrayList(secondCollection.materialize().iterator()));

    union = secondCollection.union(firstCollection);
  }
Beispiel #2
0
  @Test
  public void testQuantilesNines() {
    PTable<String, Integer> testTable =
        MemPipeline.typedTableOf(
            tableOf(strings(), ints()),
            "a",
            10,
            "a",
            20,
            "a",
            30,
            "a",
            40,
            "a",
            50,
            "a",
            60,
            "a",
            70,
            "a",
            80,
            "a",
            90,
            "a",
            100);
    Map<String, Result<Integer>> actualS =
        Quantiles.distributed(testTable, 0.9, 0.99).materializeToMap();
    Map<String, Result<Integer>> actualM =
        Quantiles.inMemory(testTable, 0.9, 0.99).materializeToMap();
    Map<String, Result<Integer>> expected =
        ImmutableMap.of("a", result(10, Pair.of(0.9, 90), Pair.of(0.99, 100)));

    assertEquals(expected, actualS);
    assertEquals(expected, actualM);
  }
Beispiel #3
0
  @Test
  public void testQuantilesExact() {
    PTable<String, Integer> testTable =
        MemPipeline.typedTableOf(
            tableOf(strings(), ints()), "a", 5, "a", 2, "a", 3, "a", 4, "a", 1);
    Map<String, Result<Integer>> actualS =
        Quantiles.distributed(testTable, 0, 0.5, 1.0).materializeToMap();
    Map<String, Result<Integer>> actualM =
        Quantiles.inMemory(testTable, 0, 0.5, 1.0).materializeToMap();
    Map<String, Result<Integer>> expected =
        ImmutableMap.of("a", result(5, Pair.of(0.0, 1), Pair.of(0.5, 3), Pair.of(1.0, 5)));

    assertEquals(expected, actualS);
    assertEquals(expected, actualM);
  }
Beispiel #4
0
  @Test
  public void testQuantilesBetween() {
    PTable<String, Integer> testTable =
        MemPipeline.typedTableOf(
            tableOf(strings(), ints()),
            "a",
            5,
            "a",
            2, // We expect the 0.5 to correspond to this element, according to the "nearest rank"
            // %ile definition.
            "a",
            4,
            "a",
            1);
    Map<String, Result<Integer>> actualS = Quantiles.distributed(testTable, 0.5).materializeToMap();
    Map<String, Result<Integer>> actualM = Quantiles.inMemory(testTable, 0.5).materializeToMap();
    Map<String, Result<Integer>> expected = ImmutableMap.of("a", result(4, Pair.of(0.5, 2)));

    assertEquals(expected, actualS);
    assertEquals(expected, actualM);
  }
 @Test
 public void testMapSideJoinLeftOuterJoin_MemPipeline() {
   runMapsideLeftOuterJoin(MemPipeline.getInstance(), true, false);
 }
 @Test
 public void testMapSideJoin_MemPipeline_Materialized() {
   runMapsideJoin(MemPipeline.getInstance(), true, true);
 }
Beispiel #7
0
 @Test
 public void testAsCollectionMemPipeline() throws IOException {
   runAsCollection(MemPipeline.getInstance());
 }
Beispiel #8
0
 @Test
 public void testPObjectMemPipeline() throws IOException {
   runPObject(MemPipeline.getInstance());
 }