コード例 #1
0
  @Test
  public void testSortBothMerge() {
    try {

      Generator generator1 =
          new Generator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
      Generator generator2 =
          new Generator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);

      final TestData.GeneratorIterator input1 =
          new TestData.GeneratorIterator(generator1, INPUT_1_SIZE);
      final TestData.GeneratorIterator input2 =
          new TestData.GeneratorIterator(generator2, INPUT_2_SIZE);

      final MatchStub matcher = new NoOpMatcher();

      final Collector<PactRecord> collector = new DiscardingOutputCollector();

      // reset the generators
      generator1.reset();
      generator2.reset();
      input1.reset();
      input2.reset();

      // compare with iterator values
      SortMergeMatchIterator<PactRecord, PactRecord, PactRecord> iterator =
          new SortMergeMatchIterator<PactRecord, PactRecord, PactRecord>(
              input1,
              input2,
              this.serializer1,
              this.comparator1,
              this.serializer2,
              this.comparator2,
              this.pairComparator11,
              this.memoryManager,
              this.ioManager,
              MEMORY_SIZE,
              64,
              0.7f,
              LocalStrategy.SORT_BOTH_MERGE,
              this.parentTask);

      long start = System.nanoTime();

      iterator.open();

      while (iterator.callWithNextKey(matcher, collector)) ;

      iterator.close();

      long elapsed = System.nanoTime() - start;
      double msecs = elapsed / (1000 * 1000);

      System.out.println("Sort-Merge Took " + msecs + " msecs.");
    } catch (Exception e) {
      e.printStackTrace();
      Assert.fail("An exception occurred during the test: " + e.getMessage());
    }
  }
コード例 #2
0
  private void doTest(
      TestData.GeneratorIterator buildInput,
      TestData.GeneratorIterator probeInput,
      Generator bgen,
      Generator pgen)
      throws Exception {
    // collect expected data
    final Map<TestData.Key, Collection<RecordMatch>> expectedFirstMatchesMap =
        HashMatchIteratorITCase.matchRecordValues(
            HashMatchIteratorITCase.collectRecordData(buildInput),
            HashMatchIteratorITCase.collectRecordData(probeInput));

    final List<Map<TestData.Key, Collection<RecordMatch>>> expectedNMatchesMapList =
        new ArrayList<Map<Key, Collection<RecordMatch>>>(NUM_PROBES);
    final JoinFunction[] nMatcher = new RecordMatchRemovingJoin[NUM_PROBES];
    for (int i = 0; i < NUM_PROBES; i++) {
      Map<TestData.Key, Collection<RecordMatch>> tmp;
      expectedNMatchesMapList.add(tmp = deepCopy(expectedFirstMatchesMap));
      nMatcher[i] = new RecordMatchRemovingJoin(tmp);
    }

    final JoinFunction firstMatcher = new RecordMatchRemovingJoin(expectedFirstMatchesMap);

    final Collector<Record> collector = new DiscardingOutputCollector<Record>();

    // reset the generators
    bgen.reset();
    pgen.reset();
    buildInput.reset();
    probeInput.reset();

    // compare with iterator values
    BuildFirstReOpenableHashMatchIterator<Record, Record, Record> iterator =
        new BuildFirstReOpenableHashMatchIterator<Record, Record, Record>(
            buildInput,
            probeInput,
            this.recordSerializer,
            this.record1Comparator,
            this.recordSerializer,
            this.record2Comparator,
            this.recordPairComparator,
            this.memoryManager,
            ioManager,
            this.parentTask,
            1.0);

    iterator.open();
    // do first join with both inputs
    while (iterator.callWithNextKey(firstMatcher, collector)) ;

    // assert that each expected match was seen for the first input
    for (Entry<TestData.Key, Collection<RecordMatch>> entry : expectedFirstMatchesMap.entrySet()) {
      if (!entry.getValue().isEmpty()) {
        Assert.fail("Collection for key " + entry.getKey() + " is not empty");
      }
    }

    for (int i = 0; i < NUM_PROBES; i++) {
      pgen.reset();
      probeInput.reset();
      // prepare ..
      iterator.reopenProbe(probeInput);
      // .. and do second join
      while (iterator.callWithNextKey(nMatcher[i], collector)) ;

      // assert that each expected match was seen for the second input
      for (Entry<TestData.Key, Collection<RecordMatch>> entry :
          expectedNMatchesMapList.get(i).entrySet()) {
        if (!entry.getValue().isEmpty()) {
          Assert.fail("Collection for key " + entry.getKey() + " is not empty");
        }
      }
    }

    iterator.close();
  }