Exemple #1
0
  @Test
  public void testFetchImpl() {
    ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0);
    Task.PullHost pullHost = new Task.PullHost("localhost", 0);

    FetchImpl expected = new FetchImpl(pullHost, SCATTERED_HASH_SHUFFLE, ebId, 1);
    FetchImpl fetch2 = new FetchImpl(pullHost, SCATTERED_HASH_SHUFFLE, ebId, 1);
    assertEquals(expected, fetch2);
    fetch2.setOffset(5);
    fetch2.setLength(10);
    assertNotEquals(expected, fetch2);
  }
Exemple #2
0
  @Test
  public void testScheduleFetchesByEvenDistributedVolumes() {
    Map<Integer, FetchGroupMeta> fetchGroups = Maps.newHashMap();
    String tableName = "test1";

    ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0);
    FetchImpl[] fetches = new FetchImpl[12];
    for (int i = 0; i < 12; i++) {
      fetches[i] =
          new FetchImpl(new Task.PullHost("localhost", 10000 + i), HASH_SHUFFLE, ebId, i / 2);
    }

    int[] VOLUMES = {100, 80, 70, 30, 10, 5};

    for (int i = 0; i < 12; i += 2) {
      fetchGroups.put(i, new FetchGroupMeta(VOLUMES[i / 2], fetches[i]).addFetche(fetches[i + 1]));
    }

    Pair<Long[], Map<String, List<FetchImpl>>[]> results;

    results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 1);
    long expected[] = {100 + 80 + 70 + 30 + 10 + 5};
    assertFetchVolumes(expected, results.getFirst());
    assertFetchImpl(fetches, results.getSecond());

    results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 2);
    long expected0[] = {140, 155};
    assertFetchVolumes(expected0, results.getFirst());
    assertFetchImpl(fetches, results.getSecond());

    results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 3);
    long expected1[] = {100, 95, 100};
    assertFetchVolumes(expected1, results.getFirst());
    assertFetchImpl(fetches, results.getSecond());

    results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 4);
    long expected2[] = {100, 80, 70, 45};
    assertFetchVolumes(expected2, results.getFirst());
    assertFetchImpl(fetches, results.getSecond());

    results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 5);
    long expected3[] = {100, 80, 70, 30, 15};
    assertFetchVolumes(expected3, results.getFirst());
    assertFetchImpl(fetches, results.getSecond());

    results = Repartitioner.makeEvenDistributedFetchImpl(fetchGroups, tableName, 6);
    long expected4[] = {100, 80, 70, 30, 10, 5};
    assertFetchVolumes(expected4, results.getFirst());
    assertFetchImpl(fetches, results.getSecond());
  }