예제 #1
0
  @Test
  public void testFetchPartitionKeys() {

    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
    List<Integer> fetchPartitionsList = Arrays.asList(0, 2);

    // insert it into server-0 store
    int fetchPartitionKeyCount = 0;
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
      if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchPartitionsList)) {
        fetchPartitionKeyCount++;
      }
    }

    Iterator<ByteArray> fetchIt =
        getAdminClient().fetchKeys(0, testStoreName, fetchPartitionsList, null, false);
    // check values
    int count = 0;
    while (fetchIt.hasNext()) {
      assertEquals(
          "Fetched key should belong to asked partitions",
          true,
          isKeyPartition(fetchIt.next(), 0, testStoreName, fetchPartitionsList));
      count++;
    }

    // assert all keys for asked partitions are returned.
    assertEquals("All keys for asked partitions should be received", fetchPartitionKeyCount, count);
  }
예제 #2
0
  @Test
  public void testRecoverData() {
    // use store with replication 2, required write 2 for this test.
    String testStoreName = "test-recovery-data";

    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
    // insert it into server-0 store
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
    }

    // assert server 1 is empty
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      assertSame("entry should NOT be present at store", 0, store.get(entry.getKey()).size());
    }

    // recover all data
    adminClient.restoreDataFromReplications(1, 2);

    // assert server 1 has all entries for its partitions
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      ByteArray key = entry.getKey();
      assertSame("entry should be present for key " + key, 1, store.get(entry.getKey()).size());
      assertEquals(
          "entry value should match",
          new String(entry.getValue()),
          new String(store.get(entry.getKey()).get(0).getValue()));
    }
  }
예제 #3
0
  @Test
  public void testUpdate() {
    final HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);

    Iterator<Pair<ByteArray, Versioned<byte[]>>> iterator =
        new AbstractIterator<Pair<ByteArray, Versioned<byte[]>>>() {

          final Iterator<Entry<ByteArray, byte[]>> entrySetItr = entrySet.entrySet().iterator();

          @Override
          protected Pair<ByteArray, Versioned<byte[]>> computeNext() {
            while (entrySetItr.hasNext()) {
              Entry<ByteArray, byte[]> entry = entrySetItr.next();
              return new Pair<ByteArray, Versioned<byte[]>>(
                  entry.getKey(), new Versioned<byte[]>(entry.getValue()));
            }
            return endOfData();
          }
        };

    getAdminClient().updateEntries(0, testStoreName, iterator, null);

    // check updated values
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      assertNotSame("entry should be present at store", 0, store.get(entry.getKey()).size());
      assertEquals(
          "entry value should match",
          new String(entry.getValue()),
          new String(store.get(entry.getKey()).get(0).getValue()));
    }
  }
 private void putAlltoStore() {
   for (Entry<ByteArray, byte[]> entry :
       ServerTestUtils.createRandomKeyValuePairs(TEST_KEYS).entrySet()) {
     try {
       failingStorageEngine.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()), null);
     } catch (Exception e) {
       // ignore
     }
   }
 }
예제 #5
0
  /** @throws IOException */
  @Test
  public void testFetchAndUpdate() throws IOException {
    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
    List<Integer> fetchAndUpdatePartitionsList = Arrays.asList(0, 2);

    // insert it into server-0 store
    int fetchPartitionKeyCount = 0;
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
      if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchAndUpdatePartitionsList)) {
        fetchPartitionKeyCount++;
      }
    }

    // assert that server1 is empty.
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet())
      assertEquals("server1 should be empty at start.", 0, store.get(entry.getKey()).size());

    // do fetch And update call server1 <-- server0
    AdminClient client = getAdminClient();
    int id = client.migratePartitions(0, 1, testStoreName, fetchAndUpdatePartitionsList, null);
    client.waitForCompletion(1, id, 60, TimeUnit.SECONDS);

    // check values
    int count = 0;
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchAndUpdatePartitionsList)) {
        assertEquals(
            "server1 store should contain fetchAndupdated partitions.",
            1,
            store.get(entry.getKey()).size());
        assertEquals(
            "entry value should match",
            new String(entry.getValue()),
            new String(store.get(entry.getKey()).get(0).getValue()));
        count++;
      }
    }

    // assert all keys for asked partitions are returned.
    assertEquals("All keys for asked partitions should be received", fetchPartitionKeyCount, count);
  }
예제 #6
0
  // check the basic rebalanceNode call.
  @Test
  public void testRebalanceNode() {
    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
    List<Integer> fetchAndUpdatePartitionsList = Arrays.asList(0, 2);

    // insert it into server-0 store
    int fetchPartitionKeyCount = 0;
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
      if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchAndUpdatePartitionsList)) {
        fetchPartitionKeyCount++;
      }
    }

    List<Integer> rebalancePartitionList = Arrays.asList(1, 3);
    RebalancePartitionsInfo stealInfo =
        new RebalancePartitionsInfo(
            1,
            0,
            rebalancePartitionList,
            new ArrayList<Integer>(0),
            Arrays.asList(testStoreName),
            0);
    int asyncId = adminClient.rebalanceNode(stealInfo);
    assertNotSame("Got a valid rebalanceAsyncId", -1, asyncId);

    getAdminClient().waitForCompletion(1, asyncId, 120, TimeUnit.SECONDS);

    // assert data is copied correctly
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      if (isKeyPartition(entry.getKey(), 1, testStoreName, rebalancePartitionList)) {
        assertSame("entry should be present at store", 1, store.get(entry.getKey()).size());
        assertEquals(
            "entry value should match",
            new String(entry.getValue()),
            new String(store.get(entry.getKey()).get(0).getValue()));
      }
    }
  }
예제 #7
0
  @Test
  public void testTruncate() throws Exception {
    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);

    // insert it into server-0 store
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
    }

    // do truncate request
    getAdminClient().truncate(0, testStoreName);

    store = getStore(0, testStoreName);

    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      assertEquals("Deleted key should be missing.", 0, store.get(entry.getKey()).size());
    }
  }
예제 #8
0
  public void testDeletePartitionEntries() {
    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);

    // insert it into server-0 store
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
    }

    List<Integer> deletePartitionsList = Arrays.asList(0, 2);

    // do delete partitions request
    getAdminClient().deletePartitions(0, testStoreName, deletePartitionsList, null);

    store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      if (isKeyPartition(entry.getKey(), 0, testStoreName, deletePartitionsList)) {
        assertEquals("deleted partitions should be missing.", 0, store.get(entry.getKey()).size());
      }
    }
  }
  private void doOperation(
      StreamOperations e, int nodeId, String storeName, List<Integer> partitionList) {
    switch (e) {
      case DELETE_PARTITIONS:
        putAlltoStore();
        getAdminClient().storeMntOps.deletePartitions(nodeId, storeName, partitionList, null);
        return;
      case FETCH_ENTRIES:
        putAlltoStore();
        consumeIterator(
            getAdminClient()
                .bulkFetchOps
                .fetchEntries(nodeId, storeName, partitionList, null, false));
        return;
      case FETCH_KEYS:
        putAlltoStore();
        consumeIterator(
            getAdminClient().bulkFetchOps.fetchKeys(nodeId, storeName, partitionList, null, false));
        return;
      case UPDATE_ENTRIES:
        getAdminClient()
            .streamingOps
            .updateEntries(
                nodeId,
                storeName,
                getRandomlyFailingIterator(ServerTestUtils.createRandomKeyValuePairs(TEST_KEYS)),
                null);
        return;
      case TRUNCATE_ENTRIES:
        putAlltoStore();
        getAdminClient().storeMntOps.truncate(nodeId, storeName);
        return;

      default:
        throw new RuntimeException("Unknown operation");
    }
  }
예제 #10
0
  @Test
  public void testOnePartitionEndToEndBasedOnVersion() throws Exception {
    long now = System.currentTimeMillis();

    // setup four nodes with one store and one partition
    final SocketStoreFactory socketStoreFactory =
        new ClientRequestExecutorPool(2, 10000, 100000, 32 * 1024);
    VoldemortServer[] servers = new VoldemortServer[4];
    int partitionMap[][] = {{0}, {1}, {2}, {3}};
    Cluster cluster =
        ServerTestUtils.startVoldemortCluster(
            4, servers, partitionMap, socketStoreFactory, true, null, STORES_XML, new Properties());

    Node node = cluster.getNodeById(0);
    String bootstrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort();
    AdminClient adminClient = new AdminClient(bootstrapUrl);

    byte[] value = {1, 2, 3, 4, 5, 6, 7, 8, 9};
    byte[] value2 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};

    // make versions
    VectorClock vc1 = new VectorClock();
    VectorClock vc2 = new VectorClock();
    VectorClock vc3 = new VectorClock();
    vc1.incrementVersion(0, now); // [0:1]
    vc2.incrementVersion(1, now - 5000); // [1:1]
    vc3.incrementVersion(0, now - 89000000); // [0:1], over a day old

    ArrayList<Pair<ByteArray, Versioned<byte[]>>> n0store =
        new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
    ArrayList<Pair<ByteArray, Versioned<byte[]>>> n1store =
        new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
    ArrayList<Pair<ByteArray, Versioned<byte[]>>> n2store =
        new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
    ArrayList<Pair<ByteArray, Versioned<byte[]>>> n3store =
        new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
    ArrayList<ByteArray> keysHashedToPar0 = new ArrayList<ByteArray>();

    // find store
    Versioned<List<StoreDefinition>> storeDefinitions =
        adminClient.metadataMgmtOps.getRemoteStoreDefList(0);
    List<StoreDefinition> StoreDefinitions = storeDefinitions.getValue();
    StoreDefinition storeDefinition = null;
    for (StoreDefinition def : StoreDefinitions) {
      if (def.getName().equals(STORE_NAME)) {
        storeDefinition = def;
        break;
      }
    }
    assertNotNull("No such store found: " + STORE_NAME, storeDefinition);

    RoutingStrategy router =
        new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster);
    while (keysHashedToPar0.size() < 7) {
      // generate random key
      Map<ByteArray, byte[]> map = ServerTestUtils.createRandomKeyValuePairs(1);
      ByteArray key = map.keySet().iterator().next();
      key.get()[0] = (byte) keysHashedToPar0.size();
      Integer masterPartition = router.getMasterPartition(key.get());
      if (masterPartition == 0) {
        keysHashedToPar0.add(key);
      } else {
        continue;
      }
    }
    ByteArray k6 = keysHashedToPar0.get(6);
    ByteArray k5 = keysHashedToPar0.get(5);
    ByteArray k4 = keysHashedToPar0.get(4);
    ByteArray k3 = keysHashedToPar0.get(3);
    ByteArray k2 = keysHashedToPar0.get(2);
    ByteArray k1 = keysHashedToPar0.get(1);
    ByteArray k0 = keysHashedToPar0.get(0);

    // insert K6 into node 0,1,2
    Versioned<byte[]> v6 = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k6, v6));
    n1store.add(Pair.create(k6, v6));
    n2store.add(Pair.create(k6, v6));

    // insert K6(conflicting value and version) into node 0,1,2,3
    Versioned<byte[]> v6ConflictEarly = new Versioned<byte[]>(value2, vc2);
    n0store.add(Pair.create(k6, v6ConflictEarly));
    n1store.add(Pair.create(k6, v6ConflictEarly));
    n2store.add(Pair.create(k6, v6ConflictEarly));
    n3store.add(Pair.create(k6, v6ConflictEarly));

    // insert K4,K5 into four nodes
    Versioned<byte[]> v5 = new Versioned<byte[]>(value, vc1);
    Versioned<byte[]> v4 = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k5, v5));
    n1store.add(Pair.create(k5, v5));
    n2store.add(Pair.create(k5, v5));
    n3store.add(Pair.create(k5, v5));
    n0store.add(Pair.create(k4, v4));
    n1store.add(Pair.create(k4, v4));
    n2store.add(Pair.create(k4, v4));
    n3store.add(Pair.create(k4, v4));

    // insert K3 into node 0,1,2
    Versioned<byte[]> v3 = new Versioned<byte[]>(value, vc2);
    n0store.add(Pair.create(k3, v3));
    n1store.add(Pair.create(k3, v3));
    n2store.add(Pair.create(k3, v3));

    // insert K3(conflicting but latest version) into node 0,1,2,3
    Versioned<byte[]> v3ConflictLate = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k3, v3ConflictLate));
    n1store.add(Pair.create(k3, v3ConflictLate));
    n2store.add(Pair.create(k3, v3ConflictLate));
    n3store.add(Pair.create(k3, v3ConflictLate));

    // insert K2 into node 0,1
    Versioned<byte[]> v2 = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k2, v2));
    n1store.add(Pair.create(k2, v2));

    // insert K1 into node 0
    Versioned<byte[]> v1 = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k1, v1));

    // insert K0(out of retention) into node 0,1,2
    Versioned<byte[]> v0 = new Versioned<byte[]>(value, vc3);
    n0store.add(Pair.create(k0, v0));
    n1store.add(Pair.create(k0, v0));
    n2store.add(Pair.create(k0, v0));

    // stream to store
    adminClient.streamingOps.updateEntries(0, STORE_NAME, n0store.iterator(), null);
    adminClient.streamingOps.updateEntries(1, STORE_NAME, n1store.iterator(), null);
    adminClient.streamingOps.updateEntries(2, STORE_NAME, n2store.iterator(), null);
    adminClient.streamingOps.updateEntries(3, STORE_NAME, n3store.iterator(), null);

    // should have FULL:2(K4,K5), LATEST_CONSISTENT:1(K3),
    // INCONSISTENT:2(K6,K2), ignored(K1,K0)
    List<String> urls = new ArrayList<String>();
    urls.add(bootstrapUrl);
    ConsistencyCheck.ComparisonType[] comparisonTypes = ConsistencyCheck.ComparisonType.values();

    for (ConsistencyCheck.ComparisonType type : comparisonTypes) {
      StringWriter sw = new StringWriter();
      ConsistencyCheck checker = new ConsistencyCheck(urls, STORE_NAME, 0, sw, type);
      Reporter reporter = null;
      checker.connect();
      reporter = checker.execute();

      assertEquals(7 - 2, reporter.numTotalKeys);
      assertEquals(3, reporter.numGoodKeys);
    }

    for (VoldemortServer vs : servers) {
      vs.stop();
    }
  }