Exemplo n.º 1
0
  @Test
  public void testGetAllWithBinaryData() throws Exception {
    Store<ByteArray, byte[], byte[]> store = getStore();
    List<ByteArray> keys = Lists.newArrayList();
    List<byte[]> values = Lists.newArrayList();

    // The Byte 0x8c is interesting, because if you use GetContent method of
    // MimeBodyPart it gets converted to 0xc2 and 0x8c
    // This thread tracks this question
    // http://stackoverflow.com/questions/23023583/mimebodypart-getcontent-corrupts-binary-data

    byte[] interestingByte = new byte[] {(byte) 0x8c};
    ByteArray interestingKey = new ByteArray(interestingByte);

    keys.add(interestingKey);
    values.add(interestingByte);

    // Add all possible byte values
    byte[] allPossibleBytes = getAllPossibleBytes();
    ByteArray allPossibleKey = new ByteArray(allPossibleBytes);
    keys.add(allPossibleKey);
    values.add(allPossibleBytes);

    assertEquals(keys.size(), values.size());
    int count = keys.size();

    for (int i = 0; i < count; i++) {
      VectorClock vc = getClock(0, 0);
      store.put(keys.get(i), new Versioned<byte[]>(values.get(i), vc), null);
    }

    Map<ByteArray, List<Versioned<byte[]>>> result = store.getAll(keys, null);
    assertGetAllValues(keys, values, result);
  }
Exemplo n.º 2
0
  @Override
  @Test
  public void testFetchedEqualsPut() throws Exception {
    System.out.println("                    Testing Fetchhed equals put                    ");
    ByteArray key = getKey();
    Store<ByteArray, byte[], byte[]> store = getStore();
    VectorClock clock = getClock(1, 1, 2, 3, 3, 4);
    byte[] value = getValue();
    System.out.println("Value chosen : " + value);
    List<Versioned<byte[]>> resultList = store.get(key, null);
    assertNotNull("Null result list obtained from a get request", resultList);
    assertEquals("Store not empty at start!", 0, resultList.size());
    Versioned<byte[]> versioned = new Versioned<byte[]>(value, clock);
    store.put(key, versioned, null);

    List<Versioned<byte[]>> found = store.get(key, null);
    assertEquals("Should only be one version stored.", 1, found.size());

    System.out.println("individual bytes");
    System.out.println("input");
    printBytes(versioned.getValue());

    System.out.println("found");
    printBytes(found.get(0).getValue());
    assertTrue("Values not equal!", valuesEqual(versioned.getValue(), found.get(0).getValue()));
  }
Exemplo n.º 3
0
  @Test
  public void testFetchPartitionKeys() {

    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
    List<Integer> fetchPartitionsList = Arrays.asList(0, 2);

    // insert it into server-0 store
    int fetchPartitionKeyCount = 0;
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
      if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchPartitionsList)) {
        fetchPartitionKeyCount++;
      }
    }

    Iterator<ByteArray> fetchIt =
        getAdminClient().fetchKeys(0, testStoreName, fetchPartitionsList, null, false);
    // check values
    int count = 0;
    while (fetchIt.hasNext()) {
      assertEquals(
          "Fetched key should belong to asked partitions",
          true,
          isKeyPartition(fetchIt.next(), 0, testStoreName, fetchPartitionsList));
      count++;
    }

    // assert all keys for asked partitions are returned.
    assertEquals("All keys for asked partitions should be received", fetchPartitionKeyCount, count);
  }
Exemplo n.º 4
0
  @Test
  public void testRecoverData() {
    // use store with replication 2, required write 2 for this test.
    String testStoreName = "test-recovery-data";

    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
    // insert it into server-0 store
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
    }

    // assert server 1 is empty
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      assertSame("entry should NOT be present at store", 0, store.get(entry.getKey()).size());
    }

    // recover all data
    adminClient.restoreDataFromReplications(1, 2);

    // assert server 1 has all entries for its partitions
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      ByteArray key = entry.getKey();
      assertSame("entry should be present for key " + key, 1, store.get(entry.getKey()).size());
      assertEquals(
          "entry value should match",
          new String(entry.getValue()),
          new String(store.get(entry.getKey()).get(0).getValue()));
    }
  }
Exemplo n.º 5
0
  protected void initializeMetadataVersions(List<StoreDefinition> storeDefs) {
    Store<ByteArray, byte[], byte[]> versionStore =
        storeRepository.getLocalStore(
            SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name());
    Properties props = new Properties();

    try {
      boolean isPropertyAdded = false;
      ByteArray metadataVersionsKey =
          new ByteArray(SystemStoreConstants.VERSIONS_METADATA_KEY.getBytes());
      List<Versioned<byte[]>> versionList = versionStore.get(metadataVersionsKey, null);
      VectorClock newClock = null;

      if (versionList != null && versionList.size() > 0) {
        byte[] versionsByteArray = versionList.get(0).getValue();
        if (versionsByteArray != null) {
          props.load(new ByteArrayInputStream(versionsByteArray));
        }
        newClock = (VectorClock) versionList.get(0).getVersion();
        newClock = newClock.incremented(0, System.currentTimeMillis());
      } else {
        newClock = new VectorClock();
      }

      // Check if version exists for cluster.xml
      if (!props.containsKey(SystemStoreConstants.CLUSTER_VERSION_KEY)) {
        props.setProperty(SystemStoreConstants.CLUSTER_VERSION_KEY, "0");
        isPropertyAdded = true;
      }

      // Check if version exists for stores.xml
      if (!props.containsKey(SystemStoreConstants.STORES_VERSION_KEY)) {
        props.setProperty(SystemStoreConstants.STORES_VERSION_KEY, "0");
        isPropertyAdded = true;
      }

      // Check if version exists for each store
      for (StoreDefinition def : storeDefs) {
        if (!props.containsKey(def.getName())) {
          props.setProperty(def.getName(), "0");
          isPropertyAdded = true;
        }
      }

      if (isPropertyAdded) {
        StringBuilder finalVersionList = new StringBuilder();
        for (String propName : props.stringPropertyNames()) {
          finalVersionList.append(propName + "=" + props.getProperty(propName) + "\n");
        }
        versionStore.put(
            metadataVersionsKey,
            new Versioned<byte[]>(finalVersionList.toString().getBytes(), newClock),
            null);
      }

    } catch (Exception e) {
      logger.error("Error while intializing metadata versions ", e);
    }
  }
  @Before
  public void setup() throws IOException {
    byte[] bytes1 = {(byte) 'A', (byte) 'B'};
    byte[] bytes2 = {(byte) 'C', (byte) 'D'};
    List<StoreDefinition> stores = ClusterTestUtils.getZZZ322StoreDefs("memory");
    StoreDefinition storeDef = stores.get(0);
    cluster = ClusterTestUtils.getZZZCluster();
    ClientConfig clientConfig = new ClientConfig();
    clientConfig.setBootstrapUrls(cluster.getNodeById(0).getSocketUrl().toString());
    clientConfig.getZoneAffinity().setEnableGetOpZoneAffinity(true);
    clientConfig.setClientZoneId(clientZoneId);
    SocketStoreClientFactory socketStoreClientFactory = new SocketStoreClientFactory(clientConfig);
    for (Integer nodeId : cluster.getNodeIds()) {
      SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
      VoldemortConfig config =
          ServerTestUtils.createServerConfigWithDefs(
              true,
              nodeId,
              TestUtils.createTempDir().getAbsolutePath(),
              cluster,
              stores,
              new Properties());
      VoldemortServer vs =
          ServerTestUtils.startVoldemortServer(socketStoreFactory, config, cluster);
      vservers.put(nodeId, vs);
      socketStoreFactories.put(nodeId, socketStoreFactory);
      Store<ByteArray, byte[], byte[]> store =
          vs.getStoreRepository().getLocalStore(storeDef.getName());
      Node node = cluster.getNodeById(nodeId);

      VectorClock version1 = new VectorClock();
      version1.incrementVersion(0, System.currentTimeMillis());
      VectorClock version2 = version1.incremented(0, System.currentTimeMillis());

      if (node.getZoneId() == clientZoneId) {
        store.put(new ByteArray(bytes1), new Versioned<byte[]>(bytes1, version1), null);
      } else {
        store.put(new ByteArray(bytes1), new Versioned<byte[]>(bytes2, version2), null);
      }
    }

    client = socketStoreClientFactory.getRawStore(storeDef.getName(), null);
  }
Exemplo n.º 7
0
 private VProto.PutResponse handlePut(VProto.PutRequest request, Store<ByteArray, byte[]> store) {
   VProto.PutResponse.Builder response = VProto.PutResponse.newBuilder();
   try {
     ByteArray key = ProtoUtils.decodeBytes(request.getKey());
     Versioned<byte[]> value = ProtoUtils.decodeVersioned(request.getVersioned());
     store.put(key, value);
   } catch (VoldemortException e) {
     response.setError(ProtoUtils.encodeError(getErrorMapper(), e));
   }
   return response.build();
 }
Exemplo n.º 8
0
 @Override
 @Test
 public void testNullKeys() throws Exception {
   System.out.println("                    Testing null Keys                    ");
   Store<ByteArray, byte[], byte[]> store = getStore();
   try {
     store.put(null, new Versioned<byte[]>(getValue(), getNewIncrementedVectorClock()), null);
     fail("Store should not put null keys!");
   } catch (IllegalArgumentException e) {
     // this is good
   } catch (NullPointerException npe) {
     // this is good
   }
   try {
     store.get(null, null);
     fail("Store should not get null keys!");
   } catch (IllegalArgumentException e) {
     // this is good
   } catch (NullPointerException npe) {
     // this is good
   }
   try {
     store.getAll(null, null);
     fail("Store should not getAll null keys!");
   } catch (IllegalArgumentException e) {
     // this is good
   } catch (NullPointerException npe) {
     // this is good
   }
   try {
     store.getAll(
         Collections.<ByteArray>singleton(null),
         Collections.<ByteArray, byte[]>singletonMap(null, null));
     fail("Store should not getAll null keys!");
   } catch (IllegalArgumentException e) {
     // this is good
   } catch (NullPointerException npe) {
     // this is good
   }
   try {
     store.delete(null, new VectorClock());
     fail("Store should not delete null keys!");
   } catch (IllegalArgumentException e) {
     // this is good
   } catch (NullPointerException npe) {
     // this is good
   }
 }
Exemplo n.º 9
0
  /** @throws IOException */
  @Test
  public void testFetchAndUpdate() throws IOException {
    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
    List<Integer> fetchAndUpdatePartitionsList = Arrays.asList(0, 2);

    // insert it into server-0 store
    int fetchPartitionKeyCount = 0;
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
      if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchAndUpdatePartitionsList)) {
        fetchPartitionKeyCount++;
      }
    }

    // assert that server1 is empty.
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet())
      assertEquals("server1 should be empty at start.", 0, store.get(entry.getKey()).size());

    // do fetch And update call server1 <-- server0
    AdminClient client = getAdminClient();
    int id = client.migratePartitions(0, 1, testStoreName, fetchAndUpdatePartitionsList, null);
    client.waitForCompletion(1, id, 60, TimeUnit.SECONDS);

    // check values
    int count = 0;
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchAndUpdatePartitionsList)) {
        assertEquals(
            "server1 store should contain fetchAndupdated partitions.",
            1,
            store.get(entry.getKey()).size());
        assertEquals(
            "entry value should match",
            new String(entry.getValue()),
            new String(store.get(entry.getKey()).get(0).getValue()));
        count++;
      }
    }

    // assert all keys for asked partitions are returned.
    assertEquals("All keys for asked partitions should be received", fetchPartitionKeyCount, count);
  }
Exemplo n.º 10
0
  @Override
  @Test
  public void testGetVersions() throws Exception {
    List<ByteArray> keys = getKeys(2);
    ByteArray key = keys.get(0);
    byte[] value = getValue();
    VectorClock vc = getClock(0, 0);
    Store<ByteArray, byte[], byte[]> store = getStore();
    store.put(key, Versioned.value(value, vc), null);
    List<Versioned<byte[]>> versioneds = store.get(key, null);
    List<Version> versions = store.getVersions(key);
    assertEquals(1, versioneds.size());
    assertTrue(versions.size() > 0);
    for (int i = 0; i < versions.size(); i++)
      assertEquals(versioneds.get(0).getVersion(), versions.get(i));

    assertEquals(0, store.getVersions(keys.get(1)).size());
  }
Exemplo n.º 11
0
  @Override
  @Test
  public void testGetAll() throws Exception {
    Store<ByteArray, byte[], byte[]> store = getStore();
    int putCount = 10;
    List<ByteArray> keys = getKeys(putCount);
    List<byte[]> values = getValues(putCount);
    assertEquals(putCount, values.size());
    for (int i = 0; i < putCount; i++) {
      VectorClock vc = getClock(0, 0);
      store.put(keys.get(i), new Versioned<byte[]>(values.get(i), vc), null);
    }

    int countForGet = putCount / 2;
    List<ByteArray> keysForGet = keys.subList(0, countForGet);
    List<byte[]> valuesForGet = values.subList(0, countForGet);
    Map<ByteArray, List<Versioned<byte[]>>> result = store.getAll(keysForGet, null);
    assertGetAllValues(keysForGet, valuesForGet, result);
  }
Exemplo n.º 12
0
  // check the basic rebalanceNode call.
  @Test
  public void testRebalanceNode() {
    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
    List<Integer> fetchAndUpdatePartitionsList = Arrays.asList(0, 2);

    // insert it into server-0 store
    int fetchPartitionKeyCount = 0;
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
      if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchAndUpdatePartitionsList)) {
        fetchPartitionKeyCount++;
      }
    }

    List<Integer> rebalancePartitionList = Arrays.asList(1, 3);
    RebalancePartitionsInfo stealInfo =
        new RebalancePartitionsInfo(
            1,
            0,
            rebalancePartitionList,
            new ArrayList<Integer>(0),
            Arrays.asList(testStoreName),
            0);
    int asyncId = adminClient.rebalanceNode(stealInfo);
    assertNotSame("Got a valid rebalanceAsyncId", -1, asyncId);

    getAdminClient().waitForCompletion(1, asyncId, 120, TimeUnit.SECONDS);

    // assert data is copied correctly
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      if (isKeyPartition(entry.getKey(), 1, testStoreName, rebalancePartitionList)) {
        assertSame("entry should be present at store", 1, store.get(entry.getKey()).size());
        assertEquals(
            "entry value should match",
            new String(entry.getValue()),
            new String(store.get(entry.getKey()).get(0).getValue()));
      }
    }
  }
Exemplo n.º 13
0
  @Test
  public void testTruncate() throws Exception {
    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);

    // insert it into server-0 store
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
    }

    // do truncate request
    getAdminClient().truncate(0, testStoreName);

    store = getStore(0, testStoreName);

    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      assertEquals("Deleted key should be missing.", 0, store.get(entry.getKey()).size());
    }
  }
Exemplo n.º 14
0
  @Test
  public void testGetWithBinaryData() throws Exception {
    Store<ByteArray, byte[], byte[]> store = getStore();

    byte[] allPossibleBytes = getAllPossibleBytes();
    ByteArray key = new ByteArray(allPossibleBytes);
    VectorClock vc = getClock(0, 0);
    Versioned<byte[]> versioned = new Versioned<byte[]>(allPossibleBytes, vc);
    store.put(key, versioned, null);

    List<Versioned<byte[]>> found = store.get(key, null);
    assertEquals("Should only be one version stored.", 1, found.size());

    System.out.println("individual bytes");
    System.out.println("input");
    printBytes(versioned.getValue());

    System.out.println("found");
    printBytes(found.get(0).getValue());
    assertTrue("Values not equal!", valuesEqual(versioned.getValue(), found.get(0).getValue()));
  }
Exemplo n.º 15
0
  public void testDeletePartitionEntries() {
    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);

    // insert it into server-0 store
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
    }

    List<Integer> deletePartitionsList = Arrays.asList(0, 2);

    // do delete partitions request
    getAdminClient().deletePartitions(0, testStoreName, deletePartitionsList, null);

    store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      if (isKeyPartition(entry.getKey(), 0, testStoreName, deletePartitionsList)) {
        assertEquals("deleted partitions should be missing.", 0, store.get(entry.getKey()).size());
      }
    }
  }
Exemplo n.º 16
0
 @Override
 public Version put(K key, Versioned<V> versioned) throws ObsoleteVersionException {
   clientStore.put(key, versioned, null);
   return versioned.getVersion();
 }
Exemplo n.º 17
0
 private void putInner(String key, Versioned<String> value) {
   innerStore.put(key, value);
 }