@Override
 public void setUp() throws Exception {
   super.setUp();
   metadataStore =
       ServerTestUtils.createMetadataStore(
           ServerTestUtils.getLocalCluster(1), ServerTestUtils.getStoreDefs(1));
 }
  private AbstractSocketService getAdminServer(
      Node node,
      Cluster cluster,
      List<StoreDefinition> storeDefs,
      StorageEngine<ByteArray, byte[], byte[]> storageEngine)
      throws IOException {
    StoreRepository storeRepository = new StoreRepository();
    storeRepository.addStorageEngine(storageEngine);
    storeRepository.addLocalStore(storageEngine);

    SocketRequestHandlerFactory requestHandlerFactory =
        new SocketRequestHandlerFactory(
            null,
            storeRepository,
            ServerTestUtils.createMetadataStore(cluster, storeDefs),
            ServerTestUtils.createServerConfig(
                useNio,
                0,
                TestUtils.createTempDir().getAbsolutePath(),
                null,
                null,
                new Properties()),
            null,
            null,
            null,
            null);
    return ServerTestUtils.getSocketService(
        useNio, requestHandlerFactory, node.getAdminPort(), 2, 2, 10000);
  }
Exemple #3
0
  // This method is susceptible to BindException issues due to TOCTOU
  // problem with getLocalCluster (which is used to construct cluster that is
  // passed in).
  // TODO: Refactor AbstractRebalanceTest to take advantage of
  // ServerTestUtils.startVoldemortCluster.
  @Override
  protected Cluster startServers(
      Cluster cluster,
      String storeXmlFile,
      List<Integer> nodeToStart,
      Map<String, String> configProps)
      throws IOException {
    for (int node : nodeToStart) {
      Properties properties = new Properties();
      if (null != configProps) {
        for (Entry<String, String> property : configProps.entrySet()) {
          properties.put(property.getKey(), property.getValue());
        }
      }

      VoldemortConfig config =
          ServerTestUtils.createServerConfig(
              useNio,
              node,
              TestUtils.createTempDir().getAbsolutePath(),
              null,
              storeXmlFile,
              properties);

      VoldemortServer server =
          ServerTestUtils.startVoldemortServer(socketStoreFactory, config, cluster);
      serverMap.put(node, server);
    }

    return cluster;
  }
  public byte[] getValidValue(ByteArray key) {
    String keyString = ByteUtils.getString(key.get(), "UTF-8");
    if (MetadataStore.CLUSTER_KEY.equals(keyString)) {
      return ByteUtils.getBytes(
          new ClusterMapper().writeCluster(ServerTestUtils.getLocalCluster(1)), "UTF-8");
    } else if (MetadataStore.STORES_KEY.equals(keyString)) {
      return ByteUtils.getBytes(
          new StoreDefinitionsMapper().writeStoreList(ServerTestUtils.getStoreDefs(1)), "UTF-8");

    } else if (MetadataStore.SERVER_STATE_KEY.equals(keyString)) {
      int i = (int) (Math.random() * VoldemortState.values().length);
      return ByteUtils.getBytes(VoldemortState.values()[i].toString(), "UTF-8");
    } else if (MetadataStore.REBALANCING_STEAL_INFO.equals(keyString)) {
      int size = (int) (Math.random() * 10);
      List<Integer> partition = new ArrayList<Integer>();
      for (int i = 0; i < size; i++) {
        partition.add((int) Math.random() * 10);
      }

      return ByteUtils.getBytes(
          new RebalancerState(
                  Arrays.asList(
                      new RebalancePartitionsInfo(
                          0,
                          (int) Math.random() * 5,
                          partition,
                          new ArrayList<Integer>(0),
                          new ArrayList<Integer>(0),
                          Arrays.asList("testStoreName"),
                          new HashMap<String, String>(),
                          new HashMap<String, String>(),
                          (int) Math.random() * 3)))
              .toJsonString(),
          "UTF-8");
    } else if (MetadataStore.GRANDFATHERING_INFO.equals(keyString)) {
      int size = (int) (Math.random() * 10);
      List<Integer> partition = new ArrayList<Integer>();
      for (int i = 0; i < size; i++) {
        partition.add((int) Math.random() * 10);
      }
      return ByteUtils.getBytes(
          new GrandfatherState(
                  Arrays.asList(
                      new RebalancePartitionsInfo(
                          0,
                          (int) Math.random() * 5,
                          partition,
                          new ArrayList<Integer>(0),
                          new ArrayList<Integer>(0),
                          Arrays.asList("testStoreName"),
                          new HashMap<String, String>(),
                          new HashMap<String, String>(),
                          (int) Math.random() * 3)))
              .toJsonString(),
          "UTF-8");
    }

    throw new RuntimeException("Unhandled key:" + keyString + " passed");
  }
  @Override
  @Before
  public void setUp() throws IOException {
    cluster = ServerTestUtils.getLocalCluster(2, new int[][] {{0, 1, 2, 3}, {4, 5, 6, 7}});
    List<StoreDefinition> storeDefs = ServerTestUtils.getStoreDefs(1);

    failingStorageEngine =
        new RandomlyFailingDelegatingStore<ByteArray, byte[], byte[]>(
            new InMemoryStorageEngine<ByteArray, byte[], byte[]>(storeDefs.get(0).getName()));
    adminServer = getAdminServer(cluster.getNodeById(0), cluster, storeDefs, failingStorageEngine);
    adminClient = ServerTestUtils.getAdminClient(cluster);
    adminServer.start();
  }
  @Test
  public void testUpdate() {
    final HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);

    Iterator<Pair<ByteArray, Versioned<byte[]>>> iterator =
        new AbstractIterator<Pair<ByteArray, Versioned<byte[]>>>() {

          final Iterator<Entry<ByteArray, byte[]>> entrySetItr = entrySet.entrySet().iterator();

          @Override
          protected Pair<ByteArray, Versioned<byte[]>> computeNext() {
            while (entrySetItr.hasNext()) {
              Entry<ByteArray, byte[]> entry = entrySetItr.next();
              return new Pair<ByteArray, Versioned<byte[]>>(
                  entry.getKey(), new Versioned<byte[]>(entry.getValue()));
            }
            return endOfData();
          }
        };

    getAdminClient().updateEntries(0, testStoreName, iterator, null);

    // check updated values
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      assertNotSame("entry should be present at store", 0, store.get(entry.getKey()).size());
      assertEquals(
          "entry value should match",
          new String(entry.getValue()),
          new String(store.get(entry.getKey()).get(0).getValue()));
    }
  }
 @Override
 public void tearDown() throws IOException, InterruptedException {
   adminClient.stop();
   for (VoldemortServer server : servers) {
     ServerTestUtils.stopVoldemortServer(server);
   }
 }
  @Test
  public void testFetchPartitionKeys() {

    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
    List<Integer> fetchPartitionsList = Arrays.asList(0, 2);

    // insert it into server-0 store
    int fetchPartitionKeyCount = 0;
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
      if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchPartitionsList)) {
        fetchPartitionKeyCount++;
      }
    }

    Iterator<ByteArray> fetchIt =
        getAdminClient().fetchKeys(0, testStoreName, fetchPartitionsList, null, false);
    // check values
    int count = 0;
    while (fetchIt.hasNext()) {
      assertEquals(
          "Fetched key should belong to asked partitions",
          true,
          isKeyPartition(fetchIt.next(), 0, testStoreName, fetchPartitionsList));
      count++;
    }

    // assert all keys for asked partitions are returned.
    assertEquals("All keys for asked partitions should be received", fetchPartitionKeyCount, count);
  }
  @Test
  public void testRecoverData() {
    // use store with replication 2, required write 2 for this test.
    String testStoreName = "test-recovery-data";

    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
    // insert it into server-0 store
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
    }

    // assert server 1 is empty
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      assertSame("entry should NOT be present at store", 0, store.get(entry.getKey()).size());
    }

    // recover all data
    adminClient.restoreDataFromReplications(1, 2);

    // assert server 1 has all entries for its partitions
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      ByteArray key = entry.getKey();
      assertSame("entry should be present for key " + key, 1, store.get(entry.getKey()).size());
      assertEquals(
          "entry value should match",
          new String(entry.getValue()),
          new String(store.get(entry.getKey()).get(0).getValue()));
    }
  }
 @Override
 public void setUp() throws Exception {
   super.setUp();
   socketService =
       ServerTestUtils.getSocketService(
           getClusterXml(), getStoreDefXml(), getValidStoreName(), getLocalNode().getSocketPort());
   socketService.start();
 }
 @Override
 @Before
 public void setUp() throws Exception {
   super.setUp();
   context =
       ServerTestUtils.getJettyServer(
           getClusterXml(),
           getStoreDefXml(),
           getValidStoreName(),
           RequestFormatType.VOLDEMORT_V1,
           getLocalNode().getHttpPort());
   server = context.getServer();
   httpStore =
       ServerTestUtils.getHttpStore(
           getValidStoreName(), RequestFormatType.VOLDEMORT_V1, getLocalNode().getHttpPort());
   url = getLocalNode().getHttpUrl().toString();
 }
 @Override
 @After
 public void tearDown() throws Exception {
   super.tearDown();
   for (VoldemortServer server : servers) {
     ServerTestUtils.stopVoldemortServer(server);
   }
   store.close();
 }
Exemple #13
0
 @Override
 protected void stopServer(List<Integer> nodesToStop) throws IOException {
   for (int node : nodesToStop) {
     try {
       ServerTestUtils.stopVoldemortServer(serverMap.get(node));
     } catch (VoldemortException e) {
       // ignore these at stop time
     }
   }
 }
 private void putAlltoStore() {
   for (Entry<ByteArray, byte[]> entry :
       ServerTestUtils.createRandomKeyValuePairs(TEST_KEYS).entrySet()) {
     try {
       failingStorageEngine.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()), null);
     } catch (Exception e) {
       // ignore
     }
   }
 }
  @Before
  public void setup() throws IOException {
    byte[] bytes1 = {(byte) 'A', (byte) 'B'};
    byte[] bytes2 = {(byte) 'C', (byte) 'D'};
    List<StoreDefinition> stores = ClusterTestUtils.getZZZ322StoreDefs("memory");
    StoreDefinition storeDef = stores.get(0);
    cluster = ClusterTestUtils.getZZZCluster();
    ClientConfig clientConfig = new ClientConfig();
    clientConfig.setBootstrapUrls(cluster.getNodeById(0).getSocketUrl().toString());
    clientConfig.getZoneAffinity().setEnableGetOpZoneAffinity(true);
    clientConfig.setClientZoneId(clientZoneId);
    SocketStoreClientFactory socketStoreClientFactory = new SocketStoreClientFactory(clientConfig);
    for (Integer nodeId : cluster.getNodeIds()) {
      SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
      VoldemortConfig config =
          ServerTestUtils.createServerConfigWithDefs(
              true,
              nodeId,
              TestUtils.createTempDir().getAbsolutePath(),
              cluster,
              stores,
              new Properties());
      VoldemortServer vs =
          ServerTestUtils.startVoldemortServer(socketStoreFactory, config, cluster);
      vservers.put(nodeId, vs);
      socketStoreFactories.put(nodeId, socketStoreFactory);
      Store<ByteArray, byte[], byte[]> store =
          vs.getStoreRepository().getLocalStore(storeDef.getName());
      Node node = cluster.getNodeById(nodeId);

      VectorClock version1 = new VectorClock();
      version1.incrementVersion(0, System.currentTimeMillis());
      VectorClock version2 = version1.incremented(0, System.currentTimeMillis());

      if (node.getZoneId() == clientZoneId) {
        store.put(new ByteArray(bytes1), new Versioned<byte[]>(bytes1, version1), null);
      } else {
        store.put(new ByteArray(bytes1), new Versioned<byte[]>(bytes2, version2), null);
      }
    }

    client = socketStoreClientFactory.getRawStore(storeDef.getName(), null);
  }
Exemple #16
0
  @After
  public void tearDown() throws Exception {
    for (VoldemortServer server : servers) {
      ServerTestUtils.stopVoldemortServer(server);
    }

    coordinator.stop();

    // clean up the temporary file created in set up
    COPY_OF_FAT_CLIENT_CONFIG_FILE.delete();
  }
  @Override
  @Before
  public void setUp() throws IOException {
    cluster = ServerTestUtils.getLocalCluster(2, new int[][] {{0, 1, 2, 3}, {4, 5, 6, 7}});
    servers = new VoldemortServer[2];

    servers[0] =
        ServerTestUtils.startVoldemortServer(
            ServerTestUtils.createServerConfig(
                useNio,
                0,
                TestUtils.createTempDir().getAbsolutePath(),
                null,
                storesXmlfile,
                new Properties()),
            cluster);
    servers[1] =
        ServerTestUtils.startVoldemortServer(
            ServerTestUtils.createServerConfig(
                useNio,
                1,
                TestUtils.createTempDir().getAbsolutePath(),
                null,
                storesXmlfile,
                new Properties()),
            cluster);

    adminClient = ServerTestUtils.getAdminClient(cluster);
  }
  public byte[] getValidValue(ByteArray key) {
    String keyString = ByteUtils.getString(key.get(), "UTF-8");
    if (MetadataStore.CLUSTER_KEY.equals(keyString)
        || MetadataStore.REBALANCING_SOURCE_CLUSTER_XML.equals(keyString)) {
      return ByteUtils.getBytes(
          new ClusterMapper().writeCluster(ServerTestUtils.getLocalCluster(1)), "UTF-8");
    } else if (MetadataStore.STORES_KEY.equals(keyString)) {
      return ByteUtils.getBytes(
          new StoreDefinitionsMapper().writeStoreList(ServerTestUtils.getStoreDefs(1)), "UTF-8");

    } else if (MetadataStore.SERVER_STATE_KEY.equals(keyString)) {
      int i = (int) (Math.random() * VoldemortState.values().length);
      return ByteUtils.getBytes(VoldemortState.values()[i].toString(), "UTF-8");
    } else if (MetadataStore.REBALANCING_STEAL_INFO.equals(keyString)) {
      int size = (int) (Math.random() * 10) + 1;
      List<Integer> partition = new ArrayList<Integer>();
      for (int i = 0; i < size; i++) {
        partition.add((int) Math.random() * 10);
      }

      List<Integer> partitionIds = partition;

      HashMap<String, List<Integer>> storeToReplicaToPartitionList = Maps.newHashMap();
      storeToReplicaToPartitionList.put("test", partitionIds);

      return ByteUtils.getBytes(
          new RebalancerState(
                  Arrays.asList(
                      new RebalanceTaskInfo(
                          0,
                          (int) Math.random() * 5,
                          storeToReplicaToPartitionList,
                          ServerTestUtils.getLocalCluster(1))))
              .toJsonString(),
          "UTF-8");
    }

    throw new RuntimeException("Unhandled key:" + keyString + " passed");
  }
  @Before
  public void setUp() throws Exception {
    final int numServers = 2;
    servers = new VoldemortServer[numServers];
    int partitionMap[][] = {{0, 1, 2, 3}, {4, 5, 6, 7}};
    cluster =
        ServerTestUtils.startVoldemortCluster(
            numServers,
            servers,
            partitionMap,
            socketStoreFactory,
            true, // useNio
            null,
            storesXmlfile,
            new Properties());

    socketUrl = servers[0].getIdentityNode().getSocketUrl().toString();
    bootStrapUrls = new String[1];
    bootStrapUrls[0] = socketUrl;

    Node node = cluster.getNodeById(0);
    String bootstrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort();
    ClientConfig clientConfig = new ClientConfig();
    clientConfig.setClientRegistryUpdateIntervalInSecs(5);
    clientConfig.setAsyncMetadataRefreshInMs(5000);
    clientConfig.setBootstrapUrls(bootstrapUrl);
    SocketStoreClientFactory storeClientFactory = new SocketStoreClientFactory(clientConfig);

    storeClient =
        new ZenStoreClient<String, String>(
            STORE_NAME,
            null,
            storeClientFactory,
            3,
            clientConfig.getClientContextName(),
            0,
            clientConfig);

    SystemStoreClientFactory<String, String> systemStoreFactory =
        new SystemStoreClientFactory<String, String>(clientConfig);

    sysStoreVersion =
        systemStoreFactory.createSystemStore(
            SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name());
    clientRegistryStore =
        systemStoreFactory.createSystemStore(
            SystemStoreConstants.SystemStoreName.voldsys$_client_registry.name());
  }
  /** @throws IOException */
  @Test
  public void testFetchAndUpdate() throws IOException {
    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
    List<Integer> fetchAndUpdatePartitionsList = Arrays.asList(0, 2);

    // insert it into server-0 store
    int fetchPartitionKeyCount = 0;
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
      if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchAndUpdatePartitionsList)) {
        fetchPartitionKeyCount++;
      }
    }

    // assert that server1 is empty.
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet())
      assertEquals("server1 should be empty at start.", 0, store.get(entry.getKey()).size());

    // do fetch And update call server1 <-- server0
    AdminClient client = getAdminClient();
    int id = client.migratePartitions(0, 1, testStoreName, fetchAndUpdatePartitionsList, null);
    client.waitForCompletion(1, id, 60, TimeUnit.SECONDS);

    // check values
    int count = 0;
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchAndUpdatePartitionsList)) {
        assertEquals(
            "server1 store should contain fetchAndupdated partitions.",
            1,
            store.get(entry.getKey()).size());
        assertEquals(
            "entry value should match",
            new String(entry.getValue()),
            new String(store.get(entry.getKey()).get(0).getValue()));
        count++;
      }
    }

    // assert all keys for asked partitions are returned.
    assertEquals("All keys for asked partitions should be received", fetchPartitionKeyCount, count);
  }
  @Override
  @Before
  public void setUp() {
    logger.info(" Initial SEED used for random number generator: " + TestUtils.SEED);
    final int numServers = 1;
    this.nodeId = 0;
    servers = new VoldemortServer[numServers];
    try {

      // Setup the cluster
      Properties props = new Properties();
      props.setProperty("rest.enable", "true");
      props.setProperty("http.enable", "true");

      Cluster customCluster = clusterMapper.readCluster(new FileReader(clusterXmlFile), false);

      cluster =
          ServerTestUtils.startVoldemortCluster(
              servers, null, clusterXmlFile, storesXmlfile, props, customCluster);

    } catch (IOException e) {
      fail("Failure to setup the cluster");
    }

    // Creating R2Store
    RESTClientConfig restClientConfig = new RESTClientConfig();
    restClientConfig
        .setHttpBootstrapURL("http://localhost:" + cluster.getNodeById(0).getRestPort())
        .setTimeoutMs(10000, TimeUnit.MILLISECONDS)
        .setMaxR2ConnectionPoolSize(100);
    clientFactory = new HttpClientFactory();
    Map<String, String> properties = new HashMap<String, String>();
    properties.put(
        HttpClientFactory.POOL_SIZE_KEY,
        Integer.toString(restClientConfig.getMaxR2ConnectionPoolSize()));
    TransportClient transportClient = clientFactory.getClient(properties);
    R2Store r2Store =
        new R2Store(
            STORE_NAME,
            restClientConfig.getHttpBootstrapURL(),
            "0",
            transportClient,
            restClientConfig,
            0);
    store = r2Store;
  }
  @Test
  public void testRebalacingSourceClusterXmlKey() {
    metadataStore.cleanAllRebalancingState();

    assertTrue("Should be null", null == metadataStore.getRebalancingSourceCluster());

    Cluster dummyCluster = ServerTestUtils.getLocalCluster(2);
    metadataStore.put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, dummyCluster);
    assertEquals("Should be equal", dummyCluster, metadataStore.getRebalancingSourceCluster());

    metadataStore.put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, (Object) null);
    assertTrue("Should be null", null == metadataStore.getRebalancingSourceCluster());

    List<Versioned<byte[]>> sourceClusterVersions =
        metadataStore.get(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, null);
    assertTrue("Just one version expected", 1 == sourceClusterVersions.size());
    assertEquals(
        "Empty string should map to null", "", new String(sourceClusterVersions.get(0).getValue()));
  }
  // check the basic rebalanceNode call.
  @Test
  public void testRebalanceNode() {
    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
    List<Integer> fetchAndUpdatePartitionsList = Arrays.asList(0, 2);

    // insert it into server-0 store
    int fetchPartitionKeyCount = 0;
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
      if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchAndUpdatePartitionsList)) {
        fetchPartitionKeyCount++;
      }
    }

    List<Integer> rebalancePartitionList = Arrays.asList(1, 3);
    RebalancePartitionsInfo stealInfo =
        new RebalancePartitionsInfo(
            1,
            0,
            rebalancePartitionList,
            new ArrayList<Integer>(0),
            Arrays.asList(testStoreName),
            0);
    int asyncId = adminClient.rebalanceNode(stealInfo);
    assertNotSame("Got a valid rebalanceAsyncId", -1, asyncId);

    getAdminClient().waitForCompletion(1, asyncId, 120, TimeUnit.SECONDS);

    // assert data is copied correctly
    store = getStore(1, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      if (isKeyPartition(entry.getKey(), 1, testStoreName, rebalancePartitionList)) {
        assertSame("entry should be present at store", 1, store.get(entry.getKey()).size());
        assertEquals(
            "entry value should match",
            new String(entry.getValue()),
            new String(store.get(entry.getKey()).get(0).getValue()));
      }
    }
  }
  @Before
  public void setup() throws IOException {

    Properties props = new Properties();
    props.put("enable.quota.limiting", "true");
    server =
        ServerTestUtils.startStandAloneVoldemortServer(
            props, "test/common/voldemort/config/single-store.xml");

    adminClient =
        new AdminClient(
            server.getMetadataStore().getCluster(), new AdminClientConfig(), new ClientConfig());
    String bootStrapUrl =
        "tcp://"
            + server.getIdentityNode().getHost()
            + ":"
            + server.getIdentityNode().getSocketPort();
    factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(bootStrapUrl));
    storeClient = factory.getStoreClient("test");
  }
  @Test
  public void testTruncate() throws Exception {
    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);

    // insert it into server-0 store
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
    }

    // do truncate request
    getAdminClient().truncate(0, testStoreName);

    store = getStore(0, testStoreName);

    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      assertEquals("Deleted key should be missing.", 0, store.get(entry.getKey()).size());
    }
  }
  public void testDeletePartitionEntries() {
    HashMap<ByteArray, byte[]> entrySet =
        ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);

    // insert it into server-0 store
    Store<ByteArray, byte[]> store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
    }

    List<Integer> deletePartitionsList = Arrays.asList(0, 2);

    // do delete partitions request
    getAdminClient().deletePartitions(0, testStoreName, deletePartitionsList, null);

    store = getStore(0, testStoreName);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
      if (isKeyPartition(entry.getKey(), 0, testStoreName, deletePartitionsList)) {
        assertEquals("deleted partitions should be missing.", 0, store.get(entry.getKey()).size());
      }
    }
  }
  public void testUpdateClusterMetadata() {
    Cluster updatedCluster = ServerTestUtils.getLocalCluster(4);
    AdminClient client = getAdminClient();
    for (int i = 0; i < NUM_RUNS; i++) {
      VectorClock clock =
          ((VectorClock) client.getRemoteCluster(0).getVersion())
              .incremented(0, System.currentTimeMillis());
      client.updateRemoteCluster(0, updatedCluster, clock);

      assertEquals(
          "Cluster should match",
          updatedCluster,
          getVoldemortServer(0).getMetadataStore().getCluster());
      assertEquals(
          "AdminClient.getMetdata() should match",
          client.getRemoteCluster(0).getValue(),
          updatedCluster);

      // version should match
      assertEquals(
          "versions should match as well.", clock, client.getRemoteCluster(0).getVersion());
    }
  }
  private void doOperation(
      StreamOperations e, int nodeId, String storeName, List<Integer> partitionList) {
    switch (e) {
      case DELETE_PARTITIONS:
        putAlltoStore();
        getAdminClient().storeMntOps.deletePartitions(nodeId, storeName, partitionList, null);
        return;
      case FETCH_ENTRIES:
        putAlltoStore();
        consumeIterator(
            getAdminClient()
                .bulkFetchOps
                .fetchEntries(nodeId, storeName, partitionList, null, false));
        return;
      case FETCH_KEYS:
        putAlltoStore();
        consumeIterator(
            getAdminClient().bulkFetchOps.fetchKeys(nodeId, storeName, partitionList, null, false));
        return;
      case UPDATE_ENTRIES:
        getAdminClient()
            .streamingOps
            .updateEntries(
                nodeId,
                storeName,
                getRandomlyFailingIterator(ServerTestUtils.createRandomKeyValuePairs(TEST_KEYS)),
                null);
        return;
      case TRUNCATE_ENTRIES:
        putAlltoStore();
        getAdminClient().storeMntOps.truncate(nodeId, storeName);
        return;

      default:
        throw new RuntimeException("Unknown operation");
    }
  }
Exemple #29
0
  @Override
  @Before
  public void setUp() throws IOException {
    final int numServers = 1;
    this.nodeId = 0;
    this.time = SystemTime.INSTANCE;
    servers = new VoldemortServer[numServers];
    int partitionMap[][] = {{0, 1, 2, 3, 4, 5, 6, 7}};
    try {

      // Setup the cluster
      cluster =
          ServerTestUtils.startVoldemortCluster(
              numServers,
              servers,
              partitionMap,
              socketStoreFactory,
              true,
              null,
              storesXmlfile,
              new Properties());

    } catch (IOException e) {
      fail("Failure to setup the cluster");
    }

    socketUrl = servers[0].getIdentityNode().getSocketUrl().toString();
    List<String> bootstrapUrls = new ArrayList<String>();
    bootstrapUrls.add(socketUrl);

    // create a copy of the config file in a temp directory and work on that
    File src = new File(FAT_CLIENT_CONFIG_PATH_ORIGINAL);
    COPY_OF_FAT_CLIENT_CONFIG_FILE =
        new File(TestUtils.createTempDir(), "clientConfigs" + System.currentTimeMillis() + ".avro");
    FileUtils.copyFile(src, COPY_OF_FAT_CLIENT_CONFIG_FILE);

    // Setup the Coordinator
    CoordinatorConfig coordinatorConfig = new CoordinatorConfig();
    coordinatorConfig
        .setBootstrapURLs(bootstrapUrls)
        .setCoordinatorCoreThreads(100)
        .setCoordinatorMaxThreads(100)
        .setFatClientConfigPath(COPY_OF_FAT_CLIENT_CONFIG_FILE.getAbsolutePath())
        .setServerPort(9999);

    try {
      StoreClientConfigService storeClientConfigs = null;
      switch (coordinatorConfig.getFatClientConfigSource()) {
        case FILE:
          storeClientConfigs = new FileBasedStoreClientConfigService(coordinatorConfig);
          break;
        case ZOOKEEPER:
          throw new UnsupportedOperationException(
              "Zookeeper-based configs are not implemented yet!");
        default:
          storeClientConfigs = null;
      }
      coordinator = new CoordinatorProxyService(coordinatorConfig, storeClientConfigs);
      coordinator.start();
    } catch (Exception e) {
      e.printStackTrace();
      fail("Failure to start the Coordinator");
    }

    Properties props = new Properties();
    props.setProperty(ClientConfig.BOOTSTRAP_URLS_PROPERTY, "http://localhost:9999");
    props.setProperty(ClientConfig.ROUTING_TIMEOUT_MS_PROPERTY, "1500");

    RESTClientFactoryConfig mainConfig = new RESTClientFactoryConfig(props, null);
    RESTClientFactory factory = new RESTClientFactory(mainConfig);

    this.client = factory.getStoreClient(STORE_NAME);
  }
Exemple #30
0
 public AdminTest(String bootstrapUrl, String storeName) {
   this.storeName = storeName;
   this.adminClient = ServerTestUtils.getAdminClient(bootstrapUrl);
 }