예제 #1
0
 @Override
 protected Cluster getCurrentCluster(int nodeId) {
   VoldemortServer server = serverMap.get(nodeId);
   if (server == null) {
     throw new VoldemortException("Node id " + nodeId + " does not exist");
   } else {
     return server.getMetadataStore().getCluster();
   }
 }
 @After
 public void tearDown() {
   client.close();
   for (VoldemortServer vs : this.vservers.values()) {
     vs.stop();
   }
   for (SocketStoreFactory ssf : this.socketStoreFactories.values()) {
     ssf.close();
   }
   ClusterTestUtils.reset();
 }
예제 #3
0
  @After
  public void teardown() {
    adminClient.close();
    server.stop();

    factory.close();
  }
  @Before
  public void setup() throws IOException {
    byte[] bytes1 = {(byte) 'A', (byte) 'B'};
    byte[] bytes2 = {(byte) 'C', (byte) 'D'};
    List<StoreDefinition> stores = ClusterTestUtils.getZZZ322StoreDefs("memory");
    StoreDefinition storeDef = stores.get(0);
    cluster = ClusterTestUtils.getZZZCluster();
    ClientConfig clientConfig = new ClientConfig();
    clientConfig.setBootstrapUrls(cluster.getNodeById(0).getSocketUrl().toString());
    clientConfig.getZoneAffinity().setEnableGetOpZoneAffinity(true);
    clientConfig.setClientZoneId(clientZoneId);
    SocketStoreClientFactory socketStoreClientFactory = new SocketStoreClientFactory(clientConfig);
    for (Integer nodeId : cluster.getNodeIds()) {
      SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
      VoldemortConfig config =
          ServerTestUtils.createServerConfigWithDefs(
              true,
              nodeId,
              TestUtils.createTempDir().getAbsolutePath(),
              cluster,
              stores,
              new Properties());
      VoldemortServer vs =
          ServerTestUtils.startVoldemortServer(socketStoreFactory, config, cluster);
      vservers.put(nodeId, vs);
      socketStoreFactories.put(nodeId, socketStoreFactory);
      Store<ByteArray, byte[], byte[]> store =
          vs.getStoreRepository().getLocalStore(storeDef.getName());
      Node node = cluster.getNodeById(nodeId);

      VectorClock version1 = new VectorClock();
      version1.incrementVersion(0, System.currentTimeMillis());
      VectorClock version2 = version1.incremented(0, System.currentTimeMillis());

      if (node.getZoneId() == clientZoneId) {
        store.put(new ByteArray(bytes1), new Versioned<byte[]>(bytes1, version1), null);
      } else {
        store.put(new ByteArray(bytes1), new Versioned<byte[]>(bytes2, version2), null);
      }
    }

    client = socketStoreClientFactory.getRawStore(storeDef.getName(), null);
  }
예제 #5
0
 public StatusServlet(VoldemortServer server, VelocityEngine engine) {
   this.server = Utils.notNull(server);
   this.velocityEngine = Utils.notNull(engine);
   this.abstractSocketService = (AbstractSocketService) server.getService(ServiceType.SOCKET);
   try {
     this.myMachine = InetAddress.getLocalHost().getHostName();
   } catch (UnknownHostException e) {
     myMachine = "unknown";
   }
 }
예제 #6
0
 private void setFetcherClass(VoldemortServer server) {
   String className =
       server.getVoldemortConfig().getAllProps().getString("file.fetcher.class", null);
   if (className == null || className.trim().length() == 0) {
     this.fileFetcher = null;
   } else {
     try {
       logger.info("Loading fetcher " + className);
       Class<?> cls = Class.forName(className.trim());
       this.fileFetcher =
           (FileFetcher)
               ReflectUtils.callConstructor(
                   cls,
                   new Class<?>[] {Props.class},
                   new Object[] {server.getVoldemortConfig().getAllProps()});
     } catch (Exception e) {
       throw new VoldemortException("Error loading file fetcher class " + className, e);
     }
   }
 }
예제 #7
0
  @Before
  public void setup() throws IOException {

    Properties props = new Properties();
    props.put("enable.quota.limiting", "true");
    server =
        ServerTestUtils.startStandAloneVoldemortServer(
            props, "test/common/voldemort/config/single-store.xml");

    adminClient =
        new AdminClient(
            server.getMetadataStore().getCluster(), new AdminClientConfig(), new ClientConfig());
    String bootStrapUrl =
        "tcp://"
            + server.getIdentityNode().getHost()
            + ":"
            + server.getIdentityNode().getSocketPort();
    factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(bootStrapUrl));
    storeClient = factory.getStoreClient("test");
  }
예제 #8
0
  @Override
  protected void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {

    String storeName = request.getParameter("store");

    String format = request.getParameter("format");
    if ("json".equals(format)) {
      outputJSON(response);
      return;
    } else {
      response.setContentType("text/html");

      long refreshTime = 600;
      String refresh = request.getParameter("refresh");
      if (refresh != null) {
        try {
          refreshTime = Integer.parseInt(refresh);
        } catch (NumberFormatException e) {
          response.sendError(HttpServletResponse.SC_BAD_REQUEST);
        }
      }

      List<Store<ByteArray, byte[], byte[]>> stores = null;
      if (storeName == null) {
        stores = server.getStoreRepository().getAllLocalStores();
      } else {
        stores = Collections.singletonList(server.getStoreRepository().getLocalStore(storeName));
      }

      Map<String, Object> params = Maps.newHashMap();
      params.put("status", abstractSocketService.getStatusManager());
      params.put("counters", Tracked.values());
      params.put("stores", stores);
      params.put("refresh", refreshTime);
      velocityEngine.render("status.vm", params, response.getOutputStream());
    }
  }
예제 #9
0
  @Test
  public void testOnePartitionEndToEndBasedOnVersion() throws Exception {
    long now = System.currentTimeMillis();

    // setup four nodes with one store and one partition
    final SocketStoreFactory socketStoreFactory =
        new ClientRequestExecutorPool(2, 10000, 100000, 32 * 1024);
    VoldemortServer[] servers = new VoldemortServer[4];
    int partitionMap[][] = {{0}, {1}, {2}, {3}};
    Cluster cluster =
        ServerTestUtils.startVoldemortCluster(
            4, servers, partitionMap, socketStoreFactory, true, null, STORES_XML, new Properties());

    Node node = cluster.getNodeById(0);
    String bootstrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort();
    AdminClient adminClient = new AdminClient(bootstrapUrl);

    byte[] value = {1, 2, 3, 4, 5, 6, 7, 8, 9};
    byte[] value2 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};

    // make versions
    VectorClock vc1 = new VectorClock();
    VectorClock vc2 = new VectorClock();
    VectorClock vc3 = new VectorClock();
    vc1.incrementVersion(0, now); // [0:1]
    vc2.incrementVersion(1, now - 5000); // [1:1]
    vc3.incrementVersion(0, now - 89000000); // [0:1], over a day old

    ArrayList<Pair<ByteArray, Versioned<byte[]>>> n0store =
        new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
    ArrayList<Pair<ByteArray, Versioned<byte[]>>> n1store =
        new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
    ArrayList<Pair<ByteArray, Versioned<byte[]>>> n2store =
        new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
    ArrayList<Pair<ByteArray, Versioned<byte[]>>> n3store =
        new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
    ArrayList<ByteArray> keysHashedToPar0 = new ArrayList<ByteArray>();

    // find store
    Versioned<List<StoreDefinition>> storeDefinitions =
        adminClient.metadataMgmtOps.getRemoteStoreDefList(0);
    List<StoreDefinition> StoreDefinitions = storeDefinitions.getValue();
    StoreDefinition storeDefinition = null;
    for (StoreDefinition def : StoreDefinitions) {
      if (def.getName().equals(STORE_NAME)) {
        storeDefinition = def;
        break;
      }
    }
    assertNotNull("No such store found: " + STORE_NAME, storeDefinition);

    RoutingStrategy router =
        new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster);
    while (keysHashedToPar0.size() < 7) {
      // generate random key
      Map<ByteArray, byte[]> map = ServerTestUtils.createRandomKeyValuePairs(1);
      ByteArray key = map.keySet().iterator().next();
      key.get()[0] = (byte) keysHashedToPar0.size();
      Integer masterPartition = router.getMasterPartition(key.get());
      if (masterPartition == 0) {
        keysHashedToPar0.add(key);
      } else {
        continue;
      }
    }
    ByteArray k6 = keysHashedToPar0.get(6);
    ByteArray k5 = keysHashedToPar0.get(5);
    ByteArray k4 = keysHashedToPar0.get(4);
    ByteArray k3 = keysHashedToPar0.get(3);
    ByteArray k2 = keysHashedToPar0.get(2);
    ByteArray k1 = keysHashedToPar0.get(1);
    ByteArray k0 = keysHashedToPar0.get(0);

    // insert K6 into node 0,1,2
    Versioned<byte[]> v6 = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k6, v6));
    n1store.add(Pair.create(k6, v6));
    n2store.add(Pair.create(k6, v6));

    // insert K6(conflicting value and version) into node 0,1,2,3
    Versioned<byte[]> v6ConflictEarly = new Versioned<byte[]>(value2, vc2);
    n0store.add(Pair.create(k6, v6ConflictEarly));
    n1store.add(Pair.create(k6, v6ConflictEarly));
    n2store.add(Pair.create(k6, v6ConflictEarly));
    n3store.add(Pair.create(k6, v6ConflictEarly));

    // insert K4,K5 into four nodes
    Versioned<byte[]> v5 = new Versioned<byte[]>(value, vc1);
    Versioned<byte[]> v4 = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k5, v5));
    n1store.add(Pair.create(k5, v5));
    n2store.add(Pair.create(k5, v5));
    n3store.add(Pair.create(k5, v5));
    n0store.add(Pair.create(k4, v4));
    n1store.add(Pair.create(k4, v4));
    n2store.add(Pair.create(k4, v4));
    n3store.add(Pair.create(k4, v4));

    // insert K3 into node 0,1,2
    Versioned<byte[]> v3 = new Versioned<byte[]>(value, vc2);
    n0store.add(Pair.create(k3, v3));
    n1store.add(Pair.create(k3, v3));
    n2store.add(Pair.create(k3, v3));

    // insert K3(conflicting but latest version) into node 0,1,2,3
    Versioned<byte[]> v3ConflictLate = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k3, v3ConflictLate));
    n1store.add(Pair.create(k3, v3ConflictLate));
    n2store.add(Pair.create(k3, v3ConflictLate));
    n3store.add(Pair.create(k3, v3ConflictLate));

    // insert K2 into node 0,1
    Versioned<byte[]> v2 = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k2, v2));
    n1store.add(Pair.create(k2, v2));

    // insert K1 into node 0
    Versioned<byte[]> v1 = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k1, v1));

    // insert K0(out of retention) into node 0,1,2
    Versioned<byte[]> v0 = new Versioned<byte[]>(value, vc3);
    n0store.add(Pair.create(k0, v0));
    n1store.add(Pair.create(k0, v0));
    n2store.add(Pair.create(k0, v0));

    // stream to store
    adminClient.streamingOps.updateEntries(0, STORE_NAME, n0store.iterator(), null);
    adminClient.streamingOps.updateEntries(1, STORE_NAME, n1store.iterator(), null);
    adminClient.streamingOps.updateEntries(2, STORE_NAME, n2store.iterator(), null);
    adminClient.streamingOps.updateEntries(3, STORE_NAME, n3store.iterator(), null);

    // should have FULL:2(K4,K5), LATEST_CONSISTENT:1(K3),
    // INCONSISTENT:2(K6,K2), ignored(K1,K0)
    List<String> urls = new ArrayList<String>();
    urls.add(bootstrapUrl);
    ConsistencyCheck.ComparisonType[] comparisonTypes = ConsistencyCheck.ComparisonType.values();

    for (ConsistencyCheck.ComparisonType type : comparisonTypes) {
      StringWriter sw = new StringWriter();
      ConsistencyCheck checker = new ConsistencyCheck(urls, STORE_NAME, 0, sw, type);
      Reporter reporter = null;
      checker.connect();
      reporter = checker.execute();

      assertEquals(7 - 2, reporter.numTotalKeys);
      assertEquals(3, reporter.numGoodKeys);
    }

    for (VoldemortServer vs : servers) {
      vs.stop();
    }
  }
예제 #10
0
  protected void outputJSON(HttpServletResponse response) {
    StringBuilder sb = new StringBuilder("{\n");

    sb.append("  \"servertime\": \"");
    sb.append(new Date());
    sb.append("\",");

    sb.append("\n  \"server\": \"");
    sb.append(myMachine);
    sb.append("\",");

    sb.append("\n  \"node\": \"");
    sb.append(server.getMetadataStore().getNodeId());
    sb.append("\",");

    sb.append("\n  \"uptime\": \"");
    sb.append(abstractSocketService.getStatusManager().getFormattedUptime());
    sb.append("\",");

    sb.append("\n  \"num_workers\": ");
    sb.append(abstractSocketService.getStatusManager().getActiveWorkersCount());
    sb.append(",");

    sb.append("\n  \"pool_size\": ");
    sb.append(abstractSocketService.getStatusManager().getWorkerPoolSize());
    sb.append(",");

    sb.append("\n  \"stores\": {");

    int i = 0;
    for (Store<ByteArray, byte[], byte[]> store : server.getStoreRepository().getAllLocalStores()) {

      if (i++ > 0) {
        sb.append(",");
      }
      sb.append("\n    \"");
      sb.append(store.getName());
      sb.append("\" : {\n");

      if (store instanceof StatTrackingStore) {
        StatTrackingStore statStore = (StatTrackingStore) store;
        Map<Tracked, RequestCounter> stats = statStore.getStats().getCounters();
        for (Tracked t : Tracked.values()) {
          if (t == Tracked.EXCEPTION) {
            continue;
          }
          sb.append(fillCommonStats(stats, t));
        }
        sb.append(",\n        \"num_exceptions\": ");
        sb.append(statStore.getStats().getCount(Tracked.EXCEPTION));
        sb.append("\n");
        sb.append("    }");
      }

      if (store instanceof QuotaLimitingStore) {
        QuotaLimitingStore quotaStore = (QuotaLimitingStore) store;
        Map<Tracked, RequestCounter> stats = quotaStore.getStats().getCounters();
        for (Tracked t : Tracked.values()) {
          if (t == Tracked.EXCEPTION) {
            continue;
          }
          sb.append(fillCommonStats(stats, t));
        }
        sb.append(",\n        \"num_exceptions\": ");
        sb.append(quotaStore.getStats().getCount(Tracked.EXCEPTION));
        sb.append("\n");
        sb.append("    }");
      }
    }

    sb.append("\n  }\n");
    sb.append("}\n");

    try {
      response.setContentType("text/plain");
      OutputStreamWriter writer = new OutputStreamWriter(response.getOutputStream());
      writer.write(sb.toString());
      writer.flush();
    } catch (Exception e) {
      throw new VoldemortException(e);
    }
  }