@Override protected Cluster getCurrentCluster(int nodeId) { VoldemortServer server = serverMap.get(nodeId); if (server == null) { throw new VoldemortException("Node id " + nodeId + " does not exist"); } else { return server.getMetadataStore().getCluster(); } }
@After public void tearDown() { client.close(); for (VoldemortServer vs : this.vservers.values()) { vs.stop(); } for (SocketStoreFactory ssf : this.socketStoreFactories.values()) { ssf.close(); } ClusterTestUtils.reset(); }
@After public void teardown() { adminClient.close(); server.stop(); factory.close(); }
@Before public void setup() throws IOException { byte[] bytes1 = {(byte) 'A', (byte) 'B'}; byte[] bytes2 = {(byte) 'C', (byte) 'D'}; List<StoreDefinition> stores = ClusterTestUtils.getZZZ322StoreDefs("memory"); StoreDefinition storeDef = stores.get(0); cluster = ClusterTestUtils.getZZZCluster(); ClientConfig clientConfig = new ClientConfig(); clientConfig.setBootstrapUrls(cluster.getNodeById(0).getSocketUrl().toString()); clientConfig.getZoneAffinity().setEnableGetOpZoneAffinity(true); clientConfig.setClientZoneId(clientZoneId); SocketStoreClientFactory socketStoreClientFactory = new SocketStoreClientFactory(clientConfig); for (Integer nodeId : cluster.getNodeIds()) { SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024); VoldemortConfig config = ServerTestUtils.createServerConfigWithDefs( true, nodeId, TestUtils.createTempDir().getAbsolutePath(), cluster, stores, new Properties()); VoldemortServer vs = ServerTestUtils.startVoldemortServer(socketStoreFactory, config, cluster); vservers.put(nodeId, vs); socketStoreFactories.put(nodeId, socketStoreFactory); Store<ByteArray, byte[], byte[]> store = vs.getStoreRepository().getLocalStore(storeDef.getName()); Node node = cluster.getNodeById(nodeId); VectorClock version1 = new VectorClock(); version1.incrementVersion(0, System.currentTimeMillis()); VectorClock version2 = version1.incremented(0, System.currentTimeMillis()); if (node.getZoneId() == clientZoneId) { store.put(new ByteArray(bytes1), new Versioned<byte[]>(bytes1, version1), null); } else { store.put(new ByteArray(bytes1), new Versioned<byte[]>(bytes2, version2), null); } } client = socketStoreClientFactory.getRawStore(storeDef.getName(), null); }
public StatusServlet(VoldemortServer server, VelocityEngine engine) { this.server = Utils.notNull(server); this.velocityEngine = Utils.notNull(engine); this.abstractSocketService = (AbstractSocketService) server.getService(ServiceType.SOCKET); try { this.myMachine = InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { myMachine = "unknown"; } }
private void setFetcherClass(VoldemortServer server) { String className = server.getVoldemortConfig().getAllProps().getString("file.fetcher.class", null); if (className == null || className.trim().length() == 0) { this.fileFetcher = null; } else { try { logger.info("Loading fetcher " + className); Class<?> cls = Class.forName(className.trim()); this.fileFetcher = (FileFetcher) ReflectUtils.callConstructor( cls, new Class<?>[] {Props.class}, new Object[] {server.getVoldemortConfig().getAllProps()}); } catch (Exception e) { throw new VoldemortException("Error loading file fetcher class " + className, e); } } }
@Before public void setup() throws IOException { Properties props = new Properties(); props.put("enable.quota.limiting", "true"); server = ServerTestUtils.startStandAloneVoldemortServer( props, "test/common/voldemort/config/single-store.xml"); adminClient = new AdminClient( server.getMetadataStore().getCluster(), new AdminClientConfig(), new ClientConfig()); String bootStrapUrl = "tcp://" + server.getIdentityNode().getHost() + ":" + server.getIdentityNode().getSocketPort(); factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(bootStrapUrl)); storeClient = factory.getStoreClient("test"); }
@Override protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { String storeName = request.getParameter("store"); String format = request.getParameter("format"); if ("json".equals(format)) { outputJSON(response); return; } else { response.setContentType("text/html"); long refreshTime = 600; String refresh = request.getParameter("refresh"); if (refresh != null) { try { refreshTime = Integer.parseInt(refresh); } catch (NumberFormatException e) { response.sendError(HttpServletResponse.SC_BAD_REQUEST); } } List<Store<ByteArray, byte[], byte[]>> stores = null; if (storeName == null) { stores = server.getStoreRepository().getAllLocalStores(); } else { stores = Collections.singletonList(server.getStoreRepository().getLocalStore(storeName)); } Map<String, Object> params = Maps.newHashMap(); params.put("status", abstractSocketService.getStatusManager()); params.put("counters", Tracked.values()); params.put("stores", stores); params.put("refresh", refreshTime); velocityEngine.render("status.vm", params, response.getOutputStream()); } }
@Test public void testOnePartitionEndToEndBasedOnVersion() throws Exception { long now = System.currentTimeMillis(); // setup four nodes with one store and one partition final SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 32 * 1024); VoldemortServer[] servers = new VoldemortServer[4]; int partitionMap[][] = {{0}, {1}, {2}, {3}}; Cluster cluster = ServerTestUtils.startVoldemortCluster( 4, servers, partitionMap, socketStoreFactory, true, null, STORES_XML, new Properties()); Node node = cluster.getNodeById(0); String bootstrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort(); AdminClient adminClient = new AdminClient(bootstrapUrl); byte[] value = {1, 2, 3, 4, 5, 6, 7, 8, 9}; byte[] value2 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; // make versions VectorClock vc1 = new VectorClock(); VectorClock vc2 = new VectorClock(); VectorClock vc3 = new VectorClock(); vc1.incrementVersion(0, now); // [0:1] vc2.incrementVersion(1, now - 5000); // [1:1] vc3.incrementVersion(0, now - 89000000); // [0:1], over a day old ArrayList<Pair<ByteArray, Versioned<byte[]>>> n0store = new ArrayList<Pair<ByteArray, Versioned<byte[]>>>(); ArrayList<Pair<ByteArray, Versioned<byte[]>>> n1store = new ArrayList<Pair<ByteArray, Versioned<byte[]>>>(); ArrayList<Pair<ByteArray, Versioned<byte[]>>> n2store = new ArrayList<Pair<ByteArray, Versioned<byte[]>>>(); ArrayList<Pair<ByteArray, Versioned<byte[]>>> n3store = new ArrayList<Pair<ByteArray, Versioned<byte[]>>>(); ArrayList<ByteArray> keysHashedToPar0 = new ArrayList<ByteArray>(); // find store Versioned<List<StoreDefinition>> storeDefinitions = adminClient.metadataMgmtOps.getRemoteStoreDefList(0); List<StoreDefinition> StoreDefinitions = storeDefinitions.getValue(); StoreDefinition storeDefinition = null; for (StoreDefinition def : StoreDefinitions) { if (def.getName().equals(STORE_NAME)) { storeDefinition = def; break; } } assertNotNull("No such store found: " + STORE_NAME, storeDefinition); RoutingStrategy router = new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster); while (keysHashedToPar0.size() < 7) { // generate random key Map<ByteArray, byte[]> map = ServerTestUtils.createRandomKeyValuePairs(1); ByteArray key = map.keySet().iterator().next(); key.get()[0] = (byte) keysHashedToPar0.size(); Integer masterPartition = router.getMasterPartition(key.get()); if (masterPartition == 0) { keysHashedToPar0.add(key); } else { continue; } } ByteArray k6 = keysHashedToPar0.get(6); ByteArray k5 = keysHashedToPar0.get(5); ByteArray k4 = keysHashedToPar0.get(4); ByteArray k3 = keysHashedToPar0.get(3); ByteArray k2 = keysHashedToPar0.get(2); ByteArray k1 = keysHashedToPar0.get(1); ByteArray k0 = keysHashedToPar0.get(0); // insert K6 into node 0,1,2 Versioned<byte[]> v6 = new Versioned<byte[]>(value, vc1); n0store.add(Pair.create(k6, v6)); n1store.add(Pair.create(k6, v6)); n2store.add(Pair.create(k6, v6)); // insert K6(conflicting value and version) into node 0,1,2,3 Versioned<byte[]> v6ConflictEarly = new Versioned<byte[]>(value2, vc2); n0store.add(Pair.create(k6, v6ConflictEarly)); n1store.add(Pair.create(k6, v6ConflictEarly)); n2store.add(Pair.create(k6, v6ConflictEarly)); n3store.add(Pair.create(k6, v6ConflictEarly)); // insert K4,K5 into four nodes Versioned<byte[]> v5 = new Versioned<byte[]>(value, vc1); Versioned<byte[]> v4 = new Versioned<byte[]>(value, vc1); n0store.add(Pair.create(k5, v5)); n1store.add(Pair.create(k5, v5)); n2store.add(Pair.create(k5, v5)); n3store.add(Pair.create(k5, v5)); n0store.add(Pair.create(k4, v4)); n1store.add(Pair.create(k4, v4)); n2store.add(Pair.create(k4, v4)); n3store.add(Pair.create(k4, v4)); // insert K3 into node 0,1,2 Versioned<byte[]> v3 = new Versioned<byte[]>(value, vc2); n0store.add(Pair.create(k3, v3)); n1store.add(Pair.create(k3, v3)); n2store.add(Pair.create(k3, v3)); // insert K3(conflicting but latest version) into node 0,1,2,3 Versioned<byte[]> v3ConflictLate = new Versioned<byte[]>(value, vc1); n0store.add(Pair.create(k3, v3ConflictLate)); n1store.add(Pair.create(k3, v3ConflictLate)); n2store.add(Pair.create(k3, v3ConflictLate)); n3store.add(Pair.create(k3, v3ConflictLate)); // insert K2 into node 0,1 Versioned<byte[]> v2 = new Versioned<byte[]>(value, vc1); n0store.add(Pair.create(k2, v2)); n1store.add(Pair.create(k2, v2)); // insert K1 into node 0 Versioned<byte[]> v1 = new Versioned<byte[]>(value, vc1); n0store.add(Pair.create(k1, v1)); // insert K0(out of retention) into node 0,1,2 Versioned<byte[]> v0 = new Versioned<byte[]>(value, vc3); n0store.add(Pair.create(k0, v0)); n1store.add(Pair.create(k0, v0)); n2store.add(Pair.create(k0, v0)); // stream to store adminClient.streamingOps.updateEntries(0, STORE_NAME, n0store.iterator(), null); adminClient.streamingOps.updateEntries(1, STORE_NAME, n1store.iterator(), null); adminClient.streamingOps.updateEntries(2, STORE_NAME, n2store.iterator(), null); adminClient.streamingOps.updateEntries(3, STORE_NAME, n3store.iterator(), null); // should have FULL:2(K4,K5), LATEST_CONSISTENT:1(K3), // INCONSISTENT:2(K6,K2), ignored(K1,K0) List<String> urls = new ArrayList<String>(); urls.add(bootstrapUrl); ConsistencyCheck.ComparisonType[] comparisonTypes = ConsistencyCheck.ComparisonType.values(); for (ConsistencyCheck.ComparisonType type : comparisonTypes) { StringWriter sw = new StringWriter(); ConsistencyCheck checker = new ConsistencyCheck(urls, STORE_NAME, 0, sw, type); Reporter reporter = null; checker.connect(); reporter = checker.execute(); assertEquals(7 - 2, reporter.numTotalKeys); assertEquals(3, reporter.numGoodKeys); } for (VoldemortServer vs : servers) { vs.stop(); } }
protected void outputJSON(HttpServletResponse response) { StringBuilder sb = new StringBuilder("{\n"); sb.append(" \"servertime\": \""); sb.append(new Date()); sb.append("\","); sb.append("\n \"server\": \""); sb.append(myMachine); sb.append("\","); sb.append("\n \"node\": \""); sb.append(server.getMetadataStore().getNodeId()); sb.append("\","); sb.append("\n \"uptime\": \""); sb.append(abstractSocketService.getStatusManager().getFormattedUptime()); sb.append("\","); sb.append("\n \"num_workers\": "); sb.append(abstractSocketService.getStatusManager().getActiveWorkersCount()); sb.append(","); sb.append("\n \"pool_size\": "); sb.append(abstractSocketService.getStatusManager().getWorkerPoolSize()); sb.append(","); sb.append("\n \"stores\": {"); int i = 0; for (Store<ByteArray, byte[], byte[]> store : server.getStoreRepository().getAllLocalStores()) { if (i++ > 0) { sb.append(","); } sb.append("\n \""); sb.append(store.getName()); sb.append("\" : {\n"); if (store instanceof StatTrackingStore) { StatTrackingStore statStore = (StatTrackingStore) store; Map<Tracked, RequestCounter> stats = statStore.getStats().getCounters(); for (Tracked t : Tracked.values()) { if (t == Tracked.EXCEPTION) { continue; } sb.append(fillCommonStats(stats, t)); } sb.append(",\n \"num_exceptions\": "); sb.append(statStore.getStats().getCount(Tracked.EXCEPTION)); sb.append("\n"); sb.append(" }"); } if (store instanceof QuotaLimitingStore) { QuotaLimitingStore quotaStore = (QuotaLimitingStore) store; Map<Tracked, RequestCounter> stats = quotaStore.getStats().getCounters(); for (Tracked t : Tracked.values()) { if (t == Tracked.EXCEPTION) { continue; } sb.append(fillCommonStats(stats, t)); } sb.append(",\n \"num_exceptions\": "); sb.append(quotaStore.getStats().getCount(Tracked.EXCEPTION)); sb.append("\n"); sb.append(" }"); } } sb.append("\n }\n"); sb.append("}\n"); try { response.setContentType("text/plain"); OutputStreamWriter writer = new OutputStreamWriter(response.getOutputStream()); writer.write(sb.toString()); writer.flush(); } catch (Exception e) { throw new VoldemortException(e); } }