@Before
public void setup() throws IOException {
byte[] bytes1 = {(byte) 'A', (byte) 'B'};
byte[] bytes2 = {(byte) 'C', (byte) 'D'};
List<StoreDefinition> stores = ClusterTestUtils.getZZZ322StoreDefs("memory");
StoreDefinition storeDef = stores.get(0);
cluster = ClusterTestUtils.getZZZCluster();
ClientConfig clientConfig = new ClientConfig();
clientConfig.setBootstrapUrls(cluster.getNodeById(0).getSocketUrl().toString());
clientConfig.getZoneAffinity().setEnableGetOpZoneAffinity(true);
clientConfig.setClientZoneId(clientZoneId);
SocketStoreClientFactory socketStoreClientFactory = new SocketStoreClientFactory(clientConfig);
for (Integer nodeId : cluster.getNodeIds()) {
SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
VoldemortConfig config =
ServerTestUtils.createServerConfigWithDefs(
true,
nodeId,
TestUtils.createTempDir().getAbsolutePath(),
cluster,
stores,
new Properties());
VoldemortServer vs =
ServerTestUtils.startVoldemortServer(socketStoreFactory, config, cluster);
vservers.put(nodeId, vs);
socketStoreFactories.put(nodeId, socketStoreFactory);
Store<ByteArray, byte[], byte[]> store =
vs.getStoreRepository().getLocalStore(storeDef.getName());
Node node = cluster.getNodeById(nodeId);
VectorClock version1 = new VectorClock();
version1.incrementVersion(0, System.currentTimeMillis());
VectorClock version2 = version1.incremented(0, System.currentTimeMillis());
if (node.getZoneId() == clientZoneId) {
store.put(new ByteArray(bytes1), new Versioned<byte[]>(bytes1, version1), null);
} else {
store.put(new ByteArray(bytes1), new Versioned<byte[]>(bytes2, version2), null);
}
}
client = socketStoreClientFactory.getRawStore(storeDef.getName(), null);
}
@Override
@Before
public void setUp() throws IOException {
cluster = ServerTestUtils.getLocalCluster(2, new int[][] {{0, 1, 2, 3}, {4, 5, 6, 7}});
List<StoreDefinition> storeDefs = ServerTestUtils.getStoreDefs(1);
failingStorageEngine =
new RandomlyFailingDelegatingStore<ByteArray, byte[], byte[]>(
new InMemoryStorageEngine<ByteArray, byte[], byte[]>(storeDefs.get(0).getName()));
adminServer = getAdminServer(cluster.getNodeById(0), cluster, storeDefs, failingStorageEngine);
adminClient = ServerTestUtils.getAdminClient(cluster);
adminServer.start();
}
private void reviveNodes(Set<Integer> failedNodes) {
for (int node : failedNodes) {
ForceFailStore<ByteArray, byte[], byte[]> forceFailStore = getForceFailStore(node);
forceFailStore.setFail(false);
if (logger.isTraceEnabled()) logger.trace("Stopped failing requests to " + node);
}
while (!failedNodes.isEmpty()) {
for (int node : failedNodes)
if (failureDetector.isAvailable(cluster.getNodeById(node))) failedNodes.remove(node);
}
}
@Before
public void setUp() throws Exception {
final int numServers = 2;
servers = new VoldemortServer[numServers];
int partitionMap[][] = {{0, 1, 2, 3}, {4, 5, 6, 7}};
cluster =
ServerTestUtils.startVoldemortCluster(
numServers,
servers,
partitionMap,
socketStoreFactory,
true, // useNio
null,
storesXmlfile,
new Properties());
socketUrl = servers[0].getIdentityNode().getSocketUrl().toString();
bootStrapUrls = new String[1];
bootStrapUrls[0] = socketUrl;
Node node = cluster.getNodeById(0);
String bootstrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort();
ClientConfig clientConfig = new ClientConfig();
clientConfig.setClientRegistryUpdateIntervalInSecs(5);
clientConfig.setAsyncMetadataRefreshInMs(5000);
clientConfig.setBootstrapUrls(bootstrapUrl);
SocketStoreClientFactory storeClientFactory = new SocketStoreClientFactory(clientConfig);
storeClient =
new ZenStoreClient<String, String>(
STORE_NAME,
null,
storeClientFactory,
3,
clientConfig.getClientContextName(),
0,
clientConfig);
SystemStoreClientFactory<String, String> systemStoreFactory =
new SystemStoreClientFactory<String, String>(clientConfig);
sysStoreVersion =
systemStoreFactory.createSystemStore(
SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name());
clientRegistryStore =
systemStoreFactory.createSystemStore(
SystemStoreConstants.SystemStoreName.voldsys$_client_registry.name());
}
@Override
@Before
public void setUp() {
logger.info(" Initial SEED used for random number generator: " + TestUtils.SEED);
final int numServers = 1;
this.nodeId = 0;
servers = new VoldemortServer[numServers];
try {
// Setup the cluster
Properties props = new Properties();
props.setProperty("rest.enable", "true");
props.setProperty("http.enable", "true");
Cluster customCluster = clusterMapper.readCluster(new FileReader(clusterXmlFile), false);
cluster =
ServerTestUtils.startVoldemortCluster(
servers, null, clusterXmlFile, storesXmlfile, props, customCluster);
} catch (IOException e) {
fail("Failure to setup the cluster");
}
// Creating R2Store
RESTClientConfig restClientConfig = new RESTClientConfig();
restClientConfig
.setHttpBootstrapURL("http://localhost:" + cluster.getNodeById(0).getRestPort())
.setTimeoutMs(10000, TimeUnit.MILLISECONDS)
.setMaxR2ConnectionPoolSize(100);
clientFactory = new HttpClientFactory();
Map<String, String> properties = new HashMap<String, String>();
properties.put(
HttpClientFactory.POOL_SIZE_KEY,
Integer.toString(restClientConfig.getMaxR2ConnectionPoolSize()));
TransportClient transportClient = clientFactory.getClient(properties);
R2Store r2Store =
new R2Store(
STORE_NAME,
restClientConfig.getHttpBootstrapURL(),
"0",
transportClient,
restClientConfig,
0);
store = r2Store;
}
@Test
public void testAddStore() throws Exception {
AdminClient adminClient = getAdminClient();
StoreDefinition definition =
new StoreDefinitionBuilder()
.setName("updateTest")
.setType(InMemoryStorageConfiguration.TYPE_NAME)
.setKeySerializer(new SerializerDefinition("string"))
.setValueSerializer(new SerializerDefinition("string"))
.setRoutingPolicy(RoutingTier.CLIENT)
.setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY)
.setReplicationFactor(1)
.setPreferredReads(1)
.setRequiredReads(1)
.setPreferredWrites(1)
.setRequiredWrites(1)
.build();
adminClient.addStore(definition);
// now test the store
StoreClientFactory factory =
new SocketStoreClientFactory(
new ClientConfig().setBootstrapUrls(cluster.getNodeById(0).getSocketUrl().toString()));
StoreClient<Object, Object> client = factory.getStoreClient("updateTest");
client.put("abc", "123");
String s = (String) client.get("abc").getValue();
assertEquals(s, "123");
// test again with a unknown store
try {
client = factory.getStoreClient("updateTest2");
client.put("abc", "123");
s = (String) client.get("abc").getValue();
assertEquals(s, "123");
fail("Should have received bootstrap failure exception");
} catch (Exception e) {
if (!(e instanceof BootstrapFailureException)) throw e;
}
// make sure that the store list we get back from AdminClient
Versioned<List<StoreDefinition>> list = adminClient.getRemoteStoreDefList(0);
assertTrue(list.getValue().contains(definition));
}
private List<Integer> getPartitions(int nodeId) {
Cluster cluster = adminClient.getAdminClientCluster();
Node node = cluster.getNodeById(nodeId);
return node.getPartitionIds();
}
@Test
public void testOnePartitionEndToEndBasedOnVersion() throws Exception {
long now = System.currentTimeMillis();
// setup four nodes with one store and one partition
final SocketStoreFactory socketStoreFactory =
new ClientRequestExecutorPool(2, 10000, 100000, 32 * 1024);
VoldemortServer[] servers = new VoldemortServer[4];
int partitionMap[][] = {{0}, {1}, {2}, {3}};
Cluster cluster =
ServerTestUtils.startVoldemortCluster(
4, servers, partitionMap, socketStoreFactory, true, null, STORES_XML, new Properties());
Node node = cluster.getNodeById(0);
String bootstrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort();
AdminClient adminClient = new AdminClient(bootstrapUrl);
byte[] value = {1, 2, 3, 4, 5, 6, 7, 8, 9};
byte[] value2 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
// make versions
VectorClock vc1 = new VectorClock();
VectorClock vc2 = new VectorClock();
VectorClock vc3 = new VectorClock();
vc1.incrementVersion(0, now); // [0:1]
vc2.incrementVersion(1, now - 5000); // [1:1]
vc3.incrementVersion(0, now - 89000000); // [0:1], over a day old
ArrayList<Pair<ByteArray, Versioned<byte[]>>> n0store =
new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
ArrayList<Pair<ByteArray, Versioned<byte[]>>> n1store =
new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
ArrayList<Pair<ByteArray, Versioned<byte[]>>> n2store =
new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
ArrayList<Pair<ByteArray, Versioned<byte[]>>> n3store =
new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
ArrayList<ByteArray> keysHashedToPar0 = new ArrayList<ByteArray>();
// find store
Versioned<List<StoreDefinition>> storeDefinitions =
adminClient.metadataMgmtOps.getRemoteStoreDefList(0);
List<StoreDefinition> StoreDefinitions = storeDefinitions.getValue();
StoreDefinition storeDefinition = null;
for (StoreDefinition def : StoreDefinitions) {
if (def.getName().equals(STORE_NAME)) {
storeDefinition = def;
break;
}
}
assertNotNull("No such store found: " + STORE_NAME, storeDefinition);
RoutingStrategy router =
new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster);
while (keysHashedToPar0.size() < 7) {
// generate random key
Map<ByteArray, byte[]> map = ServerTestUtils.createRandomKeyValuePairs(1);
ByteArray key = map.keySet().iterator().next();
key.get()[0] = (byte) keysHashedToPar0.size();
Integer masterPartition = router.getMasterPartition(key.get());
if (masterPartition == 0) {
keysHashedToPar0.add(key);
} else {
continue;
}
}
ByteArray k6 = keysHashedToPar0.get(6);
ByteArray k5 = keysHashedToPar0.get(5);
ByteArray k4 = keysHashedToPar0.get(4);
ByteArray k3 = keysHashedToPar0.get(3);
ByteArray k2 = keysHashedToPar0.get(2);
ByteArray k1 = keysHashedToPar0.get(1);
ByteArray k0 = keysHashedToPar0.get(0);
// insert K6 into node 0,1,2
Versioned<byte[]> v6 = new Versioned<byte[]>(value, vc1);
n0store.add(Pair.create(k6, v6));
n1store.add(Pair.create(k6, v6));
n2store.add(Pair.create(k6, v6));
// insert K6(conflicting value and version) into node 0,1,2,3
Versioned<byte[]> v6ConflictEarly = new Versioned<byte[]>(value2, vc2);
n0store.add(Pair.create(k6, v6ConflictEarly));
n1store.add(Pair.create(k6, v6ConflictEarly));
n2store.add(Pair.create(k6, v6ConflictEarly));
n3store.add(Pair.create(k6, v6ConflictEarly));
// insert K4,K5 into four nodes
Versioned<byte[]> v5 = new Versioned<byte[]>(value, vc1);
Versioned<byte[]> v4 = new Versioned<byte[]>(value, vc1);
n0store.add(Pair.create(k5, v5));
n1store.add(Pair.create(k5, v5));
n2store.add(Pair.create(k5, v5));
n3store.add(Pair.create(k5, v5));
n0store.add(Pair.create(k4, v4));
n1store.add(Pair.create(k4, v4));
n2store.add(Pair.create(k4, v4));
n3store.add(Pair.create(k4, v4));
// insert K3 into node 0,1,2
Versioned<byte[]> v3 = new Versioned<byte[]>(value, vc2);
n0store.add(Pair.create(k3, v3));
n1store.add(Pair.create(k3, v3));
n2store.add(Pair.create(k3, v3));
// insert K3(conflicting but latest version) into node 0,1,2,3
Versioned<byte[]> v3ConflictLate = new Versioned<byte[]>(value, vc1);
n0store.add(Pair.create(k3, v3ConflictLate));
n1store.add(Pair.create(k3, v3ConflictLate));
n2store.add(Pair.create(k3, v3ConflictLate));
n3store.add(Pair.create(k3, v3ConflictLate));
// insert K2 into node 0,1
Versioned<byte[]> v2 = new Versioned<byte[]>(value, vc1);
n0store.add(Pair.create(k2, v2));
n1store.add(Pair.create(k2, v2));
// insert K1 into node 0
Versioned<byte[]> v1 = new Versioned<byte[]>(value, vc1);
n0store.add(Pair.create(k1, v1));
// insert K0(out of retention) into node 0,1,2
Versioned<byte[]> v0 = new Versioned<byte[]>(value, vc3);
n0store.add(Pair.create(k0, v0));
n1store.add(Pair.create(k0, v0));
n2store.add(Pair.create(k0, v0));
// stream to store
adminClient.streamingOps.updateEntries(0, STORE_NAME, n0store.iterator(), null);
adminClient.streamingOps.updateEntries(1, STORE_NAME, n1store.iterator(), null);
adminClient.streamingOps.updateEntries(2, STORE_NAME, n2store.iterator(), null);
adminClient.streamingOps.updateEntries(3, STORE_NAME, n3store.iterator(), null);
// should have FULL:2(K4,K5), LATEST_CONSISTENT:1(K3),
// INCONSISTENT:2(K6,K2), ignored(K1,K0)
List<String> urls = new ArrayList<String>();
urls.add(bootstrapUrl);
ConsistencyCheck.ComparisonType[] comparisonTypes = ConsistencyCheck.ComparisonType.values();
for (ConsistencyCheck.ComparisonType type : comparisonTypes) {
StringWriter sw = new StringWriter();
ConsistencyCheck checker = new ConsistencyCheck(urls, STORE_NAME, 0, sw, type);
Reporter reporter = null;
checker.connect();
reporter = checker.execute();
assertEquals(7 - 2, reporter.numTotalKeys);
assertEquals(3, reporter.numGoodKeys);
}
for (VoldemortServer vs : servers) {
vs.stop();
}
}
public void run() {
// don't try to run slop pusher job when rebalancing
if (metadataStore
.getServerState()
.equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)) {
logger.error("Cannot run slop pusher job since Voldemort server is rebalancing");
return;
}
boolean terminatedEarly = false;
Date startTime = new Date();
logger.info("Started streaming slop pusher job at " + startTime);
SlopStorageEngine slopStorageEngine = storeRepo.getSlopStore();
ClosableIterator<Pair<ByteArray, Versioned<Slop>>> iterator = null;
if (adminClient == null) {
adminClient =
new AdminClient(
cluster,
new AdminClientConfig()
.setMaxThreads(cluster.getNumberOfNodes())
.setMaxConnectionsPerNode(1));
}
if (voldemortConfig.getSlopZonesDownToTerminate() > 0) {
// Populating the zone mapping for early termination
zoneMapping.clear();
for (Node n : cluster.getNodes()) {
if (failureDetector.isAvailable(n)) {
Set<Integer> nodes = zoneMapping.get(n.getZoneId());
if (nodes == null) {
nodes = Sets.newHashSet();
zoneMapping.put(n.getZoneId(), nodes);
}
nodes.add(n.getId());
}
}
// Check how many zones are down
int zonesDown = 0;
for (Zone zone : cluster.getZones()) {
if (zoneMapping.get(zone.getId()) == null || zoneMapping.get(zone.getId()).size() == 0)
zonesDown++;
}
// Terminate early
if (voldemortConfig.getSlopZonesDownToTerminate() <= zoneMapping.size()
&& zonesDown >= voldemortConfig.getSlopZonesDownToTerminate()) {
logger.info(
"Completed streaming slop pusher job at "
+ startTime
+ " early because "
+ zonesDown
+ " zones are down");
stopAdminClient();
return;
}
}
// Clearing the statistics
AtomicLong attemptedPushes = new AtomicLong(0);
for (Node node : cluster.getNodes()) {
attemptedByNode.put(node.getId(), 0L);
succeededByNode.put(node.getId(), 0L);
}
acquireRepairPermit();
try {
StorageEngine<ByteArray, Slop, byte[]> slopStore = slopStorageEngine.asSlopStore();
iterator = slopStore.entries();
while (iterator.hasNext()) {
Pair<ByteArray, Versioned<Slop>> keyAndVal;
try {
keyAndVal = iterator.next();
Versioned<Slop> versioned = keyAndVal.getSecond();
// Retrieve the node
int nodeId = versioned.getValue().getNodeId();
Node node = cluster.getNodeById(nodeId);
attemptedPushes.incrementAndGet();
Long attempted = attemptedByNode.get(nodeId);
attemptedByNode.put(nodeId, attempted + 1L);
if (attemptedPushes.get() % 10000 == 0)
logger.info("Attempted pushing " + attemptedPushes + " slops");
if (logger.isTraceEnabled())
logger.trace(
"Pushing slop for "
+ versioned.getValue().getNodeId()
+ " and store "
+ versioned.getValue().getStoreName());
if (failureDetector.isAvailable(node)) {
SynchronousQueue<Versioned<Slop>> slopQueue = slopQueues.get(nodeId);
if (slopQueue == null) {
// No previous slop queue, add one
slopQueue = new SynchronousQueue<Versioned<Slop>>();
slopQueues.put(nodeId, slopQueue);
consumerResults.add(
consumerExecutor.submit(new SlopConsumer(nodeId, slopQueue, slopStorageEngine)));
}
boolean offered =
slopQueue.offer(
versioned, voldemortConfig.getClientRoutingTimeoutMs(), TimeUnit.MILLISECONDS);
if (!offered) {
if (logger.isDebugEnabled())
logger.debug(
"No consumer appeared for slop in "
+ voldemortConfig.getClientConnectionTimeoutMs()
+ " ms");
}
readThrottler.maybeThrottle(nBytesRead(keyAndVal));
} else {
logger.trace(node + " declared down, won't push slop");
}
} catch (RejectedExecutionException e) {
throw new VoldemortException("Ran out of threads in executor", e);
}
}
} catch (InterruptedException e) {
logger.warn("Interrupted exception", e);
terminatedEarly = true;
} catch (Exception e) {
logger.error(e, e);
terminatedEarly = true;
} finally {
try {
if (iterator != null) iterator.close();
} catch (Exception e) {
logger.warn("Failed to close iterator cleanly as database might be closed", e);
}
// Adding the poison pill
for (SynchronousQueue<Versioned<Slop>> slopQueue : slopQueues.values()) {
try {
slopQueue.put(END);
} catch (InterruptedException e) {
logger.warn("Error putting poison pill", e);
}
}
for (Future result : consumerResults) {
try {
result.get();
} catch (Exception e) {
logger.warn("Exception in consumer", e);
}
}
// Only if exception didn't take place do we update the counts
if (!terminatedEarly) {
Map<Integer, Long> outstanding =
Maps.newHashMapWithExpectedSize(cluster.getNumberOfNodes());
for (int nodeId : succeededByNode.keySet()) {
logger.info(
"Slops to node "
+ nodeId
+ " - Succeeded - "
+ succeededByNode.get(nodeId)
+ " - Attempted - "
+ attemptedByNode.get(nodeId));
outstanding.put(nodeId, attemptedByNode.get(nodeId) - succeededByNode.get(nodeId));
}
slopStorageEngine.resetStats(outstanding);
logger.info("Completed streaming slop pusher job which started at " + startTime);
} else {
for (int nodeId : succeededByNode.keySet()) {
logger.info(
"Slops to node "
+ nodeId
+ " - Succeeded - "
+ succeededByNode.get(nodeId)
+ " - Attempted - "
+ attemptedByNode.get(nodeId));
}
logger.info("Completed early streaming slop pusher job which started at " + startTime);
}
// Shut down admin client as not to waste connections
consumerResults.clear();
slopQueues.clear();
stopAdminClient();
this.repairPermits.release();
}
}
