protected Set<String> commonMocks(int liveNodesCount) throws Exception {
shardHandlerFactoryMock.getShardHandler();
expectLastCall()
.andAnswer(
new IAnswer<ShardHandler>() {
@Override
public ShardHandler answer() throws Throwable {
log.info("SHARDHANDLER");
return shardHandlerMock;
}
})
.anyTimes();
workQueueMock.peekTopN(EasyMock.anyInt(), anyObject(Set.class), EasyMock.anyLong());
expectLastCall()
.andAnswer(
new IAnswer<List>() {
@Override
public List answer() throws Throwable {
Object result;
int count = 0;
while ((result = queue.peek()) == null) {
Thread.sleep(1000);
count++;
if (count > 1) return null;
}
return Arrays.asList(result);
}
})
.anyTimes();
workQueueMock.getTailId();
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
Object result = null;
Iterator iter = queue.iterator();
while (iter.hasNext()) {
result = iter.next();
}
return result == null ? null : ((QueueEvent) result).getId();
}
})
.anyTimes();
workQueueMock.peek(true);
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
Object result;
while ((result = queue.peek()) == null) {
Thread.sleep(1000);
}
return result;
}
})
.anyTimes();
workQueueMock.remove(anyObject(QueueEvent.class));
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
queue.remove((QueueEvent) getCurrentArguments()[0]);
return null;
}
})
.anyTimes();
workQueueMock.poll();
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return queue.poll();
}
})
.anyTimes();
zkStateReaderMock.getClusterState();
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return clusterStateMock;
}
})
.anyTimes();
zkStateReaderMock.getZkClient();
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return solrZkClientMock;
}
})
.anyTimes();
zkStateReaderMock.updateClusterState(anyBoolean());
clusterStateMock.getCollections();
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return collectionsSet;
}
})
.anyTimes();
final Set<String> liveNodes = new HashSet<>();
for (int i = 0; i < liveNodesCount; i++) {
final String address = "localhost:" + (8963 + i) + "_solr";
liveNodes.add(address);
zkStateReaderMock.getBaseUrlForNodeName(address);
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
// This works as long as this test does not use a
// webapp context with an underscore in it
return address.replaceAll("_", "/");
}
})
.anyTimes();
}
zkStateReaderMock.getClusterProps();
expectLastCall()
.andAnswer(
new IAnswer<Map>() {
@Override
public Map answer() throws Throwable {
return new HashMap();
}
});
solrZkClientMock.getZkClientTimeout();
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return 30000;
}
})
.anyTimes();
clusterStateMock.hasCollection(anyObject(String.class));
expectLastCall()
.andAnswer(
new IAnswer<Boolean>() {
@Override
public Boolean answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
return collectionsSet.contains(key);
}
})
.anyTimes();
clusterStateMock.getLiveNodes();
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return liveNodes;
}
})
.anyTimes();
solrZkClientMock.create(
anyObject(String.class),
anyObject(byte[].class),
anyObject(CreateMode.class),
anyBoolean());
expectLastCall()
.andAnswer(
new IAnswer<String>() {
@Override
public String answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
zkMap.put(key, null);
handleCreateCollMessage((byte[]) getCurrentArguments()[1]);
return key;
}
})
.anyTimes();
solrZkClientMock.makePath(anyObject(String.class), anyObject(byte[].class), anyBoolean());
expectLastCall()
.andAnswer(
new IAnswer<String>() {
@Override
public String answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
return key;
}
})
.anyTimes();
solrZkClientMock.makePath(
anyObject(String.class),
anyObject(byte[].class),
anyObject(CreateMode.class),
anyBoolean());
expectLastCall()
.andAnswer(
new IAnswer<String>() {
@Override
public String answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
return key;
}
})
.anyTimes();
solrZkClientMock.exists(anyObject(String.class), anyBoolean());
expectLastCall()
.andAnswer(
new IAnswer<Boolean>() {
@Override
public Boolean answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
return zkMap.containsKey(key);
}
})
.anyTimes();
zkMap.put("/configs/myconfig", null);
return liveNodes;
}
@Override
public void doTest() throws Exception {
boolean testsSuccesful = false;
try {
handle.clear();
handle.put("QTime", SKIPVAL);
handle.put("timestamp", SKIPVAL);
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
// make sure we have leaders for each shard
for (int j = 1; j < sliceCount; j++) {
zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
} // make sure we again have leaders for each shard
waitForRecoveriesToFinish(false);
// we cannot do delete by query
// as it's not supported for recovery
del("*:*");
List<StopableThread> threads = new ArrayList<StopableThread>();
int threadCount = 1;
int i = 0;
for (i = 0; i < threadCount; i++) {
StopableIndexingThread indexThread = new StopableIndexingThread(Integer.toString(i), true);
threads.add(indexThread);
indexThread.start();
}
threadCount = 1;
i = 0;
for (i = 0; i < threadCount; i++) {
StopableSearchThread searchThread = new StopableSearchThread();
threads.add(searchThread);
searchThread.start();
}
// TODO: we only do this sometimes so that we can sometimes compare against control,
// it's currently hard to know what requests failed when using ConcurrentSolrUpdateServer
boolean runFullThrottle = random().nextBoolean();
if (runFullThrottle) {
FullThrottleStopableIndexingThread ftIndexThread =
new FullThrottleStopableIndexingThread(clients, "ft1", true);
threads.add(ftIndexThread);
ftIndexThread.start();
}
chaosMonkey.startTheMonkey(true, 10000);
try {
long runLength;
if (RUN_LENGTH != -1) {
runLength = RUN_LENGTH;
} else {
int[] runTimes =
new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000};
runLength = runTimes[random().nextInt(runTimes.length - 1)];
}
Thread.sleep(runLength);
} finally {
chaosMonkey.stopTheMonkey();
}
for (StopableThread indexThread : threads) {
indexThread.safeStop();
}
// start any downed jetties to be sure we still will end up with a leader per shard...
// wait for stop...
for (StopableThread indexThread : threads) {
indexThread.join();
}
// try and wait for any replications and what not to finish...
Thread.sleep(2000);
// wait until there are no recoveries...
waitForThingsToLevelOut(Integer.MAX_VALUE); // Math.round((runLength / 1000.0f / 3.0f)));
// make sure we again have leaders for each shard
for (int j = 1; j < sliceCount; j++) {
zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 30000);
}
commit();
// TODO: assert we didnt kill everyone
zkStateReader.updateClusterState(true);
assertTrue(zkStateReader.getClusterState().getLiveNodes().size() > 0);
// we expect full throttle fails, but cloud client should not easily fail
for (StopableThread indexThread : threads) {
if (indexThread instanceof StopableIndexingThread
&& !(indexThread instanceof FullThrottleStopableIndexingThread)) {
assertFalse(
"There were too many update fails - we expect it can happen, but shouldn't easily",
((StopableIndexingThread) indexThread).getFailCount() > 10);
}
}
// full throttle thread can
// have request fails
checkShardConsistency(!runFullThrottle, true);
long ctrlDocs = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound();
// ensure we have added more than 0 docs
long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
assertTrue("Found " + ctrlDocs + " control docs", cloudClientDocs > 0);
if (VERBOSE)
System.out.println(
"control docs:"
+ controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()
+ "\n\n");
// try and make a collection to make sure the overseer has survived the expiration and session
// loss
// sometimes we restart zookeeper as well
if (random().nextBoolean()) {
zkServer.shutdown();
zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort());
zkServer.run();
}
CloudSolrServer client = createCloudClient("collection1");
try {
createCollection(null, "testcollection", 1, 1, 1, client, null, "conf1");
} finally {
client.shutdown();
}
List<Integer> numShardsNumReplicas = new ArrayList<Integer>(2);
numShardsNumReplicas.add(1);
numShardsNumReplicas.add(1);
checkForCollection("testcollection", numShardsNumReplicas, null);
testsSuccesful = true;
} finally {
if (!testsSuccesful) {
printLayout();
}
}
}