private void createRetrieveDocs(ResponseBuilder rb) {
// TODO: in a system with nTiers > 2, we could be passed "ids" here
// unless those requests always go to the final destination shard
// for each shard, collect the documents for that shard.
HashMap<String, Collection<ShardDoc>> shardMap = new HashMap<String, Collection<ShardDoc>>();
for (ShardDoc sdoc : rb.resultIds.values()) {
Collection<ShardDoc> shardDocs = shardMap.get(sdoc.shard);
if (shardDocs == null) {
shardDocs = new ArrayList<ShardDoc>();
shardMap.put(sdoc.shard, shardDocs);
}
shardDocs.add(sdoc);
}
SchemaField uniqueField = rb.req.getSchema().getUniqueKeyField();
// Now create a request for each shard to retrieve the stored fields
for (Collection<ShardDoc> shardDocs : shardMap.values()) {
ShardRequest sreq = new ShardRequest();
sreq.purpose = ShardRequest.PURPOSE_GET_FIELDS;
sreq.shards = new String[] {shardDocs.iterator().next().shard};
sreq.params = new ModifiableSolrParams();
// add original params
sreq.params.add(rb.req.getParams());
// no need for a sort, we already have order
sreq.params.remove(CommonParams.SORT);
// we already have the field sort values
sreq.params.remove(ResponseBuilder.FIELD_SORT_VALUES);
// make sure that the id is returned for correlation.
String fl = sreq.params.get(CommonParams.FL);
if (fl != null) {
fl = fl.trim();
// currently, "score" is synonymous with "*,score" so
// don't add "id" if the fl is empty or "score" or it would change the meaning.
if (fl.length() != 0 && !"score".equals(fl) && !"*".equals(fl)) {
sreq.params.set(CommonParams.FL, fl + ',' + uniqueField.getName());
}
}
ArrayList<String> ids = new ArrayList<String>(shardDocs.size());
for (ShardDoc shardDoc : shardDocs) {
// TODO: depending on the type, we may need more tha a simple toString()?
ids.add(shardDoc.id.toString());
}
sreq.params.add(ShardParams.IDS, StrUtils.join(ids, ','));
rb.addRequest(this, sreq);
}
}
protected void issueCreateJob(
Integer numberOfSlices,
Integer replicationFactor,
Integer maxShardsPerNode,
List<String> createNodeList,
boolean sendCreateNodeList,
boolean createNodeSetShuffle) {
Map<String, Object> propMap =
ZkNodeProps.makeMap(
Overseer.QUEUE_OPERATION,
CollectionParams.CollectionAction.CREATE.toLower(),
ZkStateReader.REPLICATION_FACTOR,
replicationFactor.toString(),
"name",
COLLECTION_NAME,
"collection.configName",
CONFIG_NAME,
OverseerCollectionProcessor.NUM_SLICES,
numberOfSlices.toString(),
ZkStateReader.MAX_SHARDS_PER_NODE,
maxShardsPerNode.toString());
if (sendCreateNodeList) {
propMap.put(
OverseerCollectionProcessor.CREATE_NODE_SET,
(createNodeList != null) ? StrUtils.join(createNodeList, ',') : null);
if (OverseerCollectionProcessor.CREATE_NODE_SET_SHUFFLE_DEFAULT != createNodeSetShuffle
|| random().nextBoolean()) {
propMap.put(OverseerCollectionProcessor.CREATE_NODE_SET_SHUFFLE, createNodeSetShuffle);
}
}
ZkNodeProps props = new ZkNodeProps(propMap);
QueueEvent qe =
new QueueEvent("id", ZkStateReader.toJSON(props), null) {
@Override
public void setBytes(byte[] bytes) {
lastProcessMessageResult = SolrResponse.deserialize(bytes);
}
};
queue.add(qe);
}
public static Map testForResponseElement(
RestTestHarness harness,
String testServerBaseUrl,
String uri,
CloudSolrClient cloudSolrClient,
List<String> jsonPath,
Object expected,
long maxTimeoutSeconds)
throws Exception {
boolean success = false;
long startTime = System.nanoTime();
Map m = null;
while (TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS)
< maxTimeoutSeconds) {
try {
m =
testServerBaseUrl == null
? getRespMap(uri, harness)
: TestSolrConfigHandlerConcurrent.getAsMap(
testServerBaseUrl + uri, cloudSolrClient);
} catch (Exception e) {
Thread.sleep(100);
continue;
}
if (Objects.equals(expected, Utils.getObjectByPath(m, false, jsonPath))) {
success = true;
break;
}
Thread.sleep(100);
}
assertTrue(
StrUtils.formatString(
"Could not get expected value ''{0}'' for path ''{1}'' full output: {2}, from server: {3}",
expected, StrUtils.join(jsonPath, '/'), getAsString(m), testServerBaseUrl),
success);
return m;
}
@Override
public int distributedProcess(ResponseBuilder rb) throws IOException {
int result = ResponseBuilder.STAGE_DONE;
if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
// Go ask each shard for it's vectors
// for each shard, collect the documents for that shard.
HashMap<String, Collection<ShardDoc>> shardMap = new HashMap<String, Collection<ShardDoc>>();
for (ShardDoc sdoc : rb.resultIds.values()) {
Collection<ShardDoc> shardDocs = shardMap.get(sdoc.shard);
if (shardDocs == null) {
shardDocs = new ArrayList<ShardDoc>();
shardMap.put(sdoc.shard, shardDocs);
}
shardDocs.add(sdoc);
}
// Now create a request for each shard to retrieve the stored fields
for (Collection<ShardDoc> shardDocs : shardMap.values()) {
ShardRequest sreq = new ShardRequest();
sreq.purpose = ShardRequest.PURPOSE_GET_FIELDS;
sreq.shards = new String[] {shardDocs.iterator().next().shard};
sreq.params = new ModifiableSolrParams();
// add original params
sreq.params.add(rb.req.getParams());
sreq.params.remove(CommonParams.Q); // remove the query
ArrayList<String> ids = new ArrayList<String>(shardDocs.size());
for (ShardDoc shardDoc : shardDocs) {
ids.add(shardDoc.id.toString());
}
sreq.params.add(TermVectorParams.DOC_IDS, StrUtils.join(ids, ','));
rb.addRequest(this, sreq);
}
result = ResponseBuilder.STAGE_DONE;
}
return result;
}
private void testCollectionsAPI() throws Exception {
// TODO: fragile - because we dont pass collection.confName, it will only
// find a default if a conf set with a name matching the collection name is found, or
// if there is only one conf set. That and the fact that other tests run first in this
// env make this pretty fragile
// create new collections rapid fire
Map<String, List<Integer>> collectionInfos = new HashMap<String, List<Integer>>();
int cnt = random().nextInt(TEST_NIGHTLY ? 6 : 3) + 1;
for (int i = 0; i < cnt; i++) {
int numShards = _TestUtil.nextInt(random(), 0, shardCount) + 1;
int replicationFactor = _TestUtil.nextInt(random(), 0, 3) + 1;
int maxShardsPerNode =
(((numShards * replicationFactor)
/ getCommonCloudSolrServer()
.getZkStateReader()
.getClusterState()
.getLiveNodes()
.size()))
+ 1;
CloudSolrServer client = null;
try {
if (i == 0) {
// Test if we can create a collection through CloudSolrServer where
// you havnt set default-collection
// This is nice because you want to be able to create you first
// collection using CloudSolrServer, and in such case there is
// nothing reasonable to set as default-collection
client = createCloudClient(null);
} else if (i == 1) {
// Test if we can create a collection through CloudSolrServer where
// you have set default-collection to a non-existing collection
// This is nice because you want to be able to create you first
// collection using CloudSolrServer, and in such case there is
// nothing reasonable to set as default-collection, but you might want
// to use the same CloudSolrServer throughout the entire
// lifetime of your client-application, so it is nice to be able to
// set a default-collection on this CloudSolrServer once and for all
// and use this CloudSolrServer to create the collection
client = createCloudClient("awholynewcollection_" + i);
}
if (secondConfigSet) {
createCollection(
collectionInfos,
"awholynewcollection_" + i,
numShards,
replicationFactor,
maxShardsPerNode,
client,
null,
"conf2");
} else {
createCollection(
collectionInfos,
"awholynewcollection_" + i,
numShards,
replicationFactor,
maxShardsPerNode,
client,
null);
}
} finally {
if (client != null) client.shutdown();
}
}
Set<Entry<String, List<Integer>>> collectionInfosEntrySet = collectionInfos.entrySet();
for (Entry<String, List<Integer>> entry : collectionInfosEntrySet) {
String collection = entry.getKey();
List<Integer> list = entry.getValue();
checkForCollection(collection, list, null);
String url = getUrlFromZk(collection);
HttpSolrServer collectionClient = new HttpSolrServer(url);
// poll for a second - it can take a moment before we are ready to serve
waitForNon403or404or503(collectionClient);
}
// sometimes we restart one of the jetty nodes
if (random().nextBoolean()) {
JettySolrRunner jetty = jettys.get(random().nextInt(jettys.size()));
ChaosMonkey.stop(jetty);
ChaosMonkey.start(jetty);
for (Entry<String, List<Integer>> entry : collectionInfosEntrySet) {
String collection = entry.getKey();
List<Integer> list = entry.getValue();
checkForCollection(collection, list, null);
String url = getUrlFromZk(collection);
HttpSolrServer collectionClient = new HttpSolrServer(url);
// poll for a second - it can take a moment before we are ready to serve
waitForNon403or404or503(collectionClient);
}
}
// sometimes we restart zookeeper
if (random().nextBoolean()) {
zkServer.shutdown();
zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort());
zkServer.run();
}
// sometimes we cause a connection loss - sometimes it will hit the overseer
if (random().nextBoolean()) {
JettySolrRunner jetty = jettys.get(random().nextInt(jettys.size()));
ChaosMonkey.causeConnectionLoss(jetty);
}
ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
for (int j = 0; j < cnt; j++) {
waitForRecoveriesToFinish("awholynewcollection_" + j, zkStateReader, false);
if (secondConfigSet) {
// let's see if they are using the second config set
byte[] data =
zkStateReader
.getZkClient()
.getData(
ZkStateReader.COLLECTIONS_ZKNODE + "/" + "awholynewcollection_" + j,
null,
null,
true);
assertNotNull(data);
ZkNodeProps props = ZkNodeProps.load(data);
String configName = props.getStr(ZkController.CONFIGNAME_PROP);
assertEquals("conf2", configName);
}
}
checkInstanceDirs(jettys.get(0));
List<String> collectionNameList = new ArrayList<String>();
collectionNameList.addAll(collectionInfos.keySet());
String collectionName = collectionNameList.get(random().nextInt(collectionNameList.size()));
String url = getUrlFromZk(collectionName);
HttpSolrServer collectionClient = new HttpSolrServer(url);
// lets try and use the solrj client to index a couple documents
SolrInputDocument doc1 = getDoc(id, 6, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall");
SolrInputDocument doc2 =
getDoc(id, 7, i1, -600, tlong, 600, t1, "humpty dumpy3 sat on a walls");
SolrInputDocument doc3 =
getDoc(id, 8, i1, -600, tlong, 600, t1, "humpty dumpy2 sat on a walled");
collectionClient.add(doc1);
collectionClient.add(doc2);
collectionClient.add(doc3);
collectionClient.commit();
assertEquals(3, collectionClient.query(new SolrQuery("*:*")).getResults().getNumFound());
// lets try a collection reload
// get core open times
Map<String, Long> urlToTimeBefore = new HashMap<String, Long>();
collectStartTimes(collectionName, urlToTimeBefore);
assertTrue(urlToTimeBefore.size() > 0);
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionAction.RELOAD.toString());
params.set("name", collectionName);
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
// we can use this client because we just want base url
final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
createNewSolrServer("", baseUrl).request(request);
// reloads make take a short while
boolean allTimesAreCorrect = waitForReloads(collectionName, urlToTimeBefore);
assertTrue("some core start times did not change on reload", allTimesAreCorrect);
waitForRecoveriesToFinish("awholynewcollection_" + (cnt - 1), zkStateReader, false);
// remove a collection
params = new ModifiableSolrParams();
params.set("action", CollectionAction.DELETE.toString());
params.set("name", collectionName);
request = new QueryRequest(params);
request.setPath("/admin/collections");
createNewSolrServer("", baseUrl).request(request);
// ensure its out of the state
checkForMissingCollection(collectionName);
// collectionNameList.remove(collectionName);
// remove an unknown collection
params = new ModifiableSolrParams();
params.set("action", CollectionAction.DELETE.toString());
params.set("name", "unknown_collection");
request = new QueryRequest(params);
request.setPath("/admin/collections");
boolean exp = false;
try {
createNewSolrServer("", baseUrl).request(request);
} catch (SolrException e) {
exp = true;
}
assertTrue("Expected exception", exp);
// create another collection should still work
params = new ModifiableSolrParams();
params.set("action", CollectionAction.CREATE.toString());
params.set("numShards", 1);
params.set(REPLICATION_FACTOR, 2);
collectionName = "acollectionafterbaddelete";
params.set("name", collectionName);
if (secondConfigSet) {
params.set("collection.configName", "conf1");
}
request = new QueryRequest(params);
request.setPath("/admin/collections");
createNewSolrServer("", baseUrl).request(request);
List<Integer> list = new ArrayList<Integer>(2);
list.add(1);
list.add(2);
checkForCollection(collectionName, list, null);
url = getUrlFromZk(collectionName);
collectionClient = new HttpSolrServer(url);
// poll for a second - it can take a moment before we are ready to serve
waitForNon403or404or503(collectionClient);
for (int j = 0; j < cnt; j++) {
waitForRecoveriesToFinish(collectionName, zkStateReader, false);
}
// test maxShardsPerNode
int numLiveNodes =
getCommonCloudSolrServer().getZkStateReader().getClusterState().getLiveNodes().size();
int numShards = (numLiveNodes / 2) + 1;
int replicationFactor = 2;
int maxShardsPerNode = 1;
collectionInfos = new HashMap<String, List<Integer>>();
CloudSolrServer client = createCloudClient("awholynewcollection_" + cnt);
try {
exp = false;
try {
createCollection(
collectionInfos,
"awholynewcollection_" + cnt,
numShards,
replicationFactor,
maxShardsPerNode,
client,
null,
"conf1");
} catch (SolrException e) {
exp = true;
}
assertTrue("expected exception", exp);
} finally {
client.shutdown();
}
// Test createNodeSet
numLiveNodes =
getCommonCloudSolrServer().getZkStateReader().getClusterState().getLiveNodes().size();
List<String> createNodeList = new ArrayList<String>();
int numOfCreateNodes = numLiveNodes / 2;
assertFalse(
"createNodeSet test is pointless with only " + numLiveNodes + " nodes running",
numOfCreateNodes == 0);
int i = 0;
for (String liveNode :
getCommonCloudSolrServer().getZkStateReader().getClusterState().getLiveNodes()) {
if (i < numOfCreateNodes) {
createNodeList.add(liveNode);
i++;
} else {
break;
}
}
maxShardsPerNode = 2;
numShards = createNodeList.size() * maxShardsPerNode;
replicationFactor = 1;
collectionInfos = new HashMap<String, List<Integer>>();
client = createCloudClient("awholynewcollection_" + (cnt + 1));
try {
createCollection(
collectionInfos,
"awholynewcollection_" + (cnt + 1),
numShards,
replicationFactor,
maxShardsPerNode,
client,
StrUtils.join(createNodeList, ','),
"conf1");
} finally {
client.shutdown();
}
checkForCollection(
collectionInfos.keySet().iterator().next(),
collectionInfos.entrySet().iterator().next().getValue(),
createNodeList);
checkNoTwoShardsUseTheSameIndexDir();
}
public int createSubRequests(ResponseBuilder rb) throws IOException {
SolrParams params = rb.req.getParams();
String id1[] = params.getParams("id");
String ids[] = params.getParams("ids");
if (id1 == null && ids == null) {
return ResponseBuilder.STAGE_DONE;
}
List<String> allIds = new ArrayList<String>();
if (id1 != null) {
for (String s : id1) {
allIds.add(s);
}
}
if (ids != null) {
for (String s : ids) {
allIds.addAll(StrUtils.splitSmart(s, ",", true));
}
}
// TODO: handle collection=...?
ZkController zkController =
rb.req.getCore().getCoreDescriptor().getCoreContainer().getZkController();
// if shards=... then use that
if (zkController != null && params.get("shards") == null) {
CloudDescriptor cloudDescriptor = rb.req.getCore().getCoreDescriptor().getCloudDescriptor();
String collection = cloudDescriptor.getCollectionName();
ClusterState clusterState = zkController.getClusterState();
DocCollection coll = clusterState.getCollection(collection);
Map<String, List<String>> sliceToId = new HashMap<String, List<String>>();
for (String id : allIds) {
Slice slice = coll.getRouter().getTargetSlice(id, null, params, coll);
List<String> idsForShard = sliceToId.get(slice.getName());
if (idsForShard == null) {
idsForShard = new ArrayList<String>(2);
sliceToId.put(slice.getName(), idsForShard);
}
idsForShard.add(id);
}
for (Map.Entry<String, List<String>> entry : sliceToId.entrySet()) {
String shard = entry.getKey();
String shardIdList = StrUtils.join(entry.getValue(), ',');
ShardRequest sreq = new ShardRequest();
sreq.purpose = 1;
// sreq.shards = new String[]{shard}; // TODO: would be nice if this would work...
sreq.shards = sliceToShards(rb, collection, shard);
sreq.actualShards = sreq.shards;
sreq.params = new ModifiableSolrParams();
sreq.params.set(
ShardParams.SHARDS_QT,
"/get"); // TODO: how to avoid hardcoding this and hit the same handler?
sreq.params.set("distrib", false);
sreq.params.set("ids", shardIdList);
rb.addRequest(this, sreq);
}
} else {
String shardIdList = StrUtils.join(allIds, ',');
ShardRequest sreq = new ShardRequest();
sreq.purpose = 1;
sreq.shards = null; // ALL
sreq.actualShards = sreq.shards;
sreq.params = new ModifiableSolrParams();
sreq.params.set(
ShardParams.SHARDS_QT,
"/get"); // TODO: how to avoid hardcoding this and hit the same handler?
sreq.params.set("distrib", false);
sreq.params.set("ids", shardIdList);
rb.addRequest(this, sreq);
}
return ResponseBuilder.STAGE_DONE;
}