protected List<SubmitCapture> mockShardHandlerForCreateJob(
Integer numberOfSlices, Integer numberOfReplica) {
List<SubmitCapture> submitCaptures = new ArrayList<>();
for (int i = 0; i < (numberOfSlices * numberOfReplica); i++) {
SubmitCapture submitCapture = new SubmitCapture();
shardHandlerMock.submit(
capture(submitCapture.shardRequestCapture),
capture(submitCapture.nodeUrlsWithoutProtocolPartCapture),
capture(submitCapture.params));
expectLastCall();
submitCaptures.add(submitCapture);
ShardResponse shardResponseWithoutException = new ShardResponse();
shardResponseWithoutException.setSolrResponse(new QueryResponse());
expect(shardHandlerMock.takeCompletedOrError()).andReturn(shardResponseWithoutException);
}
expect(shardHandlerMock.takeCompletedOrError()).andReturn(null);
return submitCaptures;
}
private boolean handleResponse(ShardResponse srsp) {
NamedList<Object> response = srsp.getSolrResponse().getResponse();
// TODO: why does this return null sometimes?
if (response == null) {
return false;
}
Boolean success = (Boolean) response.get("sync");
if (success == null) {
success = false;
}
return success;
}
private void syncToMe(
ZkController zkController,
String collection,
String shardId,
ZkNodeProps leaderProps,
CoreDescriptor cd) {
// sync everyone else
// TODO: we should do this in parallel at least
List<ZkCoreNodeProps> nodes =
zkController
.getZkStateReader()
.getReplicaProps(collection, shardId, cd.getCloudDescriptor().getCoreNodeName());
if (nodes == null) {
log.info(ZkCoreNodeProps.getCoreUrl(leaderProps) + " has no replicas");
return;
}
ZkCoreNodeProps zkLeader = new ZkCoreNodeProps(leaderProps);
for (ZkCoreNodeProps node : nodes) {
try {
log.info(
ZkCoreNodeProps.getCoreUrl(leaderProps)
+ ": try and ask "
+ node.getCoreUrl()
+ " to sync");
requestSync(
node.getBaseUrl(), node.getCoreUrl(), zkLeader.getCoreUrl(), node.getCoreName());
} catch (Exception e) {
SolrException.log(log, "Error syncing replica to leader", e);
}
}
for (; ; ) {
ShardResponse srsp = shardHandler.takeCompletedOrError();
if (srsp == null) break;
boolean success = handleResponse(srsp);
if (srsp.getException() != null) {
SolrException.log(log, "Sync request error: " + srsp.getException());
}
if (!success) {
try {
log.info(
ZkCoreNodeProps.getCoreUrl(leaderProps)
+ ": Sync failed - asking replica ("
+ srsp.getShardAddress()
+ ") to recover.");
if (isClosed) {
log.info("We have been closed, don't request that a replica recover");
} else {
requestRecovery(
leaderProps,
((ShardCoreRequest) srsp.getShardRequest()).baseUrl,
((ShardCoreRequest) srsp.getShardRequest()).coreName);
}
} catch (Exception e) {
SolrException.log(
log,
ZkCoreNodeProps.getCoreUrl(leaderProps) + ": Could not tell a replica to recover",
e);
}
} else {
log.info(
ZkCoreNodeProps.getCoreUrl(leaderProps)
+ ": "
+ " sync completed with "
+ srsp.getShardAddress());
}
}
}
public void merge(ResponseBuilder rb, ShardRequest sreq) {
// id to shard mapping, to eliminate any accidental dups
HashMap<Object, String> uniqueDoc = new HashMap<>();
NamedList<Object> shardInfo = null;
if (rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) {
shardInfo = new SimpleOrderedMap<>();
rb.rsp.getValues().add(ShardParams.SHARDS_INFO, shardInfo);
}
IndexSchema schema = rb.req.getSchema();
SchemaField uniqueKeyField = schema.getUniqueKeyField();
long numFound = 0;
Float maxScore = null;
boolean partialResults = false;
List<ShardDoc> shardDocs = new ArrayList();
for (ShardResponse srsp : sreq.responses) {
SolrDocumentList docs = null;
if (shardInfo != null) {
SimpleOrderedMap<Object> nl = new SimpleOrderedMap<>();
if (srsp.getException() != null) {
Throwable t = srsp.getException();
if (t instanceof SolrServerException) {
t = ((SolrServerException) t).getCause();
}
nl.add("error", t.toString());
StringWriter trace = new StringWriter();
t.printStackTrace(new PrintWriter(trace));
nl.add("trace", trace.toString());
if (srsp.getShardAddress() != null) {
nl.add("shardAddress", srsp.getShardAddress());
}
} else {
docs = (SolrDocumentList) srsp.getSolrResponse().getResponse().get("response");
nl.add("numFound", docs.getNumFound());
nl.add("maxScore", docs.getMaxScore());
nl.add("shardAddress", srsp.getShardAddress());
}
if (srsp.getSolrResponse() != null) {
nl.add("time", srsp.getSolrResponse().getElapsedTime());
}
shardInfo.add(srsp.getShard(), nl);
}
// now that we've added the shard info, let's only proceed if we have no error.
if (srsp.getException() != null) {
partialResults = true;
continue;
}
if (docs == null) { // could have been initialized in the shards info block above
docs = (SolrDocumentList) srsp.getSolrResponse().getResponse().get("response");
}
NamedList<?> responseHeader =
(NamedList<?>) srsp.getSolrResponse().getResponse().get("responseHeader");
if (responseHeader != null
&& Boolean.TRUE.equals(
responseHeader.get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
partialResults = true;
}
// calculate global maxScore and numDocsFound
if (docs.getMaxScore() != null) {
maxScore = maxScore == null ? docs.getMaxScore() : Math.max(maxScore, docs.getMaxScore());
}
numFound += docs.getNumFound();
SortSpec ss = rb.getSortSpec();
Sort sort = ss.getSort();
NamedList sortFieldValues =
(NamedList) (srsp.getSolrResponse().getResponse().get("merge_values"));
NamedList unmarshalledSortFieldValues = unmarshalSortValues(ss, sortFieldValues, schema);
List lst = (List) unmarshalledSortFieldValues.getVal(0);
for (int i = 0; i < docs.size(); i++) {
SolrDocument doc = docs.get(i);
Object id = doc.getFieldValue(uniqueKeyField.getName());
String prevShard = uniqueDoc.put(id, srsp.getShard());
if (prevShard != null) {
// duplicate detected
numFound--;
// For now, just always use the first encountered since we can't currently
// remove the previous one added to the priority queue. If we switched
// to the Java5 PriorityQueue, this would be easier.
continue;
// make which duplicate is used deterministic based on shard
// if (prevShard.compareTo(srsp.shard) >= 0) {
// TODO: remove previous from priority queue
// continue;
// }
}
ShardDoc shardDoc = new ShardDoc();
shardDoc.id = id;
shardDoc.shard = srsp.getShard();
shardDoc.orderInShard = i;
Object scoreObj = lst.get(i);
if (scoreObj != null) {
shardDoc.score = ((Integer) scoreObj).floatValue();
}
shardDocs.add(shardDoc);
} // end for-each-doc-in-response
} // end for-each-response
Collections.sort(
shardDocs,
new Comparator<ShardDoc>() {
@Override
public int compare(ShardDoc o1, ShardDoc o2) {
if (o1.score < o2.score) {
return 1;
} else if (o1.score > o2.score) {
return -1;
} else {
return 0; // To change body of implemented methods use File | Settings | File
// Templates.
}
}
});
int resultSize = shardDocs.size();
Map<Object, ShardDoc> resultIds = new HashMap<>();
for (int i = 0; i < shardDocs.size(); i++) {
ShardDoc shardDoc = shardDocs.get(i);
shardDoc.positionInResponse = i;
// Need the toString() for correlation with other lists that must
// be strings (like keys in highlighting, explain, etc)
resultIds.put(shardDoc.id.toString(), shardDoc);
}
// Add hits for distributed requests
// https://issues.apache.org/jira/browse/SOLR-3518
rb.rsp.addToLog("hits", numFound);
SolrDocumentList responseDocs = new SolrDocumentList();
if (maxScore != null) responseDocs.setMaxScore(maxScore);
responseDocs.setNumFound(numFound);
responseDocs.setStart(0);
// size appropriately
for (int i = 0; i < resultSize; i++) responseDocs.add(null);
// save these results in a private area so we can access them
// again when retrieving stored fields.
// TODO: use ResponseBuilder (w/ comments) or the request context?
rb.resultIds = resultIds;
rb.setResponseDocs(responseDocs);
if (partialResults) {
rb.rsp
.getResponseHeader()
.add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
}
}
