@Override
public void finishStage(ResponseBuilder rb) {
if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
NamedList termVectors = new NamedList<Object>();
Map.Entry<String, Object>[] arr = new NamedList.NamedListEntry[rb.resultIds.size()];
for (ShardRequest sreq : rb.finished) {
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) == 0
|| !sreq.params.getBool(COMPONENT_NAME, false)) {
continue;
}
for (ShardResponse srsp : sreq.responses) {
NamedList<Object> nl =
(NamedList<Object>) srsp.getSolrResponse().getResponse().get(TERM_VECTORS);
for (int i = 0; i < nl.size(); i++) {
String key = nl.getName(i);
ShardDoc sdoc = rb.resultIds.get(key);
if (null == sdoc) {
// metadata, only need from one node, leave in order
if (termVectors.indexOf(key, 0) < 0) {
termVectors.add(key, nl.getVal(i));
}
} else {
int idx = sdoc.positionInResponse;
arr[idx] = new NamedList.NamedListEntry<Object>(key, nl.getVal(i));
}
}
}
}
// remove nulls in case not all docs were able to be retrieved
termVectors.addAll(SolrPluginUtils.removeNulls(new NamedList<Object>(arr)));
rb.rsp.add(TERM_VECTORS, termVectors);
}
}
private void returnFields(ResponseBuilder rb, ShardRequest sreq) {
// Keep in mind that this could also be a shard in a multi-tiered system.
// TODO: if a multi-tiered system, it seems like some requests
// could/should bypass middlemen (like retrieving stored fields)
// TODO: merge fsv to if requested
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) != 0) {
boolean returnScores = (rb.getFieldFlags() & SolrIndexSearcher.GET_SCORES) != 0;
assert (sreq.responses.size() == 1);
ShardResponse srsp = sreq.responses.get(0);
SolrDocumentList docs =
(SolrDocumentList) srsp.getSolrResponse().getResponse().get("response");
String keyFieldName = rb.req.getSchema().getUniqueKeyField().getName();
for (SolrDocument doc : docs) {
Object id = doc.getFieldValue(keyFieldName);
ShardDoc sdoc = rb.resultIds.get(id.toString());
if (sdoc != null) {
if (returnScores && sdoc.score != null) {
doc.setField("score", sdoc.score);
}
rb._responseDocs.set(sdoc.positionInResponse, doc);
}
}
}
}
@Override
public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
if (!rb.doTerms || (sreq.purpose & ShardRequest.PURPOSE_GET_TERMS) == 0) {
return;
}
TermsHelper th = rb._termsHelper;
if (th != null) {
for (ShardResponse srsp : sreq.responses) {
th.parse((NamedList) srsp.getSolrResponse().getResponse().get("terms"));
}
}
}
@Override
public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
if (!rb.doTerms || (sreq.purpose & ShardRequest.PURPOSE_GET_TERMS) == 0) {
return;
}
TermsHelper th = rb._termsHelper;
if (th != null) {
for (ShardResponse srsp : sreq.responses) {
@SuppressWarnings("unchecked")
NamedList<NamedList<Number>> terms =
(NamedList<NamedList<Number>>) srsp.getSolrResponse().getResponse().get("terms");
th.parse(terms);
}
}
}
@Override
protected ShardResponse processRequestItems(
ShardId shardId, ShardUpsertRequest request, AtomicBoolean killed) {
ShardResponse shardResponse = new ShardResponse();
DocTableInfo tableInfo = schemas.getWritableTable(TableIdent.fromIndexName(request.index()));
for (int i = 0; i < request.itemIndices().size(); i++) {
int location = request.itemIndices().get(i);
ShardUpsertRequest.Item item = request.items().get(i);
if (killed.get()) {
throw new CancellationException();
}
try {
indexItem(
tableInfo,
request,
item,
shardId,
item.insertValues() != null, // try insert first
0);
shardResponse.add(location);
} catch (Throwable t) {
if (!TransportActions.isShardNotAvailableException(t) && !request.continueOnError()) {
throw t;
} else {
logger.debug(
"{} failed to execute upsert for [{}]/[{}]",
t,
request.shardId(),
request.type(),
item.id());
shardResponse.add(
location,
new ShardResponse.Failure(
item.id(),
ExceptionsHelper.detailedMessage(t),
(t instanceof VersionConflictEngineException)));
}
}
}
return shardResponse;
}
private void mergeResponses(ResponseBuilder rb) {
SolrDocumentList docList = new SolrDocumentList();
for (ShardRequest sreq : rb.finished) {
// if shards=shard1,shard2 was used, then we query both shards for each id and
// can get more than one response
for (ShardResponse srsp : sreq.responses) {
SolrResponse sr = srsp.getSolrResponse();
NamedList nl = sr.getResponse();
SolrDocumentList subList = (SolrDocumentList) nl.get("response");
docList.addAll(subList);
}
}
if (docList.size() <= 1 && rb.req.getParams().getParams("ids") == null) {
// if the doc was not found, then use a value of null.
rb.rsp.add("doc", docList.size() > 0 ? docList.get(0) : null);
} else {
docList.setNumFound(docList.size());
rb.rsp.add("response", docList);
}
}
@Override
public void finishStage(ResponseBuilder rb) {
if (rb.doHighlights && rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
Map.Entry<String, Object>[] arr = new NamedList.NamedListEntry[rb.resultIds.size()];
// TODO: make a generic routine to do automatic merging of id keyed data
for (ShardRequest sreq : rb.finished) {
if ((sreq.purpose & ShardRequest.PURPOSE_GET_HIGHLIGHTS) == 0) continue;
for (ShardResponse srsp : sreq.responses) {
NamedList hl = (NamedList) srsp.getSolrResponse().getResponse().get("highlighting");
for (int i = 0; i < hl.size(); i++) {
String id = hl.getName(i);
ShardDoc sdoc = rb.resultIds.get(id);
int idx = sdoc.positionInResponse;
arr[idx] = new NamedList.NamedListEntry<Object>(id, hl.getVal(i));
}
}
}
// remove nulls in case not all docs were able to be retrieved
rb.rsp.add("highlighting", SolrPluginUtils.removeNulls(new SimpleOrderedMap(arr)));
}
}
private void mergeIds(ResponseBuilder rb, ShardRequest sreq) {
SortSpec ss = rb.getSortSpec();
Sort sort = ss.getSort();
SortField[] sortFields = null;
if (sort != null) sortFields = sort.getSort();
else {
sortFields = new SortField[] {SortField.FIELD_SCORE};
}
SchemaField uniqueKeyField = rb.req.getSchema().getUniqueKeyField();
// id to shard mapping, to eliminate any accidental dups
HashMap<Object, String> uniqueDoc = new HashMap<Object, String>();
// Merge the docs via a priority queue so we don't have to sort *all* of the
// documents... we only need to order the top (rows+start)
ShardFieldSortedHitQueue queue;
queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount());
long numFound = 0;
Float maxScore = null;
for (ShardResponse srsp : sreq.responses) {
SolrDocumentList docs =
(SolrDocumentList) srsp.getSolrResponse().getResponse().get("response");
// calculate global maxScore and numDocsFound
if (docs.getMaxScore() != null) {
maxScore = maxScore == null ? docs.getMaxScore() : Math.max(maxScore, docs.getMaxScore());
}
numFound += docs.getNumFound();
NamedList sortFieldValues =
(NamedList) (srsp.getSolrResponse().getResponse().get("sort_values"));
// go through every doc in this response, construct a ShardDoc, and
// put it in the priority queue so it can be ordered.
for (int i = 0; i < docs.size(); i++) {
SolrDocument doc = docs.get(i);
Object id = doc.getFieldValue(uniqueKeyField.getName());
String prevShard = uniqueDoc.put(id, srsp.getShard());
if (prevShard != null) {
// duplicate detected
numFound--;
// For now, just always use the first encountered since we can't currently
// remove the previous one added to the priority queue. If we switched
// to the Java5 PriorityQueue, this would be easier.
continue;
// make which duplicate is used deterministic based on shard
// if (prevShard.compareTo(srsp.shard) >= 0) {
// TODO: remove previous from priority queue
// continue;
// }
}
ShardDoc shardDoc = new ShardDoc();
shardDoc.id = id;
shardDoc.shard = srsp.getShard();
shardDoc.orderInShard = i;
Object scoreObj = doc.getFieldValue("score");
if (scoreObj != null) {
if (scoreObj instanceof String) {
shardDoc.score = Float.parseFloat((String) scoreObj);
} else {
shardDoc.score = (Float) scoreObj;
}
}
shardDoc.sortFieldValues = sortFieldValues;
queue.insertWithOverflow(shardDoc);
} // end for-each-doc-in-response
} // end for-each-response
// The queue now has 0 -> queuesize docs, where queuesize <= start + rows
// So we want to pop the last documents off the queue to get
// the docs offset -> queuesize
int resultSize = queue.size() - ss.getOffset();
resultSize = Math.max(0, resultSize); // there may not be any docs in range
Map<Object, ShardDoc> resultIds = new HashMap<Object, ShardDoc>();
for (int i = resultSize - 1; i >= 0; i--) {
ShardDoc shardDoc = (ShardDoc) queue.pop();
shardDoc.positionInResponse = i;
// Need the toString() for correlation with other lists that must
// be strings (like keys in highlighting, explain, etc)
resultIds.put(shardDoc.id.toString(), shardDoc);
}
SolrDocumentList responseDocs = new SolrDocumentList();
if (maxScore != null) responseDocs.setMaxScore(maxScore);
responseDocs.setNumFound(numFound);
responseDocs.setStart(ss.getOffset());
// size appropriately
for (int i = 0; i < resultSize; i++) responseDocs.add(null);
// save these results in a private area so we can access them
// again when retrieving stored fields.
// TODO: use ResponseBuilder (w/ comments) or the request context?
rb.resultIds = resultIds;
rb._responseDocs = responseDocs;
}
