private void handleGET() {
if (parts.size() == 1) {
resp.add("solrConfig", req.getCore().getSolrConfig().toMap());
} else {
if (ConfigOverlay.NAME.equals(parts.get(1))) {
resp.add(ConfigOverlay.NAME, req.getCore().getSolrConfig().getOverlay().toMap());
} else if (RequestParams.NAME.equals(parts.get(1))) {
if (parts.size() == 3) {
RequestParams params = req.getCore().getSolrConfig().getRequestParams();
MapSolrParams p = params.getParams(parts.get(2));
Map m = new LinkedHashMap<>();
m.put(ConfigOverlay.ZNODEVER, params.getZnodeVersion());
if (p != null) {
m.put(RequestParams.NAME, ZkNodeProps.makeMap(parts.get(2), p.getMap()));
}
resp.add(SolrQueryResponse.NAME, m);
} else {
resp.add(
SolrQueryResponse.NAME, req.getCore().getSolrConfig().getRequestParams().toMap());
}
} else {
Map<String, Object> m = req.getCore().getSolrConfig().toMap();
resp.add("solrConfig", ZkNodeProps.makeMap(parts.get(1), m.get(parts.get(1))));
}
}
}
private void doCompact(
LazyDocumentLoader documentLoader,
LazyCompactor compactor,
SolrQueryResponse rsp,
String tsId,
String q)
throws IOException, SyntaxError {
Query query = depProvider.parser(q).getQuery();
Iterable<Document> docs = documentLoader.load(query, SORT);
Iterable<CompactionResult> compactionResults = compactor.compact(docs);
List<Document> docsToDelete = new LinkedList<>();
List<SolrInputDocument> docsToAdd = new LinkedList<>();
compactionResults.forEach(
it -> {
docsToDelete.addAll(it.getInputDocuments());
docsToAdd.addAll(it.getOutputDocuments());
});
depProvider.solrUpdateService().add(docsToAdd);
depProvider.solrUpdateService().delete(docsToDelete);
rsp.add("timeseries " + tsId + " oldNumDocs:", docsToDelete.size());
rsp.add("timeseries " + tsId + " newNumDocs:", docsToAdd.size());
}
private void handleAdmin(SolrQueryRequest req, SolrQueryResponse rsp, SolrParams params) {
String action = params.get("action");
if ("stop".equalsIgnoreCase(action)) {
String id = params.get("id");
DaemonStream d = daemons.get(id);
if (d != null) {
d.close();
rsp.add("result-set", new DaemonResponseStream("Deamon:" + id + " stopped on " + coreName));
} else {
rsp.add(
"result-set", new DaemonResponseStream("Deamon:" + id + " not found on " + coreName));
}
} else if ("start".equalsIgnoreCase(action)) {
String id = params.get("id");
DaemonStream d = daemons.get(id);
d.open();
rsp.add("result-set", new DaemonResponseStream("Deamon:" + id + " started on " + coreName));
} else if ("list".equalsIgnoreCase(action)) {
Collection<DaemonStream> vals = daemons.values();
rsp.add("result-set", new DaemonCollectionStream(vals));
} else if ("kill".equalsIgnoreCase(action)) {
String id = params.get("id");
DaemonStream d = daemons.remove(id);
if (d != null) {
d.close();
}
rsp.add("result-set", new DaemonResponseStream("Deamon:" + id + " killed on " + coreName));
}
}
protected void writeResponse(
SolrQueryResponse solrRsp, QueryResponseWriter responseWriter, SolrQueryRequest solrReq)
throws IOException {
Object invalidStates = solrReq.getContext().get(CloudSolrClient.STATE_VERSION);
// This is the last item added to the applyResult and the client would expect it that way.
// If that assumption is changed , it would fail. This is done to avoid an O(n) scan on
// the applyResult for each request
if (invalidStates != null) solrRsp.add(CloudSolrClient.STATE_VERSION, invalidStates);
// Now write it out
final String ct = responseWriter.getContentType(solrReq, solrRsp);
// don't call setContentType on null
if (null != ct) {
responseSetter.setContentType(ct);
}
if (solrRsp.getException() != null) {
NamedList info = new SimpleOrderedMap();
int code = ResponseUtils.getErrorInfo(solrRsp.getException(), info, logger);
// solrRsp.add("error", info);
// use protocol response exception instead of set 'error' response return to client,
responseSetter.setSolrResponseException(code, info);
}
QueryResponseWriterUtil.writeQueryResponse(
responseSetter.getResponseOutputStream(), responseWriter, solrReq, solrRsp, ct);
// fire QueryResponse write Complete
responseSetter.writeQueryResponseComplete(solrRsp);
}
@Test
public void testCollateWithFilter() throws Exception {
SolrCore core = h.getCore();
SearchComponent speller = core.getSearchComponent("spellcheck");
assertTrue("speller is null and it shouldn't be", speller != null);
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(SpellCheckComponent.COMPONENT_NAME, "true");
params.add(SpellCheckComponent.SPELLCHECK_BUILD, "true");
params.add(SpellCheckComponent.SPELLCHECK_COUNT, "10");
params.add(SpellCheckComponent.SPELLCHECK_COLLATE, "true");
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES, "5");
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "2");
params.add(CommonParams.Q, "lowerfilt:(+fauth +home +loane)");
params.add(CommonParams.FQ, "NOT(id:1)");
// Because a FilterQuery is applied which removes doc id#1 from possible hits, we would
// not want the collations to return us "lowerfilt:(+faith +hope +loaves)" as this only matches
// doc id#1.
SolrRequestHandler handler = core.getRequestHandler("spellCheckCompRH");
SolrQueryResponse rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
SolrQueryRequest req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
req.close();
NamedList values = rsp.getValues();
NamedList spellCheck = (NamedList) values.get("spellcheck");
NamedList suggestions = (NamedList) spellCheck.get("suggestions");
List<String> collations = suggestions.getAll("collation");
assertTrue(collations.size() == 1);
assertTrue(collations.get(0).equals("lowerfilt:(+faith +hope +love)"));
}
private void writeResponse(
SolrQueryResponse solrRsp, QueryResponseWriter responseWriter, Method reqMethod)
throws IOException {
try {
Object invalidStates = solrReq.getContext().get(CloudSolrClient.STATE_VERSION);
// This is the last item added to the response and the client would expect it that way.
// If that assumption is changed , it would fail. This is done to avoid an O(n) scan on
// the response for each request
if (invalidStates != null) solrRsp.add(CloudSolrClient.STATE_VERSION, invalidStates);
// Now write it out
final String ct = responseWriter.getContentType(solrReq, solrRsp);
// don't call setContentType on null
if (null != ct) response.setContentType(ct);
if (solrRsp.getException() != null) {
NamedList info = new SimpleOrderedMap();
int code = ResponseUtils.getErrorInfo(solrRsp.getException(), info, log);
solrRsp.add("error", info);
response.setStatus(code);
}
if (Method.HEAD != reqMethod) {
OutputStream out =
new CloseShieldOutputStream(
response.getOutputStream()); // Prevent close of container streams, see SOLR-8933
QueryResponseWriterUtil.writeQueryResponse(out, responseWriter, solrReq, solrRsp, ct);
}
// else http HEAD request, nothing to write out, waited this long just to get ContentType
} catch (EOFException e) {
log.info("Unable to write response, client closed connection or we are shutting down", e);
}
}
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
SolrParams params = req.getParams();
params = adjustParams(params);
req.setParams(params);
TupleStream tupleStream = null;
try {
tupleStream = this.streamFactory.constructStream(params.get("expr"));
} catch (Exception e) {
// Catch exceptions that occur while the stream is being created. This will include streaming
// expression parse rules.
SolrException.log(logger, e);
rsp.add("result-set", new DummyErrorStream(e));
return;
}
int worker = params.getInt("workerID", 0);
int numWorkers = params.getInt("numWorkers", 1);
StreamContext context = new StreamContext();
context.workerID = worker;
context.numWorkers = numWorkers;
context.setSolrClientCache(clientCache);
tupleStream.setStreamContext(context);
rsp.add("result-set", new TimerStream(new ExceptionStream(tupleStream)));
}
@Test
public void testNaNInf() throws IOException {
SolrQueryRequest req = req("dummy");
SolrQueryResponse rsp = new SolrQueryResponse();
QueryResponseWriter w = new PythonResponseWriter();
StringWriter buf = new StringWriter();
rsp.add("data1", Float.NaN);
rsp.add("data2", Double.NEGATIVE_INFINITY);
rsp.add("data3", Float.POSITIVE_INFINITY);
w.write(buf, req, rsp);
assertEquals(
buf.toString(), "{'data1':float('NaN'),'data2':-float('Inf'),'data3':float('Inf')}");
w = new RubyResponseWriter();
buf = new StringWriter();
w.write(buf, req, rsp);
assertEquals(buf.toString(), "{'data1'=>(0.0/0.0),'data2'=>-(1.0/0.0),'data3'=>(1.0/0.0)}");
w = new JSONResponseWriter();
buf = new StringWriter();
w.write(buf, req, rsp);
assertEquals(
buf.toString(), "{\"data1\":\"NaN\",\"data2\":\"-Infinity\",\"data3\":\"Infinity\"}");
req.close();
}
public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
numRequests++;
Query query = null;
Filter filter = null;
List<String> commands = StrUtils.splitSmart(req.getQueryString(), ';');
String qs = commands.size() >= 1 ? commands.get(0) : "";
query = QueryParsing.parseQuery(qs, req.getSchema());
// If the first non-query, non-filter command is a simple sort on an indexed field, then
// we can use the Lucene sort ability.
Sort sort = null;
if (commands.size() >= 2) {
sort = QueryParsing.parseSort(commands.get(1), req);
}
try {
int numHits;
ScoreDoc[] scoreDocs;
if (sort != null) {
TopFieldDocs hits =
req.getSearcher().search(query, filter, req.getStart() + req.getLimit(), sort);
scoreDocs = hits.scoreDocs;
numHits = hits.totalHits;
} else {
TopDocs hits = req.getSearcher().search(query, filter, req.getStart() + req.getLimit());
scoreDocs = hits.scoreDocs;
numHits = hits.totalHits;
}
int startRow = Math.min(numHits, req.getStart());
int endRow = Math.min(numHits, req.getStart() + req.getLimit());
int numRows = endRow - startRow;
int[] ids = new int[numRows];
Document[] data = new Document[numRows];
for (int i = startRow; i < endRow; i++) {
ids[i] = scoreDocs[i].doc;
data[i] = req.getSearcher().doc(ids[i]);
}
rsp.add(null, new DocSlice(0, numRows, ids, null, numHits, 0.0f));
/**
* ********************* rsp.setResults(new DocSlice(0,numRows,ids,null,numHits));
*
* <p>// Setting the actual document objects is optional rsp.setResults(data);
* **********************
*/
} catch (IOException e) {
rsp.setException(e);
numErrors++;
return;
}
}
/** make sure to close req after you are done using the response */
public SolrQueryResponse getResponse(SolrQueryRequest req) throws Exception {
SolrQueryResponse rsp = new SolrQueryResponse();
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
h.getCore().execute(h.getCore().getRequestHandler(null), req, rsp);
if (rsp.getException() != null) {
throw rsp.getException();
}
return rsp;
}
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
SolrParams params = req.getParams();
params = adjustParams(params);
req.setParams(params);
if (params.get("action") != null) {
handleAdmin(req, rsp, params);
return;
}
TupleStream tupleStream;
try {
tupleStream = this.streamFactory.constructStream(params.get("expr"));
} catch (Exception e) {
// Catch exceptions that occur while the stream is being created. This will include streaming
// expression parse rules.
SolrException.log(logger, e);
rsp.add("result-set", new DummyErrorStream(e));
return;
}
int worker = params.getInt("workerID", 0);
int numWorkers = params.getInt("numWorkers", 1);
StreamContext context = new StreamContext();
context.workerID = worker;
context.numWorkers = numWorkers;
context.setSolrClientCache(clientCache);
context.setModelCache(modelCache);
context.put("core", this.coreName);
context.put("solr-core", req.getCore());
tupleStream.setStreamContext(context);
// if asking for explanation then go get it
if (params.getBool("explain", false)) {
rsp.add("explanation", tupleStream.toExplanation(this.streamFactory));
}
if (tupleStream instanceof DaemonStream) {
DaemonStream daemonStream = (DaemonStream) tupleStream;
if (daemons.containsKey(daemonStream.getId())) {
daemons.remove(daemonStream.getId()).close();
}
daemonStream.setDaemons(daemons);
daemonStream.open(); // This will start the deamonStream
daemons.put(daemonStream.getId(), daemonStream);
rsp.add(
"result-set",
new DaemonResponseStream("Deamon:" + daemonStream.getId() + " started on " + coreName));
} else {
rsp.add("result-set", new TimerStream(new ExceptionStream(tupleStream)));
}
}
public static JSONDumper create(SolrQueryRequest req, File jobFile, Bits bitSet)
throws IOException {
SolrQueryResponse rsp = new SolrQueryResponse();
FileWriter writer = new FileWriter(jobFile);
ReturnFields returnFields = new SolrReturnFields(req);
rsp.setReturnFields(returnFields);
JSONDumper d = new JSONDumper(writer, req, rsp);
d.setBitset(bitSet);
return d;
}
private void handleAdminRequest() throws IOException {
SolrQueryResponse solrResp = new SolrQueryResponse();
SolrCore.preDecorateResponse(solrReq, solrResp);
handler.handleRequest(solrReq, solrResp);
SolrCore.postDecorateResponse(handler, solrReq, solrResp);
if (log.isInfoEnabled() && solrResp.getToLog().size() > 0) {
log.info(solrResp.getToLogAsString("[admin]"));
}
QueryResponseWriter respWriter =
SolrCore.DEFAULT_RESPONSE_WRITERS.get(solrReq.getParams().get(CommonParams.WT));
if (respWriter == null) respWriter = SolrCore.DEFAULT_RESPONSE_WRITERS.get("standard");
writeResponse(solrResp, respWriter, Method.getMethod(req.getMethod()));
}
public void testJSON() throws IOException {
SolrQueryRequest req = req("wt", "json", "json.nl", "arrarr");
SolrQueryResponse rsp = new SolrQueryResponse();
JSONResponseWriter w = new JSONResponseWriter();
StringWriter buf = new StringWriter();
NamedList nl = new NamedList();
nl.add("data1", "hello");
nl.add(null, 42);
rsp.add("nl", nl);
w.write(buf, req, rsp);
assertEquals(buf.toString(), "{\"nl\":[[\"data1\",\"hello\"],[null,42]]}");
req.close();
}
public void transform(
Map<String, ?> result,
SolrQueryResponse response,
GroupingSpecification groupingSpecification,
SolrDocumentSource solrDocumentSource) {
Object value = result.get(groupingSpecification.getFields()[0]);
if (TopGroups.class.isInstance(value)) {
@SuppressWarnings("unchecked")
TopGroups<BytesRef> topGroups = (TopGroups<BytesRef>) value;
SolrDocumentList docList = new SolrDocumentList();
docList.setStart(groupingSpecification.getOffset());
docList.setNumFound(topGroups.totalHitCount);
Float maxScore = Float.NEGATIVE_INFINITY;
for (GroupDocs<BytesRef> group : topGroups.groups) {
for (ScoreDoc scoreDoc : group.scoreDocs) {
if (maxScore < scoreDoc.score) {
maxScore = scoreDoc.score;
}
docList.add(solrDocumentSource.retrieve(scoreDoc));
}
}
if (maxScore != Float.NEGATIVE_INFINITY) {
docList.setMaxScore(maxScore);
}
response.add("response", docList);
}
}
/**
* conversion from a SolrQueryResponse (which is a solr-internal data format) to SolrDocumentList
* (which is a solrj-format) The conversion is done inside the solrj api using the
* BinaryResponseWriter and a very complex unfolding process via
* org.apache.solr.common.util.JavaBinCodec.marshal.
*
* @param request
* @param sqr
* @return
*/
public SolrDocumentList SolrQueryResponse2SolrDocumentList(
final SolrQueryRequest req, final SolrQueryResponse rsp) {
SolrDocumentList sdl = new SolrDocumentList();
NamedList<?> nl = rsp.getValues();
ResultContext resultContext = (ResultContext) nl.get("response");
DocList response =
resultContext == null
? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f)
: resultContext.docs;
sdl.setNumFound(response == null ? 0 : response.matches());
sdl.setStart(response == null ? 0 : response.offset());
String originalName = Thread.currentThread().getName();
if (response != null) {
try {
SolrIndexSearcher searcher = req.getSearcher();
final int responseCount = response.size();
DocIterator iterator = response.iterator();
for (int i = 0; i < responseCount; i++) {
int docid = iterator.nextDoc();
Thread.currentThread()
.setName("EmbeddedSolrConnector.SolrQueryResponse2SolrDocumentList: " + docid);
Document responsedoc = searcher.doc(docid, (Set<String>) null);
SolrDocument sordoc = doc2SolrDoc(responsedoc);
sdl.add(sordoc);
}
} catch (IOException e) {
ConcurrentLog.logException(e);
}
}
Thread.currentThread().setName(originalName);
return sdl;
}
@Test
public void testCollateWithMultipleRequestHandlers() throws Exception {
SolrCore core = h.getCore();
SearchComponent speller = core.getSearchComponent("spellcheck");
assertTrue("speller is null and it shouldn't be", speller != null);
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(SpellCheckComponent.COMPONENT_NAME, "true");
params.add(SpellCheckComponent.SPELLCHECK_DICT, "multipleFields");
params.add(SpellCheckComponent.SPELLCHECK_BUILD, "true");
params.add(SpellCheckComponent.SPELLCHECK_COUNT, "10");
params.add(SpellCheckComponent.SPELLCHECK_COLLATE, "true");
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES, "1");
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "1");
params.add(CommonParams.Q, "peac");
// SpellCheckCompRH has no "qf" defined. It will not find "peace" from "peac" despite it being
// in the dictionary
// because requrying against this Request Handler results in 0 hits.
SolrRequestHandler handler = core.getRequestHandler("spellCheckCompRH");
SolrQueryResponse rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
SolrQueryRequest req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
req.close();
NamedList values = rsp.getValues();
NamedList spellCheck = (NamedList) values.get("spellcheck");
NamedList suggestions = (NamedList) spellCheck.get("suggestions");
String singleCollation = (String) suggestions.get("collation");
assertNull(singleCollation);
// SpellCheckCompRH1 has "lowerfilt1" defined in the "qf" param. It will find "peace" from
// "peac" because
// requrying field "lowerfilt1" returns the hit.
params.remove(SpellCheckComponent.SPELLCHECK_BUILD);
handler = core.getRequestHandler("spellCheckCompRH1");
rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
req.close();
values = rsp.getValues();
spellCheck = (NamedList) values.get("spellcheck");
suggestions = (NamedList) spellCheck.get("suggestions");
singleCollation = (String) suggestions.get("collation");
assertEquals(singleCollation, "peace");
}
@Override
public void processDelete(DeleteUpdateCommand cmd) throws IOException {
if (!cmd.isDeleteById()) {
doDeleteByQuery(cmd);
return;
}
int hash = 0;
if (zkEnabled) {
zkCheck();
hash = hash(cmd);
nodes = setupRequest(hash);
} else {
isLeader = getNonZkLeaderAssumption(req);
}
boolean dropCmd = false;
if (!forwardToLeader) {
dropCmd = versionDelete(cmd);
}
if (dropCmd) {
// TODO: do we need to add anything to the response?
return;
}
ModifiableSolrParams params = null;
if (nodes != null) {
params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(
DISTRIB_UPDATE_PARAM,
(isLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString()));
if (isLeader) {
params.set(
"distrib.from",
ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
}
cmdDistrib.distribDelete(cmd, nodes, params);
}
// cmd.getIndexId == null when delete by query
// TODO: what to do when no idField?
if (returnVersions && rsp != null && cmd.getIndexedId() != null && idField != null) {
if (deleteResponse == null) {
deleteResponse = new NamedList<String>();
rsp.add("deletes", deleteResponse);
}
if (scratch == null) scratch = new CharsRef();
idField.getType().indexedToReadable(cmd.getIndexedId(), scratch);
deleteResponse.add(
scratch.toString(),
cmd
.getVersion()); // we're returning the version of the delete.. not the version of the
// doc we deleted.
}
}
// TODO: optionally fail if n replicas are not reached...
private void doFinish() {
// TODO: if not a forward and replication req is not specified, we could
// send in a background thread
cmdDistrib.finish();
Response response = cmdDistrib.getResponse();
// TODO - we may need to tell about more than one error...
// if its a forward, any fail is a problem -
// otherwise we assume things are fine if we got it locally
// until we start allowing min replication param
if (response.errors.size() > 0) {
// if one node is a RetryNode, this was a forward request
if (response.errors.get(0).node instanceof RetryNode) {
rsp.setException(response.errors.get(0).e);
}
// else
// for now we don't error - we assume if it was added locally, we
// succeeded
}
// if it is not a forward request, for each fail, try to tell them to
// recover - the doc was already added locally, so it should have been
// legit
// TODO: we should do this in the background it would seem
for (SolrCmdDistributor.Error error : response.errors) {
if (error.node instanceof RetryNode || error.e instanceof SolrException) {
// we don't try to force a leader to recover
// when we cannot forward to it
// and we assume SolrException means
// the node went down
continue;
}
// TODO: we should force their state to recovering ??
// TODO: could be sent in parallel
// TODO: do retries??
// TODO: what if its is already recovering? Right now recoveries queue up -
// should they?
String recoveryUrl = error.node.getBaseUrl();
HttpSolrServer server;
log.info("try and ask " + recoveryUrl + " to recover");
try {
server = new HttpSolrServer(recoveryUrl);
server.setSoTimeout(5000);
server.setConnectionTimeout(5000);
RequestRecovery recoverRequestCmd = new RequestRecovery();
recoverRequestCmd.setAction(CoreAdminAction.REQUESTRECOVERY);
recoverRequestCmd.setCoreName(error.node.getCoreName());
server.request(recoverRequestCmd);
} catch (Exception e) {
log.info("Could not tell a replica to recover", e);
}
}
}
@Override
public void processAdd(AddUpdateCommand cmd) throws IOException {
// TODO: check for id field?
int hash = 0;
if (zkEnabled) {
zkCheck();
hash = hash(cmd);
nodes = setupRequest(hash);
} else {
isLeader = getNonZkLeaderAssumption(req);
}
boolean dropCmd = false;
if (!forwardToLeader) {
dropCmd = versionAdd(cmd);
}
if (dropCmd) {
// TODO: do we need to add anything to the response?
return;
}
ModifiableSolrParams params = null;
if (nodes != null) {
params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(
DISTRIB_UPDATE_PARAM,
(isLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString()));
if (isLeader) {
params.set(
"distrib.from",
ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
}
params.set(
"distrib.from",
ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
cmdDistrib.distribAdd(cmd, nodes, params);
}
// TODO: what to do when no idField?
if (returnVersions && rsp != null && idField != null) {
if (addsResponse == null) {
addsResponse = new NamedList<String>();
rsp.add("adds", addsResponse);
}
if (scratch == null) scratch = new CharsRef();
idField.getType().indexedToReadable(cmd.getIndexedId(), scratch);
addsResponse.add(scratch.toString(), cmd.getVersion());
}
// TODO: keep track of errors? needs to be done at a higher level though since
// an id may fail before it gets to this processor.
// Given that, it may also make sense to move the version reporting out of this
// processor too.
}
@Test
public void testComponent() throws Exception {
SolrCore core = h.getCore();
SearchComponent sc = core.getSearchComponent("clustering");
assertTrue("sc is null and it shouldn't be", sc != null);
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(ClusteringComponent.COMPONENT_NAME, "true");
params.add(CommonParams.Q, "*:*");
params.add(ClusteringParams.USE_SEARCH_RESULTS, "true");
SolrRequestHandler handler = core.getRequestHandler("standard");
SolrQueryResponse rsp;
rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
SolrQueryRequest req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
NamedList values = rsp.getValues();
Object clusters = values.get("clusters");
// System.out.println("Clusters: " + clusters);
assertTrue("clusters is null and it shouldn't be", clusters != null);
req.close();
params = new ModifiableSolrParams();
params.add(ClusteringComponent.COMPONENT_NAME, "true");
params.add(ClusteringParams.ENGINE_NAME, "mock");
params.add(ClusteringParams.USE_COLLECTION, "true");
params.add(QueryComponent.COMPONENT_NAME, "false");
handler = core.getRequestHandler("docClustering");
rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
values = rsp.getValues();
clusters = values.get("clusters");
// System.out.println("Clusters: " + clusters);
assertTrue("clusters is null and it shouldn't be", clusters != null);
req.close();
}
private void handlePOST() throws IOException {
Iterable<ContentStream> streams = req.getContentStreams();
if (streams == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "missing content stream");
}
ArrayList<CommandOperation> ops = new ArrayList<>();
for (ContentStream stream : streams) ops.addAll(CommandOperation.parse(stream.getReader()));
List<Map> errList = CommandOperation.captureErrors(ops);
if (!errList.isEmpty()) {
resp.add(CommandOperation.ERR_MSGS, errList);
return;
}
try {
for (; ; ) {
ArrayList<CommandOperation> opsCopy = new ArrayList<>(ops.size());
for (CommandOperation op : ops) opsCopy.add(op.getCopy());
try {
if (parts.size() > 1 && RequestParams.NAME.equals(parts.get(1))) {
RequestParams params =
RequestParams.getFreshRequestParams(
req.getCore().getResourceLoader(),
req.getCore().getSolrConfig().getRequestParams());
handleParams(opsCopy, params);
} else {
ConfigOverlay overlay =
SolrConfig.getConfigOverlay(req.getCore().getResourceLoader());
handleCommands(opsCopy, overlay);
}
break; // succeeded . so no need to go over the loop again
} catch (ZkController.ResourceModifiedInZkException e) {
// retry
log.info(
"Race condition, the node is modified in ZK by someone else " + e.getMessage());
}
}
} catch (Exception e) {
resp.setException(e);
resp.add(CommandOperation.ERR_MSGS, singletonList(SchemaManager.getErrorStr(e)));
}
}
@Test
public void testExtractOnly() throws Exception {
ExtractingRequestHandler handler =
(ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");
assertTrue("handler is null and it shouldn't be", handler != null);
SolrQueryResponse rsp =
loadLocal("extraction/solr-word.pdf", ExtractingParams.EXTRACT_ONLY, "true");
assertTrue("rsp is null and it shouldn't be", rsp != null);
NamedList list = rsp.getValues();
String extraction = (String) list.get("solr-word.pdf");
assertTrue("extraction is null and it shouldn't be", extraction != null);
assertTrue(
extraction + " does not contain " + "solr-word", extraction.indexOf("solr-word") != -1);
NamedList nl = (NamedList) list.get("solr-word.pdf_metadata");
assertTrue("nl is null and it shouldn't be", nl != null);
Object title = nl.get("title");
assertTrue("title is null and it shouldn't be", title != null);
assertTrue(extraction.indexOf("<?xml") != -1);
rsp =
loadLocal(
"extraction/solr-word.pdf",
ExtractingParams.EXTRACT_ONLY,
"true",
ExtractingParams.EXTRACT_FORMAT,
ExtractingDocumentLoader.TEXT_FORMAT);
assertTrue("rsp is null and it shouldn't be", rsp != null);
list = rsp.getValues();
extraction = (String) list.get("solr-word.pdf");
assertTrue("extraction is null and it shouldn't be", extraction != null);
assertTrue(
extraction + " does not contain " + "solr-word", extraction.indexOf("solr-word") != -1);
assertTrue(extraction.indexOf("<?xml") == -1);
nl = (NamedList) list.get("solr-word.pdf_metadata");
assertTrue("nl is null and it shouldn't be", nl != null);
title = nl.get("title");
assertTrue("title is null and it shouldn't be", title != null);
}
@Test
public void testXPath() throws Exception {
ExtractingRequestHandler handler =
(ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");
assertTrue("handler is null and it shouldn't be", handler != null);
SolrQueryResponse rsp =
loadLocal(
"extraction/example.html",
ExtractingParams.XPATH_EXPRESSION,
"/xhtml:html/xhtml:body/xhtml:a/descendant:node()",
ExtractingParams.EXTRACT_ONLY,
"true");
assertTrue("rsp is null and it shouldn't be", rsp != null);
NamedList list = rsp.getValues();
String val = (String) list.get("example.html");
val = val.trim();
assertTrue(
val + " is not equal to " + "linkNews",
val.equals("linkNews") == true); // there are two <a> tags, and they get collapesd
}
protected void sendError(Throwable ex) throws IOException {
Exception exp = null;
SolrCore localCore = null;
try {
SolrQueryResponse solrResp = new SolrQueryResponse();
if (ex instanceof Exception) {
solrResp.setException((Exception) ex);
} else {
solrResp.setException(new RuntimeException(ex));
}
localCore = core;
if (solrReq == null) {
final SolrParams solrParams;
if (req != null) {
// use GET parameters if available:
solrParams = SolrRequestParsers.parseQueryString(req.getQueryString());
} else {
// we have no params at all, use empty ones:
solrParams = new MapSolrParams(Collections.<String, String>emptyMap());
}
solrReq = new SolrQueryRequestBase(core, solrParams) {};
}
QueryResponseWriter writer = core.getQueryResponseWriter(solrReq);
writeResponse(solrResp, writer, Method.GET);
} catch (Exception e) { // This error really does not matter
exp = e;
} finally {
try {
if (exp != null) {
SimpleOrderedMap info = new SimpleOrderedMap();
int code = ResponseUtils.getErrorInfo(ex, info, log);
sendError(code, info.toString());
}
} finally {
if (core == null && localCore != null) {
localCore.close();
}
}
}
}
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
SimpleOrderedMap<Object> system = new SimpleOrderedMap<Object>();
rsp.add("system", system);
ThreadMXBean tmbean = ManagementFactory.getThreadMXBean();
// Thread Count
SimpleOrderedMap<Object> nl = new SimpleOrderedMap<Object>();
nl.add("current", tmbean.getThreadCount());
nl.add("peak", tmbean.getPeakThreadCount());
nl.add("daemon", tmbean.getDaemonThreadCount());
system.add("threadCount", nl);
// Deadlocks
ThreadInfo[] tinfos;
long[] tids = tmbean.findMonitorDeadlockedThreads();
if (tids != null) {
tinfos = tmbean.getThreadInfo(tids, Integer.MAX_VALUE);
NamedList<SimpleOrderedMap<Object>> lst = new NamedList<SimpleOrderedMap<Object>>();
for (ThreadInfo ti : tinfos) {
if (ti != null) {
lst.add("thread", getThreadInfo(ti, tmbean));
}
}
system.add("deadlocks", lst);
}
// Now show all the threads....
tids = tmbean.getAllThreadIds();
tinfos = tmbean.getThreadInfo(tids, Integer.MAX_VALUE);
NamedList<SimpleOrderedMap<Object>> lst = new NamedList<SimpleOrderedMap<Object>>();
for (ThreadInfo ti : tinfos) {
if (ti != null) {
lst.add("thread", getThreadInfo(ti, tmbean));
}
}
system.add("threadDump", lst);
rsp.setHttpCaching(false);
}
/** Writes the response to the search query to the HTTP response's output stream. */
private void writeResponse(
SolrQuery query,
HttpServletRequest httpRequest,
HttpServletResponse httpResponse,
QueryResponse queryResponse)
throws IOException {
SolrCore core = SearchActivator.getInstance().getSolrCore();
// this seems to be the only way to obtain the JSON response representation
SolrQueryRequest solrRequest = new LocalSolrQueryRequest(core, query.toNamedList());
SolrQueryResponse solrResponse = new SolrQueryResponse();
// bash the query in the response to remove user info
NamedList<Object> params =
(NamedList<Object>) queryResponse.getHeader().get("params"); // $NON-NLS-1$
params.remove(CommonParams.Q);
params.add(CommonParams.Q, httpRequest.getParameter(CommonParams.Q));
NamedList<Object> values = queryResponse.getResponse();
String contextPath = httpRequest.getContextPath();
if (contextPath.length() > 0) setSearchResultContext(values, contextPath);
solrResponse.setAllValues(values);
QueryResponseWriter writer = core.getQueryResponseWriter("json"); // $NON-NLS-1$
writer.write(httpResponse.getWriter(), solrRequest, solrResponse);
}
public DocListSearcher(
final String querystring,
String sort,
final int offset,
final int count,
final String... fields) {
// construct query
final SolrQuery params =
AbstractSolrConnector.getSolrQuery(querystring, sort, offset, count, fields);
// query the server
this.request = EmbeddedSolrConnector.this.request(params);
SolrQueryResponse rsp = query(request);
NamedList<?> nl = rsp.getValues();
ResultContext resultContext = (ResultContext) nl.get("response");
if (resultContext == null)
log.warn("DocListSearcher: no response for query '" + querystring + "'");
this.response =
resultContext == null
? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f)
: resultContext.docs;
}
public SolrQueryResponse query(SolrQueryRequest req) throws SolrException {
final long startTime = System.currentTimeMillis();
// during the solr query we set the thread name to the query string to get more debugging info
// in thread dumps
String threadname = Thread.currentThread().getName();
String ql = "";
try {
ql = URLDecoder.decode(req.getParams().toString(), StandardCharsets.UTF_8.name());
} catch (UnsupportedEncodingException e) {
}
Thread.currentThread().setName("solr query: " + ql); // for debugging in Threaddump
ConcurrentLog.info("EmbeddedSolrConnector.query", "QUERY: " + ql);
// System.out.println("EmbeddedSolrConnector.query * QUERY: " + ql);
// System.out.println("STACKTRACE: " + ConcurrentLog.stackTrace());
SolrQueryResponse rsp = new SolrQueryResponse();
NamedList<Object> responseHeader = new SimpleOrderedMap<Object>();
responseHeader.add("params", req.getOriginalParams().toNamedList());
rsp.add("responseHeader", responseHeader);
// SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
// send request to solr and create a result
this.requestHandler.handleRequest(req, rsp);
// get statistics and add a header with that
Exception exception = rsp.getException();
int status =
exception == null
? 0
: exception instanceof SolrException ? ((SolrException) exception).code() : 500;
responseHeader.add("status", status);
responseHeader.add("QTime", (int) (System.currentTimeMillis() - startTime));
Thread.currentThread().setName(threadname);
// return result
return rsp;
}
@Test
public void testGroupCollapseNoGroups() throws IOException {
NamedList values = mock(NamedList.class);
when(rsp.getValues()).thenReturn(values);
when(values.get("grouped")).thenReturn(null);
when(rb.grouping()).thenReturn(true);
when(params.getBool(GroupCollapseParams.GROUP_COLLAPSE, false)).thenReturn(true);
when(params.get(GroupCollapseParams.GROUP_COLLAPSE_FL)).thenReturn("price,discount,isCloseout");
component.process(rb);
verify(rsp, never()).add(anyString(), anyObject());
}
