@Test
public void testNaNInf() throws IOException {
SolrQueryRequest req = req("dummy");
SolrQueryResponse rsp = new SolrQueryResponse();
QueryResponseWriter w = new PythonResponseWriter();
StringWriter buf = new StringWriter();
rsp.add("data1", Float.NaN);
rsp.add("data2", Double.NEGATIVE_INFINITY);
rsp.add("data3", Float.POSITIVE_INFINITY);
w.write(buf, req, rsp);
assertEquals(
buf.toString(), "{'data1':float('NaN'),'data2':-float('Inf'),'data3':float('Inf')}");
w = new RubyResponseWriter();
buf = new StringWriter();
w.write(buf, req, rsp);
assertEquals(buf.toString(), "{'data1'=>(0.0/0.0),'data2'=>-(1.0/0.0),'data3'=>(1.0/0.0)}");
w = new JSONResponseWriter();
buf = new StringWriter();
w.write(buf, req, rsp);
assertEquals(
buf.toString(), "{\"data1\":\"NaN\",\"data2\":\"-Infinity\",\"data3\":\"Infinity\"}");
req.close();
}
private void handleGET() {
if (parts.size() == 1) {
resp.add("solrConfig", req.getCore().getSolrConfig().toMap());
} else {
if (ConfigOverlay.NAME.equals(parts.get(1))) {
resp.add(ConfigOverlay.NAME, req.getCore().getSolrConfig().getOverlay().toMap());
} else if (RequestParams.NAME.equals(parts.get(1))) {
if (parts.size() == 3) {
RequestParams params = req.getCore().getSolrConfig().getRequestParams();
MapSolrParams p = params.getParams(parts.get(2));
Map m = new LinkedHashMap<>();
m.put(ConfigOverlay.ZNODEVER, params.getZnodeVersion());
if (p != null) {
m.put(RequestParams.NAME, ZkNodeProps.makeMap(parts.get(2), p.getMap()));
}
resp.add(SolrQueryResponse.NAME, m);
} else {
resp.add(
SolrQueryResponse.NAME, req.getCore().getSolrConfig().getRequestParams().toMap());
}
} else {
Map<String, Object> m = req.getCore().getSolrConfig().toMap();
resp.add("solrConfig", ZkNodeProps.makeMap(parts.get(1), m.get(parts.get(1))));
}
}
}
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
SolrParams params = req.getParams();
params = adjustParams(params);
req.setParams(params);
TupleStream tupleStream = null;
try {
tupleStream = this.streamFactory.constructStream(params.get("expr"));
} catch (Exception e) {
// Catch exceptions that occur while the stream is being created. This will include streaming
// expression parse rules.
SolrException.log(logger, e);
rsp.add("result-set", new DummyErrorStream(e));
return;
}
int worker = params.getInt("workerID", 0);
int numWorkers = params.getInt("numWorkers", 1);
StreamContext context = new StreamContext();
context.workerID = worker;
context.numWorkers = numWorkers;
context.setSolrClientCache(clientCache);
tupleStream.setStreamContext(context);
rsp.add("result-set", new TimerStream(new ExceptionStream(tupleStream)));
}
private void handleAdmin(SolrQueryRequest req, SolrQueryResponse rsp, SolrParams params) {
String action = params.get("action");
if ("stop".equalsIgnoreCase(action)) {
String id = params.get("id");
DaemonStream d = daemons.get(id);
if (d != null) {
d.close();
rsp.add("result-set", new DaemonResponseStream("Deamon:" + id + " stopped on " + coreName));
} else {
rsp.add(
"result-set", new DaemonResponseStream("Deamon:" + id + " not found on " + coreName));
}
} else if ("start".equalsIgnoreCase(action)) {
String id = params.get("id");
DaemonStream d = daemons.get(id);
d.open();
rsp.add("result-set", new DaemonResponseStream("Deamon:" + id + " started on " + coreName));
} else if ("list".equalsIgnoreCase(action)) {
Collection<DaemonStream> vals = daemons.values();
rsp.add("result-set", new DaemonCollectionStream(vals));
} else if ("kill".equalsIgnoreCase(action)) {
String id = params.get("id");
DaemonStream d = daemons.remove(id);
if (d != null) {
d.close();
}
rsp.add("result-set", new DaemonResponseStream("Deamon:" + id + " killed on " + coreName));
}
}
private void doCompact(
LazyDocumentLoader documentLoader,
LazyCompactor compactor,
SolrQueryResponse rsp,
String tsId,
String q)
throws IOException, SyntaxError {
Query query = depProvider.parser(q).getQuery();
Iterable<Document> docs = documentLoader.load(query, SORT);
Iterable<CompactionResult> compactionResults = compactor.compact(docs);
List<Document> docsToDelete = new LinkedList<>();
List<SolrInputDocument> docsToAdd = new LinkedList<>();
compactionResults.forEach(
it -> {
docsToDelete.addAll(it.getInputDocuments());
docsToAdd.addAll(it.getOutputDocuments());
});
depProvider.solrUpdateService().add(docsToAdd);
depProvider.solrUpdateService().delete(docsToDelete);
rsp.add("timeseries " + tsId + " oldNumDocs:", docsToDelete.size());
rsp.add("timeseries " + tsId + " newNumDocs:", docsToAdd.size());
}
private void writeResponse(
SolrQueryResponse solrRsp, QueryResponseWriter responseWriter, Method reqMethod)
throws IOException {
try {
Object invalidStates = solrReq.getContext().get(CloudSolrClient.STATE_VERSION);
// This is the last item added to the response and the client would expect it that way.
// If that assumption is changed , it would fail. This is done to avoid an O(n) scan on
// the response for each request
if (invalidStates != null) solrRsp.add(CloudSolrClient.STATE_VERSION, invalidStates);
// Now write it out
final String ct = responseWriter.getContentType(solrReq, solrRsp);
// don't call setContentType on null
if (null != ct) response.setContentType(ct);
if (solrRsp.getException() != null) {
NamedList info = new SimpleOrderedMap();
int code = ResponseUtils.getErrorInfo(solrRsp.getException(), info, log);
solrRsp.add("error", info);
response.setStatus(code);
}
if (Method.HEAD != reqMethod) {
OutputStream out =
new CloseShieldOutputStream(
response.getOutputStream()); // Prevent close of container streams, see SOLR-8933
QueryResponseWriterUtil.writeQueryResponse(out, responseWriter, solrReq, solrRsp, ct);
}
// else http HEAD request, nothing to write out, waited this long just to get ContentType
} catch (EOFException e) {
log.info("Unable to write response, client closed connection or we are shutting down", e);
}
}
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
SolrParams params = req.getParams();
params = adjustParams(params);
req.setParams(params);
if (params.get("action") != null) {
handleAdmin(req, rsp, params);
return;
}
TupleStream tupleStream;
try {
tupleStream = this.streamFactory.constructStream(params.get("expr"));
} catch (Exception e) {
// Catch exceptions that occur while the stream is being created. This will include streaming
// expression parse rules.
SolrException.log(logger, e);
rsp.add("result-set", new DummyErrorStream(e));
return;
}
int worker = params.getInt("workerID", 0);
int numWorkers = params.getInt("numWorkers", 1);
StreamContext context = new StreamContext();
context.workerID = worker;
context.numWorkers = numWorkers;
context.setSolrClientCache(clientCache);
context.setModelCache(modelCache);
context.put("core", this.coreName);
context.put("solr-core", req.getCore());
tupleStream.setStreamContext(context);
// if asking for explanation then go get it
if (params.getBool("explain", false)) {
rsp.add("explanation", tupleStream.toExplanation(this.streamFactory));
}
if (tupleStream instanceof DaemonStream) {
DaemonStream daemonStream = (DaemonStream) tupleStream;
if (daemons.containsKey(daemonStream.getId())) {
daemons.remove(daemonStream.getId()).close();
}
daemonStream.setDaemons(daemons);
daemonStream.open(); // This will start the deamonStream
daemons.put(daemonStream.getId(), daemonStream);
rsp.add(
"result-set",
new DaemonResponseStream("Deamon:" + daemonStream.getId() + " started on " + coreName));
} else {
rsp.add("result-set", new TimerStream(new ExceptionStream(tupleStream)));
}
}
public void transform(
Map<String, ?> result,
SolrQueryResponse response,
GroupingSpecification groupingSpecification,
SolrDocumentSource solrDocumentSource) {
Object value = result.get(groupingSpecification.getFields()[0]);
if (TopGroups.class.isInstance(value)) {
@SuppressWarnings("unchecked")
TopGroups<BytesRef> topGroups = (TopGroups<BytesRef>) value;
SolrDocumentList docList = new SolrDocumentList();
docList.setStart(groupingSpecification.getOffset());
docList.setNumFound(topGroups.totalHitCount);
Float maxScore = Float.NEGATIVE_INFINITY;
for (GroupDocs<BytesRef> group : topGroups.groups) {
for (ScoreDoc scoreDoc : group.scoreDocs) {
if (maxScore < scoreDoc.score) {
maxScore = scoreDoc.score;
}
docList.add(solrDocumentSource.retrieve(scoreDoc));
}
}
if (maxScore != Float.NEGATIVE_INFINITY) {
docList.setMaxScore(maxScore);
}
response.add("response", docList);
}
}
@Test
public void testCollateWithFilter() throws Exception {
SolrCore core = h.getCore();
SearchComponent speller = core.getSearchComponent("spellcheck");
assertTrue("speller is null and it shouldn't be", speller != null);
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(SpellCheckComponent.COMPONENT_NAME, "true");
params.add(SpellCheckComponent.SPELLCHECK_BUILD, "true");
params.add(SpellCheckComponent.SPELLCHECK_COUNT, "10");
params.add(SpellCheckComponent.SPELLCHECK_COLLATE, "true");
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES, "5");
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "2");
params.add(CommonParams.Q, "lowerfilt:(+fauth +home +loane)");
params.add(CommonParams.FQ, "NOT(id:1)");
// Because a FilterQuery is applied which removes doc id#1 from possible hits, we would
// not want the collations to return us "lowerfilt:(+faith +hope +loaves)" as this only matches
// doc id#1.
SolrRequestHandler handler = core.getRequestHandler("spellCheckCompRH");
SolrQueryResponse rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
SolrQueryRequest req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
req.close();
NamedList values = rsp.getValues();
NamedList spellCheck = (NamedList) values.get("spellcheck");
NamedList suggestions = (NamedList) spellCheck.get("suggestions");
List<String> collations = suggestions.getAll("collation");
assertTrue(collations.size() == 1);
assertTrue(collations.get(0).equals("lowerfilt:(+faith +hope +love)"));
}
protected void writeResponse(
SolrQueryResponse solrRsp, QueryResponseWriter responseWriter, SolrQueryRequest solrReq)
throws IOException {
Object invalidStates = solrReq.getContext().get(CloudSolrClient.STATE_VERSION);
// This is the last item added to the applyResult and the client would expect it that way.
// If that assumption is changed , it would fail. This is done to avoid an O(n) scan on
// the applyResult for each request
if (invalidStates != null) solrRsp.add(CloudSolrClient.STATE_VERSION, invalidStates);
// Now write it out
final String ct = responseWriter.getContentType(solrReq, solrRsp);
// don't call setContentType on null
if (null != ct) {
responseSetter.setContentType(ct);
}
if (solrRsp.getException() != null) {
NamedList info = new SimpleOrderedMap();
int code = ResponseUtils.getErrorInfo(solrRsp.getException(), info, logger);
// solrRsp.add("error", info);
// use protocol response exception instead of set 'error' response return to client,
responseSetter.setSolrResponseException(code, info);
}
QueryResponseWriterUtil.writeQueryResponse(
responseSetter.getResponseOutputStream(), responseWriter, solrReq, solrRsp, ct);
// fire QueryResponse write Complete
responseSetter.writeQueryResponseComplete(solrRsp);
}
@Test
public void testCollateWithMultipleRequestHandlers() throws Exception {
SolrCore core = h.getCore();
SearchComponent speller = core.getSearchComponent("spellcheck");
assertTrue("speller is null and it shouldn't be", speller != null);
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(SpellCheckComponent.COMPONENT_NAME, "true");
params.add(SpellCheckComponent.SPELLCHECK_DICT, "multipleFields");
params.add(SpellCheckComponent.SPELLCHECK_BUILD, "true");
params.add(SpellCheckComponent.SPELLCHECK_COUNT, "10");
params.add(SpellCheckComponent.SPELLCHECK_COLLATE, "true");
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES, "1");
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "1");
params.add(CommonParams.Q, "peac");
// SpellCheckCompRH has no "qf" defined. It will not find "peace" from "peac" despite it being
// in the dictionary
// because requrying against this Request Handler results in 0 hits.
SolrRequestHandler handler = core.getRequestHandler("spellCheckCompRH");
SolrQueryResponse rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
SolrQueryRequest req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
req.close();
NamedList values = rsp.getValues();
NamedList spellCheck = (NamedList) values.get("spellcheck");
NamedList suggestions = (NamedList) spellCheck.get("suggestions");
String singleCollation = (String) suggestions.get("collation");
assertNull(singleCollation);
// SpellCheckCompRH1 has "lowerfilt1" defined in the "qf" param. It will find "peace" from
// "peac" because
// requrying field "lowerfilt1" returns the hit.
params.remove(SpellCheckComponent.SPELLCHECK_BUILD);
handler = core.getRequestHandler("spellCheckCompRH1");
rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
req.close();
values = rsp.getValues();
spellCheck = (NamedList) values.get("spellcheck");
suggestions = (NamedList) spellCheck.get("suggestions");
singleCollation = (String) suggestions.get("collation");
assertEquals(singleCollation, "peace");
}
public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) {
numRequests++;
Query query = null;
Filter filter = null;
List<String> commands = StrUtils.splitSmart(req.getQueryString(), ';');
String qs = commands.size() >= 1 ? commands.get(0) : "";
query = QueryParsing.parseQuery(qs, req.getSchema());
// If the first non-query, non-filter command is a simple sort on an indexed field, then
// we can use the Lucene sort ability.
Sort sort = null;
if (commands.size() >= 2) {
sort = QueryParsing.parseSort(commands.get(1), req);
}
try {
int numHits;
ScoreDoc[] scoreDocs;
if (sort != null) {
TopFieldDocs hits =
req.getSearcher().search(query, filter, req.getStart() + req.getLimit(), sort);
scoreDocs = hits.scoreDocs;
numHits = hits.totalHits;
} else {
TopDocs hits = req.getSearcher().search(query, filter, req.getStart() + req.getLimit());
scoreDocs = hits.scoreDocs;
numHits = hits.totalHits;
}
int startRow = Math.min(numHits, req.getStart());
int endRow = Math.min(numHits, req.getStart() + req.getLimit());
int numRows = endRow - startRow;
int[] ids = new int[numRows];
Document[] data = new Document[numRows];
for (int i = startRow; i < endRow; i++) {
ids[i] = scoreDocs[i].doc;
data[i] = req.getSearcher().doc(ids[i]);
}
rsp.add(null, new DocSlice(0, numRows, ids, null, numHits, 0.0f));
/**
* ********************* rsp.setResults(new DocSlice(0,numRows,ids,null,numHits));
*
* <p>// Setting the actual document objects is optional rsp.setResults(data);
* **********************
*/
} catch (IOException e) {
rsp.setException(e);
numErrors++;
return;
}
}
@Override
public void processDelete(DeleteUpdateCommand cmd) throws IOException {
if (!cmd.isDeleteById()) {
doDeleteByQuery(cmd);
return;
}
int hash = 0;
if (zkEnabled) {
zkCheck();
hash = hash(cmd);
nodes = setupRequest(hash);
} else {
isLeader = getNonZkLeaderAssumption(req);
}
boolean dropCmd = false;
if (!forwardToLeader) {
dropCmd = versionDelete(cmd);
}
if (dropCmd) {
// TODO: do we need to add anything to the response?
return;
}
ModifiableSolrParams params = null;
if (nodes != null) {
params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(
DISTRIB_UPDATE_PARAM,
(isLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString()));
if (isLeader) {
params.set(
"distrib.from",
ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
}
cmdDistrib.distribDelete(cmd, nodes, params);
}
// cmd.getIndexId == null when delete by query
// TODO: what to do when no idField?
if (returnVersions && rsp != null && cmd.getIndexedId() != null && idField != null) {
if (deleteResponse == null) {
deleteResponse = new NamedList<String>();
rsp.add("deletes", deleteResponse);
}
if (scratch == null) scratch = new CharsRef();
idField.getType().indexedToReadable(cmd.getIndexedId(), scratch);
deleteResponse.add(
scratch.toString(),
cmd
.getVersion()); // we're returning the version of the delete.. not the version of the
// doc we deleted.
}
}
@Override
public void processAdd(AddUpdateCommand cmd) throws IOException {
// TODO: check for id field?
int hash = 0;
if (zkEnabled) {
zkCheck();
hash = hash(cmd);
nodes = setupRequest(hash);
} else {
isLeader = getNonZkLeaderAssumption(req);
}
boolean dropCmd = false;
if (!forwardToLeader) {
dropCmd = versionAdd(cmd);
}
if (dropCmd) {
// TODO: do we need to add anything to the response?
return;
}
ModifiableSolrParams params = null;
if (nodes != null) {
params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(
DISTRIB_UPDATE_PARAM,
(isLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString()));
if (isLeader) {
params.set(
"distrib.from",
ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
}
params.set(
"distrib.from",
ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
cmdDistrib.distribAdd(cmd, nodes, params);
}
// TODO: what to do when no idField?
if (returnVersions && rsp != null && idField != null) {
if (addsResponse == null) {
addsResponse = new NamedList<String>();
rsp.add("adds", addsResponse);
}
if (scratch == null) scratch = new CharsRef();
idField.getType().indexedToReadable(cmd.getIndexedId(), scratch);
addsResponse.add(scratch.toString(), cmd.getVersion());
}
// TODO: keep track of errors? needs to be done at a higher level though since
// an id may fail before it gets to this processor.
// Given that, it may also make sense to move the version reporting out of this
// processor too.
}
@Test
public void testComponent() throws Exception {
SolrCore core = h.getCore();
SearchComponent sc = core.getSearchComponent("clustering");
assertTrue("sc is null and it shouldn't be", sc != null);
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(ClusteringComponent.COMPONENT_NAME, "true");
params.add(CommonParams.Q, "*:*");
params.add(ClusteringParams.USE_SEARCH_RESULTS, "true");
SolrRequestHandler handler = core.getRequestHandler("standard");
SolrQueryResponse rsp;
rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
SolrQueryRequest req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
NamedList values = rsp.getValues();
Object clusters = values.get("clusters");
// System.out.println("Clusters: " + clusters);
assertTrue("clusters is null and it shouldn't be", clusters != null);
req.close();
params = new ModifiableSolrParams();
params.add(ClusteringComponent.COMPONENT_NAME, "true");
params.add(ClusteringParams.ENGINE_NAME, "mock");
params.add(ClusteringParams.USE_COLLECTION, "true");
params.add(QueryComponent.COMPONENT_NAME, "false");
handler = core.getRequestHandler("docClustering");
rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
values = rsp.getValues();
clusters = values.get("clusters");
// System.out.println("Clusters: " + clusters);
assertTrue("clusters is null and it shouldn't be", clusters != null);
req.close();
}
private void handlePOST() throws IOException {
Iterable<ContentStream> streams = req.getContentStreams();
if (streams == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "missing content stream");
}
ArrayList<CommandOperation> ops = new ArrayList<>();
for (ContentStream stream : streams) ops.addAll(CommandOperation.parse(stream.getReader()));
List<Map> errList = CommandOperation.captureErrors(ops);
if (!errList.isEmpty()) {
resp.add(CommandOperation.ERR_MSGS, errList);
return;
}
try {
for (; ; ) {
ArrayList<CommandOperation> opsCopy = new ArrayList<>(ops.size());
for (CommandOperation op : ops) opsCopy.add(op.getCopy());
try {
if (parts.size() > 1 && RequestParams.NAME.equals(parts.get(1))) {
RequestParams params =
RequestParams.getFreshRequestParams(
req.getCore().getResourceLoader(),
req.getCore().getSolrConfig().getRequestParams());
handleParams(opsCopy, params);
} else {
ConfigOverlay overlay =
SolrConfig.getConfigOverlay(req.getCore().getResourceLoader());
handleCommands(opsCopy, overlay);
}
break; // succeeded . so no need to go over the loop again
} catch (ZkController.ResourceModifiedInZkException e) {
// retry
log.info(
"Race condition, the node is modified in ZK by someone else " + e.getMessage());
}
}
} catch (Exception e) {
resp.setException(e);
resp.add(CommandOperation.ERR_MSGS, singletonList(SchemaManager.getErrorStr(e)));
}
}
public void testJSON() throws IOException {
SolrQueryRequest req = req("wt", "json", "json.nl", "arrarr");
SolrQueryResponse rsp = new SolrQueryResponse();
JSONResponseWriter w = new JSONResponseWriter();
StringWriter buf = new StringWriter();
NamedList nl = new NamedList();
nl.add("data1", "hello");
nl.add(null, 42);
rsp.add("nl", nl);
w.write(buf, req, rsp);
assertEquals(buf.toString(), "{\"nl\":[[\"data1\",\"hello\"],[null,42]]}");
req.close();
}
/**
* Handler "PERSIST" action
*
* @param req
* @param rsp
* @return true if a modification has resulted that requires persistance of the CoreContainer
* configuration.
* @throws SolrException
*/
protected boolean handlePersistAction(SolrQueryRequest req, SolrQueryResponse rsp)
throws SolrException {
SolrParams params = req.getParams();
boolean doPersist = false;
String fileName = params.get(CoreAdminParams.FILE);
if (fileName != null) {
File file = new File(coreContainer.getConfigFile().getParentFile(), fileName);
coreContainer.persistFile(file);
rsp.add("saved", file.getAbsolutePath());
doPersist = false;
} else if (!coreContainer.isPersistent()) {
throw new SolrException(SolrException.ErrorCode.FORBIDDEN, "Persistence is not enabled");
} else doPersist = true;
return doPersist;
}
private void handleCommands(List<CommandOperation> ops, ConfigOverlay overlay)
throws IOException {
for (CommandOperation op : ops) {
switch (op.name) {
case SET_PROPERTY:
overlay = applySetProp(op, overlay);
break;
case UNSET_PROPERTY:
overlay = applyUnset(op, overlay);
break;
case SET_USER_PROPERTY:
overlay = applySetUserProp(op, overlay);
break;
case UNSET_USER_PROPERTY:
overlay = applyUnsetUserProp(op, overlay);
break;
case UPDATE_REQHANDLER:
case CREATE_REQHANDLER:
overlay = applyRequestHandler(op, overlay);
break;
case DELETE_REQHANDLER:
overlay = applyDeleteHandler(op, overlay);
break;
}
}
List errs = CommandOperation.captureErrors(ops);
if (!errs.isEmpty()) {
resp.add(CommandOperation.ERR_MSGS, errs);
return;
}
SolrResourceLoader loader = req.getCore().getResourceLoader();
if (loader instanceof ZkSolrResourceLoader) {
ZkController.persistConfigResourceToZooKeeper(
loader,
overlay.getZnodeVersion(),
ConfigOverlay.RESOURCE_NAME,
overlay.toByteArray(),
true);
} else {
SolrResourceLoader.persistConfLocally(
loader, ConfigOverlay.RESOURCE_NAME, overlay.toByteArray());
req.getCore().getCoreDescriptor().getCoreContainer().reload(req.getCore().getName());
}
}
@Override
@SuppressWarnings("PMD.SignatureDeclareThrowsException")
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
String joinKey = req.getParams().get(JOIN_KEY);
String fq = req.getParams().get(FQ);
int ppc = req.getParams().getInt(POINTS_PER_CHUNK, 100000);
int pageSize = req.getParams().getInt(PAGE_SIZE, 100);
depProvider.init(req, rsp);
LazyCompactor compactor = depProvider.compactor(ppc, req.getSearcher().getSchema());
LazyDocumentLoader documentLoader = depProvider.documentLoader(pageSize, req.getSearcher());
if (isBlank(joinKey) && isBlank(fq)) {
LOGGER.error("Neither join key nor filter query given.");
rsp.add(
"error",
join(
"",
"Neither join key nor filter query given.",
"Get help at https://chronix.gitbooks.io/chronix/content/document_compaction.html."));
return;
}
// no join key => compact documents matching fq
if (isBlank(joinKey)) {
compact(documentLoader, compactor, rsp, fq, fq);
depProvider.solrUpdateService().commit();
return;
}
// determine time series identified by joinKey
SolrFacetService facetService = depProvider.solrFacetService();
Query filterQuery = isBlank(fq) ? new MatchAllDocsQuery() : depProvider.parser(fq).getQuery();
List<NamedList<Object>> pivotResult = facetService.pivot(joinKey, filterQuery);
// compact each time series' constituting documents
facetService
.toTimeSeriesIds(pivotResult)
.parallelStream()
.forEach(
tsId ->
compact(documentLoader, compactor, rsp, tsId.toString(), and(tsId.toQuery(), fq)));
depProvider.solrUpdateService().commit();
}
/**
* Handle 'CREATE' action.
*
* @param req
* @param rsp
* @return true if a modification has resulted that requires persistance of the CoreContainer
* configuration.
* @throws SolrException in case of a configuration error.
*/
protected boolean handleCreateAction(SolrQueryRequest req, SolrQueryResponse rsp)
throws SolrException {
try {
SolrParams params = req.getParams();
String name = params.get(CoreAdminParams.NAME);
CoreDescriptor dcore =
new CoreDescriptor(coreContainer, name, params.get(CoreAdminParams.INSTANCE_DIR));
// fillup optional parameters
String opts = params.get(CoreAdminParams.CONFIG);
if (opts != null) dcore.setConfigName(opts);
opts = params.get(CoreAdminParams.SCHEMA);
if (opts != null) dcore.setSchemaName(opts);
opts = params.get(CoreAdminParams.DATA_DIR);
if (opts != null) dcore.setDataDir(opts);
// Process all property.name=value parameters and set them as name=value core properties
Properties coreProperties = new Properties();
Iterator<String> parameterNamesIterator = params.getParameterNamesIterator();
while (parameterNamesIterator.hasNext()) {
String parameterName = parameterNamesIterator.next();
if (parameterName.startsWith(CoreAdminParams.PROPERTY_PREFIX)) {
String parameterValue = params.get(parameterName);
String propertyName =
parameterName.substring(CoreAdminParams.PROPERTY_PREFIX.length()); // skip prefix
coreProperties.put(propertyName, parameterValue);
}
}
dcore.setCoreProperties(coreProperties);
SolrCore core = coreContainer.create(dcore);
coreContainer.register(name, core, false);
rsp.add("core", core.getName());
return coreContainer.isPersistent();
} catch (Exception ex) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Error executing default implementation of CREATE",
ex);
}
}
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
SimpleOrderedMap<Object> system = new SimpleOrderedMap<Object>();
rsp.add("system", system);
ThreadMXBean tmbean = ManagementFactory.getThreadMXBean();
// Thread Count
SimpleOrderedMap<Object> nl = new SimpleOrderedMap<Object>();
nl.add("current", tmbean.getThreadCount());
nl.add("peak", tmbean.getPeakThreadCount());
nl.add("daemon", tmbean.getDaemonThreadCount());
system.add("threadCount", nl);
// Deadlocks
ThreadInfo[] tinfos;
long[] tids = tmbean.findMonitorDeadlockedThreads();
if (tids != null) {
tinfos = tmbean.getThreadInfo(tids, Integer.MAX_VALUE);
NamedList<SimpleOrderedMap<Object>> lst = new NamedList<SimpleOrderedMap<Object>>();
for (ThreadInfo ti : tinfos) {
if (ti != null) {
lst.add("thread", getThreadInfo(ti, tmbean));
}
}
system.add("deadlocks", lst);
}
// Now show all the threads....
tids = tmbean.getAllThreadIds();
tinfos = tmbean.getThreadInfo(tids, Integer.MAX_VALUE);
NamedList<SimpleOrderedMap<Object>> lst = new NamedList<SimpleOrderedMap<Object>>();
for (ThreadInfo ti : tinfos) {
if (ti != null) {
lst.add("thread", getThreadInfo(ti, tmbean));
}
}
system.add("threadDump", lst);
rsp.setHttpCaching(false);
}
public SolrQueryResponse query(SolrQueryRequest req) throws SolrException {
final long startTime = System.currentTimeMillis();
// during the solr query we set the thread name to the query string to get more debugging info
// in thread dumps
String threadname = Thread.currentThread().getName();
String ql = "";
try {
ql = URLDecoder.decode(req.getParams().toString(), StandardCharsets.UTF_8.name());
} catch (UnsupportedEncodingException e) {
}
Thread.currentThread().setName("solr query: " + ql); // for debugging in Threaddump
ConcurrentLog.info("EmbeddedSolrConnector.query", "QUERY: " + ql);
// System.out.println("EmbeddedSolrConnector.query * QUERY: " + ql);
// System.out.println("STACKTRACE: " + ConcurrentLog.stackTrace());
SolrQueryResponse rsp = new SolrQueryResponse();
NamedList<Object> responseHeader = new SimpleOrderedMap<Object>();
responseHeader.add("params", req.getOriginalParams().toNamedList());
rsp.add("responseHeader", responseHeader);
// SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
// send request to solr and create a result
this.requestHandler.handleRequest(req, rsp);
// get statistics and add a header with that
Exception exception = rsp.getException();
int status =
exception == null
? 0
: exception instanceof SolrException ? ((SolrException) exception).code() : 500;
responseHeader.add("status", status);
responseHeader.add("QTime", (int) (System.currentTimeMillis() - startTime));
Thread.currentThread().setName(threadname);
// return result
return rsp;
}
/**
* Handle "STATUS" action
*
* @param req
* @param rsp
* @return true if a modification has resulted that requires persistance of the CoreContainer
* configuration.
*/
protected boolean handleStatusAction(SolrQueryRequest req, SolrQueryResponse rsp)
throws SolrException {
SolrParams params = req.getParams();
String cname = params.get(CoreAdminParams.CORE);
boolean doPersist = false;
NamedList<Object> status = new SimpleOrderedMap<Object>();
try {
if (cname == null) {
for (String name : coreContainer.getCoreNames()) {
status.add(name, getCoreStatus(coreContainer, name));
}
} else {
status.add(cname, getCoreStatus(coreContainer, cname));
}
rsp.add("status", status);
doPersist = false; // no state change
return doPersist;
} catch (Exception ex) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, "Error handling 'status' action ", ex);
}
}
@Test
public void testExtendedCollate() throws Exception {
SolrCore core = h.getCore();
SearchComponent speller = core.getSearchComponent("spellcheck");
assertTrue("speller is null and it shouldn't be", speller != null);
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(CommonParams.QT, "spellCheckCompRH");
params.add(CommonParams.Q, "lowerfilt:(+fauth +home +loane)");
params.add(SpellCheckComponent.SPELLCHECK_EXTENDED_RESULTS, "true");
params.add(SpellCheckComponent.COMPONENT_NAME, "true");
params.add(SpellCheckComponent.SPELLCHECK_BUILD, "true");
params.add(SpellCheckComponent.SPELLCHECK_COUNT, "10");
params.add(SpellCheckComponent.SPELLCHECK_COLLATE, "true");
// Testing backwards-compatible behavior.
// Returns 1 collation as a single string.
// All words are "correct" per the dictionary, but this collation would
// return no results if tried.
SolrRequestHandler handler = core.getRequestHandler("spellCheckCompRH");
SolrQueryResponse rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
SolrQueryRequest req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
req.close();
NamedList values = rsp.getValues();
NamedList spellCheck = (NamedList) values.get("spellcheck");
NamedList suggestions = (NamedList) spellCheck.get("suggestions");
String singleCollation = (String) suggestions.get("collation");
assertEquals("lowerfilt:(+faith +homer +loaves)", singleCollation);
// Testing backwards-compatible response format but will only return a
// collation that would return results.
params.remove(SpellCheckComponent.SPELLCHECK_BUILD);
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES, "5");
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "1");
handler = core.getRequestHandler("spellCheckCompRH");
rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
req.close();
values = rsp.getValues();
spellCheck = (NamedList) values.get("spellcheck");
suggestions = (NamedList) spellCheck.get("suggestions");
singleCollation = (String) suggestions.get("collation");
assertEquals("lowerfilt:(+faith +hope +loaves)", singleCollation);
// Testing returning multiple collations if more than one valid
// combination exists.
params.remove(SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES);
params.remove(SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS);
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES, "5");
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "2");
handler = core.getRequestHandler("spellCheckCompRH");
rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
req.close();
values = rsp.getValues();
spellCheck = (NamedList) values.get("spellcheck");
suggestions = (NamedList) spellCheck.get("suggestions");
List<String> collations = suggestions.getAll("collation");
assertTrue(collations.size() == 2);
for (String multipleCollation : collations) {
assertTrue(
multipleCollation.equals("lowerfilt:(+faith +hope +love)")
|| multipleCollation.equals("lowerfilt:(+faith +hope +loaves)"));
}
// Testing return multiple collations with expanded collation response
// format.
params.add(SpellCheckComponent.SPELLCHECK_COLLATE_EXTENDED_RESULTS, "true");
handler = core.getRequestHandler("spellCheckCompRH");
rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
req.close();
values = rsp.getValues();
spellCheck = (NamedList) values.get("spellcheck");
suggestions = (NamedList) spellCheck.get("suggestions");
List<NamedList> expandedCollationList = suggestions.getAll("collation");
Set<String> usedcollations = new HashSet<String>();
assertTrue(expandedCollationList.size() == 2);
for (NamedList expandedCollation : expandedCollationList) {
String multipleCollation = (String) expandedCollation.get("collationQuery");
assertTrue(
multipleCollation.equals("lowerfilt:(+faith +hope +love)")
|| multipleCollation.equals("lowerfilt:(+faith +hope +loaves)"));
assertTrue(!usedcollations.contains(multipleCollation));
usedcollations.add(multipleCollation);
int hits = (Integer) expandedCollation.get("hits");
assertTrue(hits == 1);
NamedList misspellingsAndCorrections =
(NamedList) expandedCollation.get("misspellingsAndCorrections");
assertTrue(misspellingsAndCorrections.size() == 3);
String correctionForFauth = (String) misspellingsAndCorrections.get("fauth");
String correctionForHome = (String) misspellingsAndCorrections.get("home");
String correctionForLoane = (String) misspellingsAndCorrections.get("loane");
assertTrue(correctionForFauth.equals("faith"));
assertTrue(correctionForHome.equals("hope"));
assertTrue(correctionForLoane.equals("love") || correctionForLoane.equals("loaves"));
}
}
public void doDeleteByQuery(DeleteUpdateCommand cmd) throws IOException {
// even in non zk mode, tests simulate updates from a leader
if (!zkEnabled) {
isLeader = getNonZkLeaderAssumption(req);
} else {
zkCheck();
}
// NONE: we are the first to receive this deleteByQuery
// - it must be forwarded to the leader of every shard
// TO: we are a leader receiving a forwarded deleteByQuery... we must:
// - block all updates (use VersionInfo)
// - flush *all* updates going to our replicas
// - forward the DBQ to our replicas and wait for the response
// - log + execute the local DBQ
// FROM: we are a replica receiving a DBQ from our leader
// - log + execute the local DBQ
DistribPhase phase = DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
if (zkEnabled && DistribPhase.NONE == phase) {
boolean leaderForAnyShard = false; // start off by assuming we are not a leader for any shard
Map<String, Slice> slices = zkController.getClusterState().getSlices(collection);
if (slices == null) {
throw new SolrException(
ErrorCode.BAD_REQUEST,
"Cannot find collection:"
+ collection
+ " in "
+ zkController.getClusterState().getCollections());
}
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(DISTRIB_UPDATE_PARAM, DistribPhase.TOLEADER.toString());
List<Node> leaders = new ArrayList<Node>(slices.size());
for (Map.Entry<String, Slice> sliceEntry : slices.entrySet()) {
String sliceName = sliceEntry.getKey();
ZkNodeProps leaderProps;
try {
leaderProps = zkController.getZkStateReader().getLeaderProps(collection, sliceName);
} catch (InterruptedException e) {
throw new SolrException(
ErrorCode.SERVICE_UNAVAILABLE, "Exception finding leader for shard " + sliceName, e);
}
// TODO: What if leaders changed in the meantime?
// should we send out slice-at-a-time and if a node returns "hey, I'm not a leader" (or we
// get an error because it went down) then look up the new leader?
// Am I the leader for this slice?
ZkCoreNodeProps coreLeaderProps = new ZkCoreNodeProps(leaderProps);
String leaderNodeName = coreLeaderProps.getCoreNodeName();
String coreName = req.getCore().getName();
String coreNodeName = zkController.getNodeName() + "_" + coreName;
isLeader = coreNodeName.equals(leaderNodeName);
if (isLeader) {
// don't forward to ourself
leaderForAnyShard = true;
} else {
leaders.add(new StdNode(coreLeaderProps));
}
}
params.remove("commit"); // this will be distributed from the local commit
cmdDistrib.distribDelete(cmd, leaders, params);
if (!leaderForAnyShard) {
return;
}
// change the phase to TOLEADER so we look up and forward to our own replicas (if any)
phase = DistribPhase.TOLEADER;
}
List<Node> replicas = null;
if (zkEnabled && DistribPhase.TOLEADER == phase) {
// This core should be a leader
isLeader = true;
replicas = setupRequest();
} else if (DistribPhase.FROMLEADER == phase) {
isLeader = false;
}
if (vinfo == null) {
super.processDelete(cmd);
return;
}
// at this point, there is an update we need to try and apply.
// we may or may not be the leader.
// Find the version
long versionOnUpdate = cmd.getVersion();
if (versionOnUpdate == 0) {
String versionOnUpdateS = req.getParams().get(VERSION_FIELD);
versionOnUpdate = versionOnUpdateS == null ? 0 : Long.parseLong(versionOnUpdateS);
}
versionOnUpdate = Math.abs(versionOnUpdate); // normalize to positive version
boolean isReplay = (cmd.getFlags() & UpdateCommand.REPLAY) != 0;
boolean leaderLogic = isLeader && !isReplay;
if (!leaderLogic && versionOnUpdate == 0) {
throw new SolrException(ErrorCode.BAD_REQUEST, "missing _version_ on update from leader");
}
vinfo.blockUpdates();
try {
if (versionsStored) {
if (leaderLogic) {
long version = vinfo.getNewClock();
cmd.setVersion(-version);
// TODO update versions in all buckets
doLocalDelete(cmd);
} else {
cmd.setVersion(-versionOnUpdate);
if (ulog.getState() != UpdateLog.State.ACTIVE
&& (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {
// we're not in an active state, and this update isn't from a replay, so buffer it.
cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
ulog.deleteByQuery(cmd);
return;
}
doLocalDelete(cmd);
}
}
// since we don't know which documents were deleted, the easiest thing to do is to invalidate
// all real-time caches (i.e. UpdateLog) which involves also getting a new version of the
// IndexReader
// (so cache misses will see up-to-date data)
} finally {
vinfo.unblockUpdates();
}
// forward to all replicas
if (leaderLogic && replicas != null) {
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(VERSION_FIELD, Long.toString(cmd.getVersion()));
params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
params.set(
"update.from",
ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
cmdDistrib.distribDelete(cmd, replicas, params);
cmdDistrib.finish();
}
if (returnVersions && rsp != null) {
if (deleteByQueryResponse == null) {
deleteByQueryResponse = new NamedList<String>();
rsp.add("deleteByQuery", deleteByQueryResponse);
}
deleteByQueryResponse.add(cmd.getQuery(), cmd.getVersion());
}
}
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
rsp.add("analysis", doAnalysis(req));
}
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
// Make sure the cores is enabled
CoreContainer cores = getCoreContainer();
if (cores == null) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "Core container instance missing");
}
boolean doPersist = false;
// Pick the action
SolrParams params = req.getParams();
CoreAdminAction action = CoreAdminAction.STATUS;
String a = params.get(CoreAdminParams.ACTION);
if (a != null) {
action = CoreAdminAction.get(a);
if (action == null) {
doPersist = this.handleCustomAction(req, rsp);
}
}
if (action != null) {
switch (action) {
case CREATE:
{
doPersist = this.handleCreateAction(req, rsp);
break;
}
case RENAME:
{
doPersist = this.handleRenameAction(req, rsp);
break;
}
case ALIAS:
{
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"'ALIAS' is not supported " + req.getParams().get(CoreAdminParams.ACTION));
}
case UNLOAD:
{
doPersist = this.handleUnloadAction(req, rsp);
break;
}
case STATUS:
{
doPersist = this.handleStatusAction(req, rsp);
break;
}
case PERSIST:
{
doPersist = this.handlePersistAction(req, rsp);
break;
}
case RELOAD:
{
doPersist = this.handleReloadAction(req, rsp);
break;
}
case SWAP:
{
doPersist = this.handleSwapAction(req, rsp);
break;
}
case MERGEINDEXES:
{
doPersist = this.handleMergeAction(req, rsp);
break;
}
default:
{
doPersist = this.handleCustomAction(req, rsp);
break;
}
case LOAD:
break;
}
}
// Should we persist the changes?
if (doPersist) {
cores.persist();
rsp.add("saved", cores.getConfigFile().getAbsolutePath());
}
rsp.setHttpCaching(false);
}
@Override
public void load(
SolrQueryRequest req,
SolrQueryResponse rsp,
ContentStream stream,
UpdateRequestProcessor processor)
throws Exception {
Parser parser = null;
String streamType = req.getParams().get(ExtractingParams.STREAM_TYPE, null);
if (streamType != null) {
// Cache? Parsers are lightweight to construct and thread-safe, so I'm told
MediaType mt = MediaType.parse(streamType.trim().toLowerCase(Locale.ROOT));
parser = new DefaultParser(config.getMediaTypeRegistry()).getParsers().get(mt);
} else {
parser = autoDetectParser;
}
if (parser != null) {
Metadata metadata = new Metadata();
// If you specify the resource name (the filename, roughly) with this parameter,
// then Tika can make use of it in guessing the appropriate MIME type:
String resourceName = req.getParams().get(ExtractingParams.RESOURCE_NAME, null);
if (resourceName != null) {
metadata.add(TikaMetadataKeys.RESOURCE_NAME_KEY, resourceName);
}
// Provide stream's content type as hint for auto detection
if (stream.getContentType() != null) {
metadata.add(HttpHeaders.CONTENT_TYPE, stream.getContentType());
}
InputStream inputStream = null;
try {
inputStream = stream.getStream();
metadata.add(ExtractingMetadataConstants.STREAM_NAME, stream.getName());
metadata.add(ExtractingMetadataConstants.STREAM_SOURCE_INFO, stream.getSourceInfo());
metadata.add(ExtractingMetadataConstants.STREAM_SIZE, String.valueOf(stream.getSize()));
metadata.add(ExtractingMetadataConstants.STREAM_CONTENT_TYPE, stream.getContentType());
// HtmlParser and TXTParser regard Metadata.CONTENT_ENCODING in metadata
String charset = ContentStreamBase.getCharsetFromContentType(stream.getContentType());
if (charset != null) {
metadata.add(HttpHeaders.CONTENT_ENCODING, charset);
}
String xpathExpr = params.get(ExtractingParams.XPATH_EXPRESSION);
boolean extractOnly = params.getBool(ExtractingParams.EXTRACT_ONLY, false);
SolrContentHandler handler =
factory.createSolrContentHandler(metadata, params, req.getSchema());
ContentHandler parsingHandler = handler;
StringWriter writer = null;
BaseMarkupSerializer serializer = null;
if (extractOnly == true) {
String extractFormat = params.get(ExtractingParams.EXTRACT_FORMAT, "xml");
writer = new StringWriter();
if (extractFormat.equals(TEXT_FORMAT)) {
serializer = new TextSerializer();
serializer.setOutputCharStream(writer);
serializer.setOutputFormat(new OutputFormat("Text", "UTF-8", true));
} else {
serializer = new XMLSerializer(writer, new OutputFormat("XML", "UTF-8", true));
}
if (xpathExpr != null) {
Matcher matcher = PARSER.parse(xpathExpr);
serializer
.startDocument(); // The MatchingContentHandler does not invoke startDocument. See
// http://tika.markmail.org/message/kknu3hw7argwiqin
parsingHandler = new MatchingContentHandler(serializer, matcher);
} else {
parsingHandler = serializer;
}
} else if (xpathExpr != null) {
Matcher matcher = PARSER.parse(xpathExpr);
parsingHandler = new MatchingContentHandler(handler, matcher);
} // else leave it as is
try {
// potentially use a wrapper handler for parsing, but we still need the SolrContentHandler
// for getting the document.
ParseContext context = parseContextConfig.create();
context.set(Parser.class, parser);
context.set(HtmlMapper.class, MostlyPassthroughHtmlMapper.INSTANCE);
// Password handling
RegexRulesPasswordProvider epp = new RegexRulesPasswordProvider();
String pwMapFile = params.get(ExtractingParams.PASSWORD_MAP_FILE);
if (pwMapFile != null && pwMapFile.length() > 0) {
InputStream is = req.getCore().getResourceLoader().openResource(pwMapFile);
if (is != null) {
log.debug("Password file supplied: " + pwMapFile);
epp.parse(is);
}
}
context.set(PasswordProvider.class, epp);
String resourcePassword = params.get(ExtractingParams.RESOURCE_PASSWORD);
if (resourcePassword != null) {
epp.setExplicitPassword(resourcePassword);
log.debug("Literal password supplied for file " + resourceName);
}
parser.parse(inputStream, parsingHandler, metadata, context);
} catch (TikaException e) {
if (ignoreTikaException)
log.warn(
new StringBuilder("skip extracting text due to ")
.append(e.getLocalizedMessage())
.append(". metadata=")
.append(metadata.toString())
.toString());
else throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
if (extractOnly == false) {
addDoc(handler);
} else {
// serializer is not null, so we need to call endDoc on it if using xpath
if (xpathExpr != null) {
serializer.endDocument();
}
rsp.add(stream.getName(), writer.toString());
writer.close();
String[] names = metadata.names();
NamedList metadataNL = new NamedList();
for (int i = 0; i < names.length; i++) {
String[] vals = metadata.getValues(names[i]);
metadataNL.add(names[i], vals);
}
rsp.add(stream.getName() + "_metadata", metadataNL);
}
} catch (SAXException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
} finally {
IOUtils.closeQuietly(inputStream);
}
} else {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Stream type of "
+ streamType
+ " didn't match any known parsers. Please supply the "
+ ExtractingParams.STREAM_TYPE
+ " parameter.");
}
}
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
// Don't do anything if the framework is unknown
if (watcher == null) {
rsp.add("error", "Logging Not Initalized");
return;
}
rsp.add("watcher", watcher.getName());
SolrParams params = req.getParams();
if (params.get("threshold") != null) {
watcher.setThreshold(params.get("threshold"));
}
// Write something at each level
if (params.get("test") != null) {
log.trace("trace message");
log.debug("debug message");
log.info("info (with exception)", new RuntimeException("test"));
log.warn("warn (with exception)", new RuntimeException("test"));
log.error("error (with exception)", new RuntimeException("test"));
}
String[] set = params.getParams("set");
if (set != null) {
for (String pair : set) {
String[] split = pair.split(":");
if (split.length != 2) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"Invalid format, expected level:value, got " + pair);
}
String category = split[0];
String level = split[1];
watcher.setLogLevel(category, level);
}
}
String since = req.getParams().get("since");
if (since != null) {
long time = -1;
try {
time = Long.parseLong(since);
} catch (Exception ex) {
throw new SolrException(ErrorCode.BAD_REQUEST, "invalid timestamp: " + since);
}
AtomicBoolean found = new AtomicBoolean(false);
SolrDocumentList docs = watcher.getHistory(time, found);
if (docs == null) {
rsp.add("error", "History not enabled");
return;
} else {
SimpleOrderedMap<Object> info = new SimpleOrderedMap<Object>();
if (time > 0) {
info.add("since", time);
info.add("found", found);
} else {
info.add("levels", watcher.getAllLevels()); // show for the first request
}
info.add("last", watcher.getLastEvent());
info.add("buffer", watcher.getHistorySize());
info.add("threshold", watcher.getThreshold());
rsp.add("info", info);
rsp.add("history", docs);
}
} else {
rsp.add("levels", watcher.getAllLevels());
List<LoggerInfo> loggers = new ArrayList<LoggerInfo>(watcher.getAllLoggers());
Collections.sort(loggers);
List<SimpleOrderedMap<?>> info = new ArrayList<SimpleOrderedMap<?>>();
for (LoggerInfo wrap : loggers) {
info.add(wrap.getInfo());
}
rsp.add("loggers", info);
}
rsp.setHttpCaching(false);
}
