private QueryResponse invokeCdcrAction(CloudSolrClient client, CdcrParams.CdcrAction action)
throws IOException, SolrServerException {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CommonParams.QT, "/cdcr");
params.set(CommonParams.ACTION, action.toLower());
return client.query(params);
}
public void run() {
// create new collections rapid fire
Map<String, List<Integer>> collectionInfos = new HashMap<String, List<Integer>>();
int cnt = random().nextInt(TEST_NIGHTLY ? 13 : 3) + 1;
for (int i = 0; i < cnt; i++) {
String collectionName = "awholynewstresscollection_" + name + "_" + i;
int numShards = _TestUtil.nextInt(random(), 0, shardCount * 2) + 1;
int replicationFactor = _TestUtil.nextInt(random(), 0, 3) + 1;
int maxShardsPerNode =
(((numShards * 2 * replicationFactor)
/ getCommonCloudSolrServer()
.getZkStateReader()
.getClusterState()
.getLiveNodes()
.size()))
+ 1;
CloudSolrServer client = null;
try {
if (i == 0) {
client = createCloudClient(null);
} else if (i == 1) {
client = createCloudClient(collectionName);
}
createCollection(
collectionInfos,
collectionName,
numShards,
replicationFactor,
maxShardsPerNode,
client,
null,
"conf1");
// remove collection
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionAction.DELETE.toString());
params.set("name", collectionName);
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
if (client == null) {
client = createCloudClient(null);
}
client.request(request);
} catch (SolrServerException e) {
e.printStackTrace();
throw new RuntimeException(e);
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e);
} finally {
if (client != null) client.shutdown();
}
}
}
private void deleteCollectionWithDownNodes() throws Exception {
String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
// now try to remove a collection when a couple of it's nodes are down
if (secondConfigSet) {
createCollection(
null, "halfdeletedcollection2", 3, 2, 6, createNewSolrServer("", baseUrl), null, "conf2");
} else {
createCollection(
null, "halfdeletedcollection2", 3, 2, 6, createNewSolrServer("", baseUrl), null);
}
waitForRecoveriesToFinish("halfdeletedcollection2", false);
// stop a couple nodes
ChaosMonkey.stop(jettys.get(0));
ChaosMonkey.stop(jettys.get(1));
baseUrl = getBaseUrl((HttpSolrServer) clients.get(2));
// remove a collection
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionAction.DELETE.toString());
params.set("name", "halfdeletedcollection2");
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
createNewSolrServer("", baseUrl).request(request);
cloudClient.getZkStateReader().updateClusterState(true);
assertFalse(
cloudClient.getZkStateReader().getClusterState().hasCollection("halfdeletedcollection2"));
}
/** Generate external process results (if they have not already been generated). */
@Override
public void prepare(ResponseBuilder rb) throws IOException {
SolrParams params = rb.req.getParams();
if (!params.getBool(getName(), false)) {
return;
}
XJoinResults<?> results = (XJoinResults<?>) rb.req.getContext().get(getResultsTag());
if (results != null) {
return;
}
// generate external process results, by passing 'external' prefixed parameters
// from the query string to our factory
String prefix = getName() + "." + XJoinParameters.EXTERNAL_PREFIX + ".";
ModifiableSolrParams externalParams = new ModifiableSolrParams();
for (Iterator<String> it = params.getParameterNamesIterator(); it.hasNext(); ) {
String name = it.next();
if (name.startsWith(prefix)) {
externalParams.set(name.substring(prefix.length()), params.get(name));
}
}
results = factory.getResults(externalParams);
rb.req.getContext().put(getResultsTag(), results);
}
private void processAliases(Aliases aliases, List<String> collectionsList) {
String collection = solrReq.getParams().get(COLLECTION_PROP);
if (collection != null) {
collectionsList = StrUtils.splitSmart(collection, ",", true);
}
if (collectionsList != null) {
Set<String> newCollectionsList = new HashSet<>(collectionsList.size());
for (String col : collectionsList) {
String al = aliases.getCollectionAlias(col);
if (al != null) {
List<String> aliasList = StrUtils.splitSmart(al, ",", true);
newCollectionsList.addAll(aliasList);
} else {
newCollectionsList.add(col);
}
}
if (newCollectionsList.size() > 0) {
StringBuilder collectionString = new StringBuilder();
Iterator<String> it = newCollectionsList.iterator();
int sz = newCollectionsList.size();
for (int i = 0; i < sz; i++) {
collectionString.append(it.next());
if (i < newCollectionsList.size() - 1) {
collectionString.append(",");
}
}
ModifiableSolrParams params = new ModifiableSolrParams(solrReq.getParams());
params.set(COLLECTION_PROP, collectionString.toString());
solrReq.setParams(params);
}
}
}
public static ModifiableSolrParams params(String... params) {
ModifiableSolrParams msp = new ModifiableSolrParams();
for (int i = 0; i < params.length; i += 2) {
msp.add(params[i], params[i + 1]);
}
return msp;
}
@Override
public QParser createParser(
String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
ModifiableSolrParams customParams = new ModifiableSolrParams();
if (qstr.contains("#")) {
CUSTOM_QF[3] = "tweet_hashtags^1.5";
} else {
CUSTOM_QF[3] = "tweet_hashtags^0.5";
}
if (qstr.contains(LANG_EN)) {
CUSTOM_QF[0] = "text_en^1.25";
qstr = qstr.replace(LANG_EN, "");
}
if (qstr.contains(LANG_DE)) {
CUSTOM_QF[1] = "text_de^1.25";
qstr = qstr.replace(LANG_DE, "");
}
if (qstr.contains(LANG_RU)) {
CUSTOM_QF[2] = "text_ru^1.25";
qstr = qstr.replace(LANG_RU, "");
}
customParams.add(DisMaxParams.QF, CUSTOM_QF);
params = SolrParams.wrapAppended(params, customParams);
return new ExtendedDismaxQParser(qstr, localParams, params, req);
}
/**
* Adds extra parameters to a Solr query for better term searches, including custom options. More
* specifically, adds parameters for requesting the score to be included in the results, for
* requesting a spellcheck result, and sets the {@code start} and {@code rows} parameters when
* missing.
*
* @param originalParams the original Solr parameters to enhance
* @param queryOptions extra options to include in the query; these override the default values,
* but don't override values already set in the query
* @return the enhanced parameters
*/
public static SolrParams enhanceParams(
SolrParams originalParams, Map<String, String> queryOptions) {
if (originalParams == null) {
return null;
}
ModifiableSolrParams newParams = new ModifiableSolrParams();
newParams.set(CommonParams.START, "0");
newParams.set(CommonParams.ROWS, "1000");
newParams.set(CommonParams.FL, "* score");
if (queryOptions != null) {
for (Map.Entry<String, String> item : queryOptions.entrySet()) {
newParams.set(item.getKey(), item.getValue());
}
}
for (Map.Entry<String, Object> item : originalParams.toNamedList()) {
if (item.getValue() != null && item.getValue() instanceof String[]) {
newParams.set(item.getKey(), (String[]) item.getValue());
} else {
newParams.set(item.getKey(), String.valueOf(item.getValue()));
}
}
if (newParams.get(SPELLCHECK) == null) {
newParams.set(SPELLCHECK, Boolean.toString(true));
newParams.set(SpellingParams.SPELLCHECK_COLLATE, Boolean.toString(true));
}
return newParams;
}
/**
* get the solr document list from a query response This differs from getResponseByParams in such
* a way that it does only create the fields of the response but never search snippets and there
* are also no facets generated.
*
* @param params
* @return
* @throws IOException
* @throws SolrException
*/
@Override
public SolrDocumentList getDocumentListByParams(ModifiableSolrParams params)
throws IOException, SolrException {
SolrQueryRequest req = this.request(params);
SolrQueryResponse response = null;
String q = params.get(CommonParams.Q);
String fq = params.get(CommonParams.FQ);
String sort = params.get(CommonParams.SORT);
String threadname = Thread.currentThread().getName();
try {
if (q != null)
Thread.currentThread()
.setName(
"solr query: q = "
+ q
+ (fq == null ? "" : ", fq = " + fq)
+ (sort == null ? "" : ", sort = " + sort)); // for debugging in Threaddump
response = this.query(req);
if (q != null) Thread.currentThread().setName(threadname);
if (response == null) throw new IOException("response == null");
return SolrQueryResponse2SolrDocumentList(req, response);
} finally {
req.close();
SolrRequestInfo.clearRequestInfo();
}
}
private ModifiableSolrParams getSTEP2Params(
RecommendQuery query, Integer maxReuslts, Map<String, Integer> userInteractionMap) {
ModifiableSolrParams solrParams = new ModifiableSolrParams();
String queryString =
createQueryToFindProdLikedBySimilarSocialUsers(
userInteractionMap, contentFilter, MAX_USER_OCCURENCE_COUNT);
String filterQueryString =
RecommendationQueryUtils.buildFilterForContentBasedFiltering(contentFilter);
if (alreadyBoughtProducts != null && alreadyBoughtProducts.size() > 0) {
if (filterQueryString.trim().length() > 0) {
filterQueryString += " OR ";
}
filterQueryString +=
RecommendationQueryUtils.buildFilterForAlreadyBoughtProducts("id", alreadyBoughtProducts);
}
solrParams.set("q", queryString);
solrParams.set("fq", filterQueryString);
solrParams.set("fl", "id");
solrParams.set("rows", maxReuslts);
return solrParams;
}
private QueryResponse getCdcrQueue(CloudSolrClient client)
throws SolrServerException, IOException {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CommonParams.QT, "/cdcr");
params.set(CommonParams.ACTION, CdcrParams.QUEUES);
return client.query(params);
}
private ModifiableSolrParams getInteractionsFromMeAndUSersThatILikedOrCommented(String user) {
ModifiableSolrParams solrParams = new ModifiableSolrParams();
String queryString =
"id:(\"" + user + "\"^2) OR users_that_posted_a_snapshot_to_me:(\"" + user + "\")";
solrParams.set("q", queryString);
solrParams.set("rows", Integer.MAX_VALUE);
return solrParams;
}
@Override
public void processDelete(DeleteUpdateCommand cmd) throws IOException {
if (!cmd.isDeleteById()) {
doDeleteByQuery(cmd);
return;
}
int hash = 0;
if (zkEnabled) {
zkCheck();
hash = hash(cmd);
nodes = setupRequest(hash);
} else {
isLeader = getNonZkLeaderAssumption(req);
}
boolean dropCmd = false;
if (!forwardToLeader) {
dropCmd = versionDelete(cmd);
}
if (dropCmd) {
// TODO: do we need to add anything to the response?
return;
}
ModifiableSolrParams params = null;
if (nodes != null) {
params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(
DISTRIB_UPDATE_PARAM,
(isLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString()));
if (isLeader) {
params.set(
"distrib.from",
ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
}
cmdDistrib.distribDelete(cmd, nodes, params);
}
// cmd.getIndexId == null when delete by query
// TODO: what to do when no idField?
if (returnVersions && rsp != null && cmd.getIndexedId() != null && idField != null) {
if (deleteResponse == null) {
deleteResponse = new NamedList<String>();
rsp.add("deletes", deleteResponse);
}
if (scratch == null) scratch = new CharsRef();
idField.getType().indexedToReadable(cmd.getIndexedId(), scratch);
deleteResponse.add(
scratch.toString(),
cmd
.getVersion()); // we're returning the version of the delete.. not the version of the
// doc we deleted.
}
}
@Override
public void processAdd(AddUpdateCommand cmd) throws IOException {
// TODO: check for id field?
int hash = 0;
if (zkEnabled) {
zkCheck();
hash = hash(cmd);
nodes = setupRequest(hash);
} else {
isLeader = getNonZkLeaderAssumption(req);
}
boolean dropCmd = false;
if (!forwardToLeader) {
dropCmd = versionAdd(cmd);
}
if (dropCmd) {
// TODO: do we need to add anything to the response?
return;
}
ModifiableSolrParams params = null;
if (nodes != null) {
params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(
DISTRIB_UPDATE_PARAM,
(isLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString()));
if (isLeader) {
params.set(
"distrib.from",
ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
}
params.set(
"distrib.from",
ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
cmdDistrib.distribAdd(cmd, nodes, params);
}
// TODO: what to do when no idField?
if (returnVersions && rsp != null && idField != null) {
if (addsResponse == null) {
addsResponse = new NamedList<String>();
rsp.add("adds", addsResponse);
}
if (scratch == null) scratch = new CharsRef();
idField.getType().indexedToReadable(cmd.getIndexedId(), scratch);
addsResponse.add(scratch.toString(), cmd.getVersion());
}
// TODO: keep track of errors? needs to be done at a higher level though since
// an id may fail before it gets to this processor.
// Given that, it may also make sense to move the version reporting out of this
// processor too.
}
@Override
public RecommendResponse recommend(RecommendQuery query, Integer maxReuslts) {
productsToFilter = new ArrayList<String>();
String queryString = "*:*";
String sortCriteria = "user_count_purchased desc";
ModifiableSolrParams solrParams = new ModifiableSolrParams();
QueryResponse response = null;
RecommendResponse searchResponse = new RecommendResponse();
try {
String filterQueryString =
RecommendationQueryUtils.buildFilterForContentBasedFiltering(contentFilter);
if (query.getProductIds() != null && query.getProductIds().size() > 0) {
productsToFilter.addAll(query.getProductIds());
}
if (query.getUser() != null) {
if (alreadyBoughtProducts != null) {
productsToFilter.addAll(alreadyBoughtProducts);
}
}
if (productsToFilter != null && productsToFilter.size() > 0) {
if (filterQueryString.length() > 0) {
filterQueryString += " OR ";
}
filterQueryString +=
RecommendationQueryUtils.buildFilterForAlreadyBoughtProducts(productsToFilter);
}
solrParams.set("q", queryString);
solrParams.set("fq", filterQueryString);
solrParams.set("sort", sortCriteria);
solrParams.set("rows", maxReuslts);
response =
SolrServiceContainer.getInstance()
.getRecommendService()
.getSolrServer()
.query(solrParams);
// fill response object
List<String> extractedRecommendations =
RecommendationQueryUtils.extractRecommendationIds(
response.getBeans(CustomerAction.class));
searchResponse.setResultItems(extractedRecommendations);
searchResponse.setElapsedTime(response.getElapsedTime());
SolrDocumentList docResults = response.getResults();
searchResponse.setNumFound(docResults.getNumFound());
} catch (SolrServerException e) {
e.printStackTrace();
searchResponse.setNumFound(0);
searchResponse.setResultItems(new ArrayList<String>());
searchResponse.setElapsedTime(-1);
}
return searchResponse;
}
@Override
public QParser createParser(
String qStr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
Log.info("createParser");
ModifiableSolrParams modparams = new ModifiableSolrParams(params);
String modQ = filter(qStr);
modparams.set("q", modQ);
return req.getCore().getQueryPlugin(parserImpl).createParser(modQ, localParams, modparams, req);
}
@Test
public void testCollateWithFilter() throws Exception {
SolrCore core = h.getCore();
SearchComponent speller = core.getSearchComponent("spellcheck");
assertTrue("speller is null and it shouldn't be", speller != null);
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(SpellCheckComponent.COMPONENT_NAME, "true");
params.add(SpellCheckComponent.SPELLCHECK_BUILD, "true");
params.add(SpellCheckComponent.SPELLCHECK_COUNT, "10");
params.add(SpellCheckComponent.SPELLCHECK_COLLATE, "true");
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES, "5");
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "2");
params.add(CommonParams.Q, "lowerfilt:(+fauth +home +loane)");
params.add(CommonParams.FQ, "NOT(id:1)");
// Because a FilterQuery is applied which removes doc id#1 from possible hits, we would
// not want the collations to return us "lowerfilt:(+faith +hope +loaves)" as this only matches
// doc id#1.
SolrRequestHandler handler = core.getRequestHandler("spellCheckCompRH");
SolrQueryResponse rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
SolrQueryRequest req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
req.close();
NamedList values = rsp.getValues();
NamedList spellCheck = (NamedList) values.get("spellcheck");
NamedList suggestions = (NamedList) spellCheck.get("suggestions");
List<String> collations = suggestions.getAll("collation");
assertTrue(collations.size() == 1);
assertTrue(collations.get(0).equals("lowerfilt:(+faith +hope +love)"));
}
public Tuple getModel(String zkHost, String collection, String modelID, long checkMillis)
throws IOException {
Model model = null;
long currentTime = new Date().getTime();
synchronized (this) {
model = models.get(modelID);
if (model != null && ((currentTime - model.getLastChecked()) <= checkMillis)) {
return model.getTuple();
}
if (model != null) {
// model is expired
models.remove(modelID);
}
}
// Model is not in cache or has expired so fetch the model
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("q", "name_s:" + modelID);
params.set("fl", "terms_ss, idfs_ds, weights_ds, iteration_i, _version_");
params.set("sort", "iteration_i desc");
StreamContext streamContext = new StreamContext();
streamContext.setSolrClientCache(solrClientCache);
CloudSolrStream stream = new CloudSolrStream(zkHost, collection, params);
stream.setStreamContext(streamContext);
Tuple tuple = null;
try {
stream.open();
tuple = stream.read();
if (tuple.EOF) {
return null;
}
} finally {
stream.close();
}
synchronized (this) {
// check again to see if another thread has updated the same model
Model m = models.get(modelID);
if (m != null) {
Tuple t = m.getTuple();
long v = t.getLong("_version_");
if (v >= tuple.getLong("_version_")) {
return t;
} else {
models.put(modelID, new Model(tuple, currentTime));
return tuple;
}
} else {
models.put(modelID, new Model(tuple, currentTime));
return tuple;
}
}
}
@Override
public SolrParams getParams() {
if (action == null) {
throw new RuntimeException("no action specified!");
}
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, action.toString());
if (fileName != null) {
params.set(CoreAdminParams.FILE, fileName);
}
return params;
}
@Override
public SolrParams getParams() {
if (action == null) {
throw new RuntimeException("no action specified!");
}
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, action.toString());
params.set(CoreAdminParams.CORE, core);
if (other != null) {
params.set(CoreAdminParams.OTHER, other);
}
return params;
}
@Override
public void processCommit(CommitUpdateCommand cmd) throws IOException {
if (zkEnabled) {
zkCheck();
}
if (vinfo != null) {
vinfo.lockForUpdate();
}
try {
if (ulog == null
|| ulog.getState() == UpdateLog.State.ACTIVE
|| (cmd.getFlags() & UpdateCommand.REPLAY) != 0) {
super.processCommit(cmd);
} else {
log.info(
"Ignoring commit while not ACTIVE - state: "
+ ulog.getState()
+ " replay:"
+ (cmd.getFlags() & UpdateCommand.REPLAY));
}
} finally {
if (vinfo != null) {
vinfo.unlockForUpdate();
}
}
// TODO: we should consider this? commit everyone in the current collection
if (zkEnabled) {
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
if (!req.getParams().getBool(COMMIT_END_POINT, false)) {
params.set(COMMIT_END_POINT, true);
String nodeName =
req.getCore().getCoreDescriptor().getCoreContainer().getZkController().getNodeName();
String shardZkNodeName = nodeName + "_" + req.getCore().getName();
List<Node> nodes =
getCollectionUrls(
req,
req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName(),
shardZkNodeName);
if (nodes != null) {
cmdDistrib.distribCommit(cmd, nodes, params);
finish();
}
}
}
}
/**
* Tests the {@link
* FieldAnalysisRequestHandler#resolveAnalysisRequest(org.apache.solr.request.SolrQueryRequest)}
*/
@Test
public void testResolveAnalysisRequest() throws Exception {
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(AnalysisParams.FIELD_NAME, "text,nametext");
params.add(AnalysisParams.FIELD_TYPE, "whitetok,keywordtok");
params.add(AnalysisParams.FIELD_VALUE, "the quick red fox jumped over the lazy brown dogs");
params.add(CommonParams.Q, "fox brown");
SolrQueryRequest req = new LocalSolrQueryRequest(h.getCore(), params);
FieldAnalysisRequest request = handler.resolveAnalysisRequest(req);
List<String> fieldNames = request.getFieldNames();
assertEquals("Expecting 2 field names", 2, fieldNames.size());
assertEquals("text", fieldNames.get(0));
assertEquals("nametext", fieldNames.get(1));
List<String> fieldTypes = request.getFieldTypes();
assertEquals("Expecting 2 field types", 2, fieldTypes.size());
assertEquals("whitetok", fieldTypes.get(0));
assertEquals("keywordtok", fieldTypes.get(1));
assertEquals("the quick red fox jumped over the lazy brown dogs", request.getFieldValue());
assertEquals("fox brown", request.getQuery());
assertFalse(request.isShowMatch());
req.close();
// testing overide of query value using analysis.query param
params.add(AnalysisParams.QUERY, "quick lazy");
req = new LocalSolrQueryRequest(h.getCore(), params);
request = handler.resolveAnalysisRequest(req);
assertEquals("quick lazy", request.getQuery());
req.close();
// testing analysis.showmatch param
params.add(AnalysisParams.SHOW_MATCH, "false");
req = new LocalSolrQueryRequest(h.getCore(), params);
request = handler.resolveAnalysisRequest(req);
assertFalse(request.isShowMatch());
req.close();
params.set(AnalysisParams.SHOW_MATCH, "true");
req = new LocalSolrQueryRequest(h.getCore(), params);
request = handler.resolveAnalysisRequest(req);
assertTrue(request.isShowMatch());
req.close();
// testing absence of query value
params.remove(CommonParams.Q);
params.remove(AnalysisParams.QUERY);
req = new LocalSolrQueryRequest(h.getCore(), params);
request = handler.resolveAnalysisRequest(req);
assertNull(request.getQuery());
req.close();
}
private void deleteAlias(String alias) throws SolrServerException, IOException {
if (random().nextBoolean()) {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("name", alias);
params.set("action", CollectionAction.DELETEALIAS.toString());
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
NamedList<Object> result =
createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(0))).request(request);
} else {
CollectionAdminResponse resp =
CollectionAdminRequest.deleteAlias(
alias, createNewSolrServer("", getBaseUrl((HttpSolrServer) clients.get(0))));
}
}
/**
* the usage of getResponseByParams is disencouraged for the embedded Solr connector. Please use
* request(SolrParams) instead. Reason: Solr makes a very complex folding/unfolding including data
* compression for SolrQueryResponses.
*/
@Override
public QueryResponse getResponseByParams(ModifiableSolrParams params) throws IOException {
if (this.server == null) throw new IOException("server disconnected");
// during the solr query we set the thread name to the query string to get more debugging info
// in thread dumps
String threadname = Thread.currentThread().getName();
String ql = "";
try {
ql = URLDecoder.decode(params.toString(), StandardCharsets.UTF_8.name());
} catch (UnsupportedEncodingException e) {
}
Thread.currentThread().setName("solr query: q=" + ql);
ConcurrentLog.info("EmbeddedSolrConnector.getResponseByParams", "QUERY: " + ql);
// System.out.println("EmbeddedSolrConnector.getResponseByParams * QUERY: " + ql);
// System.out.println("STACKTRACE: " + ConcurrentLog.stackTrace());
QueryResponse rsp;
try {
rsp = this.server.query(params);
Thread.currentThread().setName(threadname);
if (rsp != null)
if (log.isFine()) log.fine(rsp.getResults().getNumFound() + " results for " + ql);
return rsp;
} catch (final SolrServerException e) {
throw new IOException(e);
} catch (final Throwable e) {
throw new IOException("Error executing query", e);
}
}
@Test
public void testCollateWithMultipleRequestHandlers() throws Exception {
SolrCore core = h.getCore();
SearchComponent speller = core.getSearchComponent("spellcheck");
assertTrue("speller is null and it shouldn't be", speller != null);
ModifiableSolrParams params = new ModifiableSolrParams();
params.add(SpellCheckComponent.COMPONENT_NAME, "true");
params.add(SpellCheckComponent.SPELLCHECK_DICT, "multipleFields");
params.add(SpellCheckComponent.SPELLCHECK_BUILD, "true");
params.add(SpellCheckComponent.SPELLCHECK_COUNT, "10");
params.add(SpellCheckComponent.SPELLCHECK_COLLATE, "true");
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATION_TRIES, "1");
params.add(SpellCheckComponent.SPELLCHECK_MAX_COLLATIONS, "1");
params.add(CommonParams.Q, "peac");
// SpellCheckCompRH has no "qf" defined. It will not find "peace" from "peac" despite it being
// in the dictionary
// because requrying against this Request Handler results in 0 hits.
SolrRequestHandler handler = core.getRequestHandler("spellCheckCompRH");
SolrQueryResponse rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
SolrQueryRequest req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
req.close();
NamedList values = rsp.getValues();
NamedList spellCheck = (NamedList) values.get("spellcheck");
NamedList suggestions = (NamedList) spellCheck.get("suggestions");
String singleCollation = (String) suggestions.get("collation");
assertNull(singleCollation);
// SpellCheckCompRH1 has "lowerfilt1" defined in the "qf" param. It will find "peace" from
// "peac" because
// requrying field "lowerfilt1" returns the hit.
params.remove(SpellCheckComponent.SPELLCHECK_BUILD);
handler = core.getRequestHandler("spellCheckCompRH1");
rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap());
req = new LocalSolrQueryRequest(core, params);
handler.handleRequest(req, rsp);
req.close();
values = rsp.getValues();
spellCheck = (NamedList) values.get("spellcheck");
suggestions = (NamedList) spellCheck.get("suggestions");
singleCollation = (String) suggestions.get("collation");
assertEquals(singleCollation, "peace");
}
protected void indexDoc(List<CloudJettyRunner> skipServers, Object... fields)
throws IOException, SolrServerException {
SolrInputDocument doc = new SolrInputDocument();
addFields(doc, fields);
addFields(doc, "rnd_b", true);
controlClient.add(doc);
UpdateRequest ureq = new UpdateRequest();
ureq.add(doc);
ModifiableSolrParams params = new ModifiableSolrParams();
for (CloudJettyRunner skip : skipServers) {
params.add("test.distrib.skip.servers", skip.url + "/");
}
ureq.setParams(params);
ureq.process(cloudClient);
}
@Override
protected QueryResponse queryServer(ModifiableSolrParams params) throws SolrServerException {
if (r.nextBoolean()) return super.queryServer(params);
if (r.nextBoolean()) params.set("collection", DEFAULT_COLLECTION);
QueryResponse rsp = getCommonCloudSolrServer().query(params);
return rsp;
}
@Override
public SolrParams getParams() {
if (action == null) {
throw new RuntimeException("no action specified!");
}
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, action.toString());
params.set(CoreAdminParams.CORE, core);
if (indexDirs != null) {
for (String indexDir : indexDirs) {
params.set(CoreAdminParams.INDEX_DIR, indexDir);
}
}
if (srcCores != null) {
for (String srcCore : srcCores) {
params.set(CoreAdminParams.SRC_CORE, srcCore);
}
}
return params;
}
/** Get a document by ID using real-time get */
public JavaRDD<SolrDocument> get(JavaSparkContext jsc, final String docId)
throws SolrServerException {
CloudSolrClient cloudSolrServer = getSolrClient(zkHost);
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("collection", collection);
params.set("qt", "/get");
params.set("id", docId);
QueryResponse resp = null;
try {
resp = cloudSolrServer.query(params);
} catch (Exception exc) {
if (exc instanceof SolrServerException) {
throw (SolrServerException) exc;
} else {
throw new SolrServerException(exc);
}
}
SolrDocument doc = (SolrDocument) resp.getResponse().get("doc");
List<SolrDocument> list = (doc != null) ? Arrays.asList(doc) : new ArrayList<SolrDocument>();
return jsc.parallelize(list, 1);
}
/**
* Replaces the original query in the Solr parameters with the suggested spellchecked query. It
* also fixes the boost query, if any.
*
* @param originalParams the original Solr parameters to fix
* @param suggestedQuery the suggested query
* @return new Solr parameters with the query and boost query fixed
*/
public static SolrParams applySpellcheckSuggestion(
SolrParams originalParams, String suggestedQuery) {
if (originalParams == null) {
return null;
}
if (StringUtils.isBlank(suggestedQuery)) {
return originalParams;
}
ModifiableSolrParams newParams = new ModifiableSolrParams(originalParams);
String newQuery = suggestedQuery;
// Since the spelling suggestion might not be that good, also search for the original user input
if (StringUtils.isNotEmpty(originalParams.get(SpellingParams.SPELLCHECK_Q))) {
newQuery = originalParams.get(CommonParams.Q) + "^10 " + suggestedQuery;
}
// Check if the last term in the query is a word stub search which, in case the request comes
// from a
// user-triggered search for terms from the UI, is a prefix search for the last typed word
Matcher originalStub = WORD_STUB.matcher(newParams.get(CommonParams.Q));
Matcher newStub = WORD_STUB.matcher(suggestedQuery);
if (originalStub.find()
&& newStub.find()
&& !StringUtils.equals(originalStub.group(2), newStub.group(2))) {
// Since word stubs aren't complete words, they may wrongly be "corrected" to a full word that
// doesn't match
// what the user started typing; include both the original stub and the "corrected" stub in
// the query
newQuery += ' ' + originalStub.group() + "^1.5";
// Also fix the boost query
String boostQuery = newParams.get(DisMaxParams.BQ);
if (StringUtils.isNotEmpty(boostQuery)) {
newParams.add(DisMaxParams.BQ, boostQuery.replace(originalStub.group(2), newStub.group(2)));
}
}
// Replace the query
newParams.set(CommonParams.Q, newQuery);
return newParams;
}
