/** test range query performance */
public void XtestFilteringPerformance() throws Exception {
int indexSize = 19999;
float fractionCovered = .1f;
String l = t(0);
String u = t((int) (indexSize * 10 * fractionCovered));
SolrQueryRequest req = lrf.makeRequest();
QParser parser = QParser.getParser("foomany_s:[" + l + " TO " + u + "]", null, req);
Query rangeQ = parser.parse();
List<Query> filters = new ArrayList<Query>();
filters.add(rangeQ);
req.close();
parser =
QParser.getParser(
"{!dismax qf=t10_100_ws pf=t10_100_ws ps=20}" + t(0) + ' ' + t(1) + ' ' + t(2),
null,
req);
Query q = parser.parse();
// SolrIndexSearcher searcher = req.getSearcher();
// DocSet range = searcher.getDocSet(rangeQ, null);
createIndex2(indexSize, "foomany_s", "t10_100_ws");
// doListGen(100, q, filters, false, true);
doListGen(500, q, filters, false, true);
req.close();
}
/** test range query performance */
public void XtestRangePerformance() throws Exception {
int indexSize = 1999;
float fractionCovered = 1.0f;
String l = t(0);
String u = t((int) (indexSize * 10 * fractionCovered));
SolrQueryRequest req = lrf.makeRequest();
QParser parser = QParser.getParser("foomany_s:[" + l + " TO " + u + "]", null, req);
Query range = parser.parse();
QParser parser2 = QParser.getParser("{!frange l=" + l + " u=" + u + "}foomany_s", null, req);
Query frange = parser2.parse();
req.close();
createIndex2(indexSize, "foomany_s");
doSetGen(1, range);
doSetGen(1, frange); // load field cache
doSetGen(100, range);
doSetGen(10000, frange);
}
public Query parse() throws SyntaxError {
String reRankQueryString = localParams.get("reRankQuery");
boolean scale = localParams.getBool("scale", false);
QParser reRankParser = QParser.getParser(reRankQueryString, null, req);
Query reRankQuery = reRankParser.parse();
int reRankDocs = localParams.getInt("reRankDocs", 200);
reRankDocs = Math.max(1, reRankDocs); //
double reRankWeight = localParams.getDouble("reRankWeight", 2.0d);
int start = params.getInt(CommonParams.START, 0);
int rows = params.getInt(CommonParams.ROWS, 10);
int length = start + rows;
return new ReRankQuery(reRankQuery, reRankDocs, reRankWeight, length, scale);
}
/**
* The form of the sort specification string currently parsed is:
*
* <pre>
* SortSpec ::= SingleSort [, SingleSort]*
* SingleSort ::= <fieldname|function> SortDirection
* SortDirection ::= top | desc | bottom | asc
* </pre>
*
* Examples:
*
* <pre>
* score desc #normal sort by score (will return null)
* weight bottom #sort by weight ascending
* weight desc #sort by weight descending
* height desc,weight desc #sort by height descending, and use weight descending to break any ties
* height desc,weight asc #sort by height descending, using weight ascending as a tiebreaker
* </pre>
*
* @return a SortSpec object populated with the appropriate Sort (which may be null if default
* score sort is used) and SchemaFields (where applicable) using hardcoded default count &
* offset values.
*/
public static SortSpec parseSortSpec(String sortSpec, SolrQueryRequest req) {
if (sortSpec == null || sortSpec.length() == 0) return newEmptySortSpec();
List<SortField> sorts = new ArrayList<SortField>(4);
List<SchemaField> fields = new ArrayList<SchemaField>(4);
try {
StrParser sp = new StrParser(sortSpec);
while (sp.pos < sp.end) {
sp.eatws();
final int start = sp.pos;
// short circuit test for a really simple field name
String field = sp.getId(null);
Exception qParserException = null;
if (field == null || !Character.isWhitespace(sp.peekChar())) {
// let's try it as a function instead
field = null;
String funcStr = sp.val.substring(start);
QParser parser = QParser.getParser(funcStr, FunctionQParserPlugin.NAME, req);
Query q = null;
try {
if (parser instanceof FunctionQParser) {
FunctionQParser fparser = (FunctionQParser) parser;
fparser.setParseMultipleSources(false);
fparser.setParseToEnd(false);
q = fparser.getQuery();
if (fparser.localParams != null) {
if (fparser.valFollowedParams) {
// need to find the end of the function query via the string parser
int leftOver = fparser.sp.end - fparser.sp.pos;
sp.pos = sp.end - leftOver; // reset our parser to the same amount of leftover
} else {
// the value was via the "v" param in localParams, so we need to find
// the end of the local params themselves to pick up where we left off
sp.pos = start + fparser.localParamsEnd;
}
} else {
// need to find the end of the function query via the string parser
int leftOver = fparser.sp.end - fparser.sp.pos;
sp.pos = sp.end - leftOver; // reset our parser to the same amount of leftover
}
} else {
// A QParser that's not for function queries.
// It must have been specified via local params.
q = parser.getQuery();
assert parser.getLocalParams() != null;
sp.pos = start + parser.localParamsEnd;
}
Boolean top = sp.getSortDirection();
if (null != top) {
// we have a Query and a valid direction
if (q instanceof FunctionQuery) {
sorts.add(((FunctionQuery) q).getValueSource().getSortField(top));
} else {
sorts.add((new QueryValueSource(q, 0.0f)).getSortField(top));
}
fields.add(null);
continue;
}
} catch (Exception e) {
// hang onto this in case the string isn't a full field name either
qParserException = e;
}
}
// if we made it here, we either have a "simple" field name,
// or there was a problem parsing the string as a complex func/quer
if (field == null) {
// try again, simple rules for a field name with no whitespace
sp.pos = start;
field = sp.getSimpleString();
}
Boolean top = sp.getSortDirection();
if (null == top) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Can't determine a Sort Order (asc or desc) in sort spec " + sp);
}
if (SCORE.equals(field)) {
if (top) {
sorts.add(SortField.FIELD_SCORE);
} else {
sorts.add(new SortField(null, SortField.Type.SCORE, true));
}
fields.add(null);
} else if (DOCID.equals(field)) {
sorts.add(new SortField(null, SortField.Type.DOC, top));
fields.add(null);
} else {
// try to find the field
SchemaField sf = req.getSchema().getFieldOrNull(field);
if (null == sf) {
if (null != qParserException) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"sort param could not be parsed as a query, and is not a "
+ "field that exists in the index: "
+ field,
qParserException);
}
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "sort param field can't be found: " + field);
}
sorts.add(sf.getSortField(top));
fields.add(sf);
}
}
} catch (SyntaxError e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "error in sort: " + sortSpec, e);
}
// normalize a sort on score desc to null
if (sorts.size() == 1 && sorts.get(0) == SortField.FIELD_SCORE) {
return newEmptySortSpec();
}
Sort s = new Sort(sorts.toArray(new SortField[sorts.size()]));
return new SortSpec(s, fields);
}
@Override
public void prepare(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrParams params = req.getParams();
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
SolrQueryResponse rsp = rb.rsp;
// Set field flags
String fl = params.get(CommonParams.FL);
int fieldFlags = 0;
if (fl != null) {
fieldFlags |= SolrPluginUtils.setReturnFields(fl, rsp);
}
rb.setFieldFlags(fieldFlags);
String defType = params.get(QueryParsing.DEFTYPE, QParserPlugin.DEFAULT_QTYPE);
// get it from the response builder to give a different component a chance
// to set it.
String queryString = rb.getQueryString();
if (queryString == null) {
// this is the normal way it's set.
queryString = params.get(CommonParams.Q);
rb.setQueryString(queryString);
}
try {
QParser parser = QParser.getParser(rb.getQueryString(), defType, req);
Query q = parser.getQuery();
if (q == null) {
// normalize a null query to a query that matches nothing
q = new BooleanQuery();
}
rb.setQuery(q);
rb.setSortSpec(parser.getSort(true));
rb.setQparser(parser);
String[] fqs = req.getParams().getParams(CommonParams.FQ);
if (fqs != null && fqs.length != 0) {
List<Query> filters = rb.getFilters();
if (filters == null) {
filters = new ArrayList<Query>();
rb.setFilters(filters);
}
for (String fq : fqs) {
if (fq != null && fq.trim().length() != 0) {
QParser fqp = QParser.getParser(fq, null, req);
filters.add(fqp.getQuery());
}
}
}
} catch (ParseException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
// TODO: temporary... this should go in a different component.
String shards = params.get(ShardParams.SHARDS);
if (shards != null) {
List<String> lst = StrUtils.splitSmart(shards, ",", true);
rb.shards = lst.toArray(new String[lst.size()]);
}
String shards_rows = params.get(ShardParams.SHARDS_ROWS);
if (shards_rows != null) {
rb.shards_rows = Integer.parseInt(shards_rows);
}
String shards_start = params.get(ShardParams.SHARDS_START);
if (shards_start != null) {
rb.shards_start = Integer.parseInt(shards_start);
}
}
/** Actually run the query */
@Override
public void process(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
SolrIndexSearcher searcher = req.getSearcher();
if (rb.getQueryCommand().getOffset() < 0) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "'start' parameter cannot be negative");
}
// -1 as flag if not set.
long timeAllowed = (long) params.getInt(CommonParams.TIME_ALLOWED, -1);
// Optional: This could also be implemented by the top-level searcher sending
// a filter that lists the ids... that would be transparent to
// the request handler, but would be more expensive (and would preserve score
// too if desired).
String ids = params.get(ShardParams.IDS);
if (ids != null) {
SchemaField idField = req.getSchema().getUniqueKeyField();
List<String> idArr = StrUtils.splitSmart(ids, ",", true);
int[] luceneIds = new int[idArr.size()];
int docs = 0;
for (int i = 0; i < idArr.size(); i++) {
int id =
req.getSearcher()
.getFirstMatch(
new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
if (id >= 0) luceneIds[docs++] = id;
}
DocListAndSet res = new DocListAndSet();
res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
if (rb.isNeedDocSet()) {
List<Query> queries = new ArrayList<Query>();
queries.add(rb.getQuery());
List<Query> filters = rb.getFilters();
if (filters != null) queries.addAll(filters);
res.docSet = searcher.getDocSet(queries);
}
rb.setResults(res);
rsp.add("response", rb.getResults().docList);
return;
}
SolrIndexSearcher.QueryCommand cmd = rb.getQueryCommand();
cmd.setTimeAllowed(timeAllowed);
SolrIndexSearcher.QueryResult result = new SolrIndexSearcher.QueryResult();
//
// grouping / field collapsing
//
boolean doGroup = params.getBool(GroupParams.GROUP, false);
if (doGroup) {
try {
cmd.groupCommands = new ArrayList<Grouping.Command>();
String[] fields = params.getParams(GroupParams.GROUP_FIELD);
String[] funcs = params.getParams(GroupParams.GROUP_FUNC);
String[] queries = params.getParams(GroupParams.GROUP_QUERY);
String groupSortStr = params.get(GroupParams.GROUP_SORT);
Sort groupSort =
groupSortStr != null ? QueryParsing.parseSort(groupSortStr, req.getSchema()) : null;
int limitDefault = cmd.getLen(); // this is normally from "rows"
int docsPerGroupDefault = params.getInt(GroupParams.GROUP_LIMIT, 1);
// temporary: implement all group-by-field as group-by-func
if (funcs == null) {
funcs = fields;
} else if (fields != null) {
// catenate functions and fields
String[] both = new String[fields.length + funcs.length];
System.arraycopy(fields, 0, both, 0, fields.length);
System.arraycopy(funcs, 0, both, fields.length, funcs.length);
funcs = both;
}
if (funcs != null) {
for (String groupByStr : funcs) {
QParser parser = QParser.getParser(groupByStr, "func", rb.req);
Query q = parser.getQuery();
Grouping.CommandFunc gc = new Grouping.CommandFunc();
gc.groupSort = groupSort;
if (q instanceof FunctionQuery) {
gc.groupBy = ((FunctionQuery) q).getValueSource();
} else {
gc.groupBy = new QueryValueSource(q, 0.0f);
}
gc.key = groupByStr;
gc.groupLimit = limitDefault;
gc.docsPerGroup = docsPerGroupDefault;
cmd.groupCommands.add(gc);
}
}
if (cmd.groupCommands.size() == 0) cmd.groupCommands = null;
if (cmd.groupCommands != null) {
if (rb.doHighlights || rb.isDebug()) {
// we need a single list of the returned docs
cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
}
searcher.search(result, cmd);
rb.setResult(result);
rsp.add("grouped", result.groupedResults);
// TODO: get "hits" a different way to log
return;
}
} catch (ParseException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
}
// normal search result
searcher.search(result, cmd);
rb.setResult(result);
rsp.add("response", rb.getResults().docList);
rsp.getToLog().add("hits", rb.getResults().docList.matches());
doFieldSortValues(rb, searcher);
doPrefetch(rb);
}