int doListGen(int iter, Query q, List<Query> filt, boolean cacheQuery, boolean cacheFilt)
throws Exception {
SolrQueryRequest req = lrf.makeRequest();
SolrIndexSearcher searcher = req.getSearcher();
long start = System.currentTimeMillis();
// These aren't public in SolrIndexSearcher
int NO_CHECK_QCACHE = 0x80000000;
int GET_DOCSET = 0x40000000;
int NO_CHECK_FILTERCACHE = 0x20000000;
int GET_SCORES = 0x01;
int ret = 0;
for (int i = 0; i < iter; i++) {
DocList l =
searcher.getDocList(
q,
filt,
(Sort) null,
0,
10,
(cacheQuery ? 0 : NO_CHECK_QCACHE) | (cacheFilt ? 0 : NO_CHECK_FILTERCACHE));
ret += l.matches();
}
long end = System.currentTimeMillis();
System.out.println(
"ret=" + ret + " time=" + (end - start) + " throughput=" + iter * 1000 / (end - start + 1));
req.close();
assertTrue(ret > 0); // make sure we did some work
return ret;
}
// code to produce docsets for non-docsetproducer queries
public static DocSet createDocSetGeneric(SolrIndexSearcher searcher, Query query)
throws IOException {
int maxDoc = searcher.getIndexReader().maxDoc();
DocSetCollector collector = new DocSetCollector(maxDoc);
// This may throw an ExitableDirectoryReader.ExitingReaderException
// but we should not catch it here, as we don't know how this DocSet will be used (it could be
// negated before use) or cached.
searcher.search(query, collector);
return collector.getDocSet();
}
int doSetGen(int iter, Query q) throws Exception {
SolrQueryRequest req = lrf.makeRequest();
SolrIndexSearcher searcher = req.getSearcher();
long start = System.currentTimeMillis();
int ret = 0;
for (int i = 0; i < iter; i++) {
DocSet set = searcher.getDocSetNC(q, null);
ret += set.size();
}
long end = System.currentTimeMillis();
System.out.println(
"ret=" + ret + " time=" + (end - start) + " throughput=" + iter * 1000 / (end - start + 1));
req.close();
assertTrue(ret > 0); // make sure we did some work
return ret;
}
public static DocSet createDocSet(SolrIndexSearcher searcher, Term term) throws IOException {
DirectoryReader reader = searcher.getRawReader(); // raw reader to avoid extra wrapping overhead
int maxDoc = searcher.getIndexReader().maxDoc();
int smallSetSize = smallSetSize(maxDoc);
String field = term.field();
BytesRef termVal = term.bytes();
int maxCount = 0;
int firstReader = -1;
List<LeafReaderContext> leaves = reader.leaves();
PostingsEnum[] postList =
new PostingsEnum
[leaves
.size()]; // use array for slightly higher scanning cost, but fewer memory
// allocations
for (LeafReaderContext ctx : leaves) {
assert leaves.get(ctx.ord) == ctx;
LeafReader r = ctx.reader();
Fields f = r.fields();
Terms t = f.terms(field);
if (t == null) continue; // field is missing
TermsEnum te = t.iterator();
if (te.seekExact(termVal)) {
maxCount += te.docFreq();
postList[ctx.ord] = te.postings(null, PostingsEnum.NONE);
if (firstReader < 0) firstReader = ctx.ord;
}
}
if (maxCount == 0) {
return DocSet.EMPTY;
}
if (maxCount <= smallSetSize) {
return createSmallSet(leaves, postList, maxCount, firstReader);
}
return createBigSet(leaves, postList, maxDoc, firstReader);
}
public DelegatingCollector getFilterCollector(IndexSearcher indexSearcher) {
try {
SolrIndexSearcher searcher = (SolrIndexSearcher) indexSearcher;
SortedDocValues docValues = null;
FunctionQuery funcQuery = null;
docValues = DocValues.getSorted(searcher.getLeafReader(), this.field);
FieldType fieldType = null;
if (this.max != null) {
if (this.max.indexOf("(") == -1) {
fieldType = searcher.getSchema().getField(this.max).getType();
} else {
LocalSolrQueryRequest request = null;
try {
SolrParams params = new ModifiableSolrParams();
request = new LocalSolrQueryRequest(searcher.getCore(), params);
FunctionQParser functionQParser = new FunctionQParser(this.max, null, null, request);
funcQuery = (FunctionQuery) functionQParser.parse();
} catch (Exception e) {
throw new IOException(e);
} finally {
request.close();
}
}
}
if (this.min != null) {
if (this.min.indexOf("(") == -1) {
fieldType = searcher.getSchema().getField(this.min).getType();
} else {
LocalSolrQueryRequest request = null;
try {
SolrParams params = new ModifiableSolrParams();
request = new LocalSolrQueryRequest(searcher.getCore(), params);
FunctionQParser functionQParser = new FunctionQParser(this.min, null, null, request);
funcQuery = (FunctionQuery) functionQParser.parse();
} catch (Exception e) {
throw new IOException(e);
} finally {
request.close();
}
}
}
int maxDoc = searcher.maxDoc();
int leafCount = searcher.getTopReaderContext().leaves().size();
// Deal with boosted docs.
// We have to deal with it here rather then the constructor because
// because the QueryElevationComponent runs after the Queries are constructed.
IntIntOpenHashMap boostDocs = null;
Map context = null;
SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
if (info != null) {
context = info.getReq().getContext();
}
if (this.boosted == null && context != null) {
this.boosted =
(Map<BytesRef, Integer>) context.get(QueryElevationComponent.BOOSTED_PRIORITY);
}
boostDocs = getBoostDocs(searcher, this.boosted, context);
if (this.min != null || this.max != null) {
return new CollapsingFieldValueCollector(
maxDoc,
leafCount,
docValues,
this.nullPolicy,
max != null ? this.max : this.min,
max != null,
this.needsScores,
fieldType,
boostDocs,
funcQuery,
searcher);
} else {
return new CollapsingScoreCollector(
maxDoc, leafCount, docValues, this.nullPolicy, boostDocs);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public Query parse() throws ParseException {
// this will combine the results that are found for each
// administrative level
BooleanQuery allQuery = new BooleanQuery();
// attempt to create a query on city (low level administrative division)
String city = localParams.get(CITY);
if (city != null) {
city = city.toLowerCase();
SchemaField nameField = req.getSchema().getField(NAME_FIELD);
FieldType nameFieldType = nameField.getType();
Query cityQuery = nameFieldType.getFieldQuery(this, nameField, city);
allQuery.add(cityQuery, Occur.MUST);
}
// attempt to create a query on state (mid level administrative division)
String state = localParams.get(STATE);
if (state != null) {
state = state.toLowerCase();
SchemaField stateField = req.getSchema().getField(STATE_FIELD);
FieldType stateFieldType = stateField.getType();
Query stateQuery = stateFieldType.getFieldQuery(this, stateField, state);
allQuery.add(stateQuery, Occur.MUST);
}
// attempt to create a query on city (high level administrative division)
String country = localParams.get(COUNTRY);
if (country != null) {
country = country.toLowerCase();
SchemaField countryField = req.getSchema().getField(COUNTRY_FIELD);
FieldType countryFieldType = countryField.getType();
Query countryQuery = countryFieldType.getFieldQuery(this, countryField, country);
allQuery.add(countryQuery, Occur.MUST);
}
String latitude = null;
String longitude = null;
// no location provided, computer user's location via reverse-ip lookup
if (allQuery.getClauses().length == 0) {
HttpServletRequest httpreq = req.getHttpServletRequest();
String ip = httpreq.getRemoteAddr();
LatLng currLoc = geoTargeter.getCurrentLocation(ip);
if (currLoc != null) {
latitude = Double.toString(currLoc.getLat());
longitude = Double.toString(currLoc.getLng());
}
} else {
SolrIndexSearcher searcher = req.getSearcher();
Document geocodeDoc = null;
try {
Sort s = new Sort(new SortField(POPULATION_FIELD, SortField.LONG, true));
DocList docs = searcher.getDocList(allQuery, new ArrayList<Query>(), s, 0, 1, 0);
if (docs == null) return query;
DocIterator iter = docs.iterator();
int geocodeDocId = iter.nextDoc();
geocodeDoc = searcher.doc(geocodeDocId);
} catch (Exception e) {
e.printStackTrace();
return query;
}
latitude = geocodeDoc.get("latitude");
longitude = geocodeDoc.get("longitude");
}
// combine the spatial and free-text queries
BooleanQuery finalQuery = new BooleanQuery();
// if no location is provided and user's location cannot be determined,
// do not search location
if (latitude != null && longitude != null) {
String distance = localParams.get(DISTANCE);
try {
Double.parseDouble(distance);
} catch (Exception e) {
distance = SEARCH_RADIUS;
}
SpatialFilterQParserPlugin spatialFilter = new SpatialFilterQParserPlugin();
ModifiableSolrParams spatialParams = new ModifiableSolrParams();
spatialParams.add(SpatialParams.POINT, latitude + "," + longitude);
spatialParams.add(SpatialParams.DISTANCE, distance);
spatialParams.add(CommonParams.FL, LOCATION_FIELD);
Query spatialQuery =
spatialFilter.createParser(qstr, spatialParams, spatialParams, req).parse();
finalQuery.add(spatialQuery, Occur.MUST);
}
// get results from default LuceneQParser
Query defQuery = new LuceneQParserPlugin().createParser(qstr, localParams, params, req).parse();
finalQuery.add(defQuery, Occur.MUST);
return finalQuery;
}
protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher)
throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
// The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't
// currently have an option to return sort field values. Because of this, we
// take the documents given and re-derive the sort values.
boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES, false);
if (fsv) {
Sort sort = rb.getSortSpec().getSort();
SortField[] sortFields =
sort == null ? new SortField[] {SortField.FIELD_SCORE} : sort.getSort();
NamedList sortVals = new NamedList(); // order is important for the sort fields
Field field = new Field("dummy", "", Field.Store.YES, Field.Index.NO); // a dummy Field
SolrIndexReader reader = searcher.getReader();
SolrIndexReader[] readers = reader.getLeafReaders();
SolrIndexReader subReader = reader;
if (readers.length == 1) {
// if there is a single segment, use that subReader and avoid looking up each time
subReader = readers[0];
readers = null;
}
int[] offsets = reader.getLeafOffsets();
for (SortField sortField : sortFields) {
int type = sortField.getType();
if (type == SortField.SCORE || type == SortField.DOC) continue;
FieldComparator comparator = null;
FieldComparator comparators[] =
(readers == null) ? null : new FieldComparator[readers.length];
String fieldname = sortField.getField();
FieldType ft = fieldname == null ? null : req.getSchema().getFieldTypeNoEx(fieldname);
DocList docList = rb.getResults().docList;
ArrayList<Object> vals = new ArrayList<Object>(docList.size());
DocIterator it = rb.getResults().docList.iterator();
int offset = 0;
int idx = 0;
while (it.hasNext()) {
int doc = it.nextDoc();
if (readers != null) {
idx = SolrIndexReader.readerIndex(doc, offsets);
subReader = readers[idx];
offset = offsets[idx];
comparator = comparators[idx];
}
if (comparator == null) {
comparator = sortField.getComparator(1, 0);
comparator = comparator.setNextReader(subReader, offset);
if (comparators != null) comparators[idx] = comparator;
}
doc -= offset; // adjust for what segment this is in
comparator.copy(0, doc);
Object val = comparator.value(0);
// Sortable float, double, int, long types all just use a string
// comparator. For these, we need to put the type into a readable
// format. One reason for this is that XML can't represent all
// string values (or even all unicode code points).
// indexedToReadable() should be a no-op and should
// thus be harmless anyway (for all current ways anyway)
if (val instanceof String) {
field.setValue((String) val);
val = ft.toObject(field);
}
// Must do the same conversion when sorting by a
// String field in Lucene, which returns the terms
// data as BytesRef:
if (val instanceof BytesRef) {
field.setValue(((BytesRef) val).utf8ToString());
val = ft.toObject(field);
}
vals.add(val);
}
sortVals.add(fieldname, vals);
}
rsp.add("sort_values", sortVals);
}
}
/** Actually run the query */
@Override
public void process(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
SolrIndexSearcher searcher = req.getSearcher();
if (rb.getQueryCommand().getOffset() < 0) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "'start' parameter cannot be negative");
}
// -1 as flag if not set.
long timeAllowed = (long) params.getInt(CommonParams.TIME_ALLOWED, -1);
// Optional: This could also be implemented by the top-level searcher sending
// a filter that lists the ids... that would be transparent to
// the request handler, but would be more expensive (and would preserve score
// too if desired).
String ids = params.get(ShardParams.IDS);
if (ids != null) {
SchemaField idField = req.getSchema().getUniqueKeyField();
List<String> idArr = StrUtils.splitSmart(ids, ",", true);
int[] luceneIds = new int[idArr.size()];
int docs = 0;
for (int i = 0; i < idArr.size(); i++) {
int id =
req.getSearcher()
.getFirstMatch(
new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
if (id >= 0) luceneIds[docs++] = id;
}
DocListAndSet res = new DocListAndSet();
res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
if (rb.isNeedDocSet()) {
List<Query> queries = new ArrayList<Query>();
queries.add(rb.getQuery());
List<Query> filters = rb.getFilters();
if (filters != null) queries.addAll(filters);
res.docSet = searcher.getDocSet(queries);
}
rb.setResults(res);
rsp.add("response", rb.getResults().docList);
return;
}
SolrIndexSearcher.QueryCommand cmd = rb.getQueryCommand();
cmd.setTimeAllowed(timeAllowed);
SolrIndexSearcher.QueryResult result = new SolrIndexSearcher.QueryResult();
//
// grouping / field collapsing
//
boolean doGroup = params.getBool(GroupParams.GROUP, false);
if (doGroup) {
try {
cmd.groupCommands = new ArrayList<Grouping.Command>();
String[] fields = params.getParams(GroupParams.GROUP_FIELD);
String[] funcs = params.getParams(GroupParams.GROUP_FUNC);
String[] queries = params.getParams(GroupParams.GROUP_QUERY);
String groupSortStr = params.get(GroupParams.GROUP_SORT);
Sort groupSort =
groupSortStr != null ? QueryParsing.parseSort(groupSortStr, req.getSchema()) : null;
int limitDefault = cmd.getLen(); // this is normally from "rows"
int docsPerGroupDefault = params.getInt(GroupParams.GROUP_LIMIT, 1);
// temporary: implement all group-by-field as group-by-func
if (funcs == null) {
funcs = fields;
} else if (fields != null) {
// catenate functions and fields
String[] both = new String[fields.length + funcs.length];
System.arraycopy(fields, 0, both, 0, fields.length);
System.arraycopy(funcs, 0, both, fields.length, funcs.length);
funcs = both;
}
if (funcs != null) {
for (String groupByStr : funcs) {
QParser parser = QParser.getParser(groupByStr, "func", rb.req);
Query q = parser.getQuery();
Grouping.CommandFunc gc = new Grouping.CommandFunc();
gc.groupSort = groupSort;
if (q instanceof FunctionQuery) {
gc.groupBy = ((FunctionQuery) q).getValueSource();
} else {
gc.groupBy = new QueryValueSource(q, 0.0f);
}
gc.key = groupByStr;
gc.groupLimit = limitDefault;
gc.docsPerGroup = docsPerGroupDefault;
cmd.groupCommands.add(gc);
}
}
if (cmd.groupCommands.size() == 0) cmd.groupCommands = null;
if (cmd.groupCommands != null) {
if (rb.doHighlights || rb.isDebug()) {
// we need a single list of the returned docs
cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
}
searcher.search(result, cmd);
rb.setResult(result);
rsp.add("grouped", result.groupedResults);
// TODO: get "hits" a different way to log
return;
}
} catch (ParseException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
}
// normal search result
searcher.search(result, cmd);
rb.setResult(result);
rsp.add("response", rb.getResults().docList);
rsp.getToLog().add("hits", rb.getResults().docList.matches());
doFieldSortValues(rb, searcher);
doPrefetch(rb);
}
public void handleMergeFields(ResponseBuilder rb, SolrIndexSearcher searcher)
throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
// The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't
// currently have an option to return sort field values. Because of this, we
// take the documents given and re-derive the sort values.
//
// TODO: See SOLR-5595
boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES, false);
if (fsv) {
NamedList<Object[]> sortVals = new NamedList<>(); // order is important for the sort fields
IndexReaderContext topReaderContext = searcher.getTopReaderContext();
List<LeafReaderContext> leaves = topReaderContext.leaves();
LeafReaderContext currentLeaf = null;
if (leaves.size() == 1) {
// if there is a single segment, use that subReader and avoid looking up each time
currentLeaf = leaves.get(0);
leaves = null;
}
DocList docList = rb.getResults().docList;
// sort ids from lowest to highest so we can access them in order
int nDocs = docList.size();
final long[] sortedIds = new long[nDocs];
final float[] scores = new float[nDocs]; // doc scores, parallel to sortedIds
DocList docs = rb.getResults().docList;
DocIterator it = docs.iterator();
for (int i = 0; i < nDocs; i++) {
sortedIds[i] = (((long) it.nextDoc()) << 32) | i;
scores[i] = docs.hasScores() ? it.score() : Float.NaN;
}
// sort ids and scores together
new InPlaceMergeSorter() {
@Override
protected void swap(int i, int j) {
long tmpId = sortedIds[i];
float tmpScore = scores[i];
sortedIds[i] = sortedIds[j];
scores[i] = scores[j];
sortedIds[j] = tmpId;
scores[j] = tmpScore;
}
@Override
protected int compare(int i, int j) {
return Long.compare(sortedIds[i], sortedIds[j]);
}
}.sort(0, sortedIds.length);
SortSpec sortSpec = rb.getSortSpec();
Sort sort = searcher.weightSort(sortSpec.getSort());
SortField[] sortFields =
sort == null ? new SortField[] {SortField.FIELD_SCORE} : sort.getSort();
List<SchemaField> schemaFields = sortSpec.getSchemaFields();
for (int fld = 0; fld < schemaFields.size(); fld++) {
SchemaField schemaField = schemaFields.get(fld);
FieldType ft = null == schemaField ? null : schemaField.getType();
SortField sortField = sortFields[fld];
SortField.Type type = sortField.getType();
// :TODO: would be simpler to always serialize every position of SortField[]
if (type == SortField.Type.SCORE || type == SortField.Type.DOC) continue;
FieldComparator<?> comparator = null;
LeafFieldComparator leafComparator = null;
Object[] vals = new Object[nDocs];
int lastIdx = -1;
int idx = 0;
for (int i = 0; i < sortedIds.length; ++i) {
long idAndPos = sortedIds[i];
float score = scores[i];
int doc = (int) (idAndPos >>> 32);
int position = (int) idAndPos;
if (leaves != null) {
idx = ReaderUtil.subIndex(doc, leaves);
currentLeaf = leaves.get(idx);
if (idx != lastIdx) {
// we switched segments. invalidate comparator.
comparator = null;
}
}
if (comparator == null) {
comparator = sortField.getComparator(1, 0);
leafComparator = comparator.getLeafComparator(currentLeaf);
}
doc -= currentLeaf.docBase; // adjust for what segment this is in
leafComparator.setScorer(new FakeScorer(doc, score));
leafComparator.copy(0, doc);
Object val = comparator.value(0);
if (null != ft) val = ft.marshalSortValue(val);
vals[position] = val;
}
sortVals.add(sortField.getField(), vals);
}
rsp.add("merge_values", sortVals);
}
}
