int doListGen(int iter, Query q, List<Query> filt, boolean cacheQuery, boolean cacheFilt) throws Exception { SolrQueryRequest req = lrf.makeRequest(); SolrIndexSearcher searcher = req.getSearcher(); long start = System.currentTimeMillis(); // These aren't public in SolrIndexSearcher int NO_CHECK_QCACHE = 0x80000000; int GET_DOCSET = 0x40000000; int NO_CHECK_FILTERCACHE = 0x20000000; int GET_SCORES = 0x01; int ret = 0; for (int i = 0; i < iter; i++) { DocList l = searcher.getDocList( q, filt, (Sort) null, 0, 10, (cacheQuery ? 0 : NO_CHECK_QCACHE) | (cacheFilt ? 0 : NO_CHECK_FILTERCACHE)); ret += l.matches(); } long end = System.currentTimeMillis(); System.out.println( "ret=" + ret + " time=" + (end - start) + " throughput=" + iter * 1000 / (end - start + 1)); req.close(); assertTrue(ret > 0); // make sure we did some work return ret; }
// code to produce docsets for non-docsetproducer queries public static DocSet createDocSetGeneric(SolrIndexSearcher searcher, Query query) throws IOException { int maxDoc = searcher.getIndexReader().maxDoc(); DocSetCollector collector = new DocSetCollector(maxDoc); // This may throw an ExitableDirectoryReader.ExitingReaderException // but we should not catch it here, as we don't know how this DocSet will be used (it could be // negated before use) or cached. searcher.search(query, collector); return collector.getDocSet(); }
int doSetGen(int iter, Query q) throws Exception { SolrQueryRequest req = lrf.makeRequest(); SolrIndexSearcher searcher = req.getSearcher(); long start = System.currentTimeMillis(); int ret = 0; for (int i = 0; i < iter; i++) { DocSet set = searcher.getDocSetNC(q, null); ret += set.size(); } long end = System.currentTimeMillis(); System.out.println( "ret=" + ret + " time=" + (end - start) + " throughput=" + iter * 1000 / (end - start + 1)); req.close(); assertTrue(ret > 0); // make sure we did some work return ret; }
public static DocSet createDocSet(SolrIndexSearcher searcher, Term term) throws IOException { DirectoryReader reader = searcher.getRawReader(); // raw reader to avoid extra wrapping overhead int maxDoc = searcher.getIndexReader().maxDoc(); int smallSetSize = smallSetSize(maxDoc); String field = term.field(); BytesRef termVal = term.bytes(); int maxCount = 0; int firstReader = -1; List<LeafReaderContext> leaves = reader.leaves(); PostingsEnum[] postList = new PostingsEnum [leaves .size()]; // use array for slightly higher scanning cost, but fewer memory // allocations for (LeafReaderContext ctx : leaves) { assert leaves.get(ctx.ord) == ctx; LeafReader r = ctx.reader(); Fields f = r.fields(); Terms t = f.terms(field); if (t == null) continue; // field is missing TermsEnum te = t.iterator(); if (te.seekExact(termVal)) { maxCount += te.docFreq(); postList[ctx.ord] = te.postings(null, PostingsEnum.NONE); if (firstReader < 0) firstReader = ctx.ord; } } if (maxCount == 0) { return DocSet.EMPTY; } if (maxCount <= smallSetSize) { return createSmallSet(leaves, postList, maxCount, firstReader); } return createBigSet(leaves, postList, maxDoc, firstReader); }
public DelegatingCollector getFilterCollector(IndexSearcher indexSearcher) { try { SolrIndexSearcher searcher = (SolrIndexSearcher) indexSearcher; SortedDocValues docValues = null; FunctionQuery funcQuery = null; docValues = DocValues.getSorted(searcher.getLeafReader(), this.field); FieldType fieldType = null; if (this.max != null) { if (this.max.indexOf("(") == -1) { fieldType = searcher.getSchema().getField(this.max).getType(); } else { LocalSolrQueryRequest request = null; try { SolrParams params = new ModifiableSolrParams(); request = new LocalSolrQueryRequest(searcher.getCore(), params); FunctionQParser functionQParser = new FunctionQParser(this.max, null, null, request); funcQuery = (FunctionQuery) functionQParser.parse(); } catch (Exception e) { throw new IOException(e); } finally { request.close(); } } } if (this.min != null) { if (this.min.indexOf("(") == -1) { fieldType = searcher.getSchema().getField(this.min).getType(); } else { LocalSolrQueryRequest request = null; try { SolrParams params = new ModifiableSolrParams(); request = new LocalSolrQueryRequest(searcher.getCore(), params); FunctionQParser functionQParser = new FunctionQParser(this.min, null, null, request); funcQuery = (FunctionQuery) functionQParser.parse(); } catch (Exception e) { throw new IOException(e); } finally { request.close(); } } } int maxDoc = searcher.maxDoc(); int leafCount = searcher.getTopReaderContext().leaves().size(); // Deal with boosted docs. // We have to deal with it here rather then the constructor because // because the QueryElevationComponent runs after the Queries are constructed. IntIntOpenHashMap boostDocs = null; Map context = null; SolrRequestInfo info = SolrRequestInfo.getRequestInfo(); if (info != null) { context = info.getReq().getContext(); } if (this.boosted == null && context != null) { this.boosted = (Map<BytesRef, Integer>) context.get(QueryElevationComponent.BOOSTED_PRIORITY); } boostDocs = getBoostDocs(searcher, this.boosted, context); if (this.min != null || this.max != null) { return new CollapsingFieldValueCollector( maxDoc, leafCount, docValues, this.nullPolicy, max != null ? this.max : this.min, max != null, this.needsScores, fieldType, boostDocs, funcQuery, searcher); } else { return new CollapsingScoreCollector( maxDoc, leafCount, docValues, this.nullPolicy, boostDocs); } } catch (Exception e) { throw new RuntimeException(e); } }
@Override public Query parse() throws ParseException { // this will combine the results that are found for each // administrative level BooleanQuery allQuery = new BooleanQuery(); // attempt to create a query on city (low level administrative division) String city = localParams.get(CITY); if (city != null) { city = city.toLowerCase(); SchemaField nameField = req.getSchema().getField(NAME_FIELD); FieldType nameFieldType = nameField.getType(); Query cityQuery = nameFieldType.getFieldQuery(this, nameField, city); allQuery.add(cityQuery, Occur.MUST); } // attempt to create a query on state (mid level administrative division) String state = localParams.get(STATE); if (state != null) { state = state.toLowerCase(); SchemaField stateField = req.getSchema().getField(STATE_FIELD); FieldType stateFieldType = stateField.getType(); Query stateQuery = stateFieldType.getFieldQuery(this, stateField, state); allQuery.add(stateQuery, Occur.MUST); } // attempt to create a query on city (high level administrative division) String country = localParams.get(COUNTRY); if (country != null) { country = country.toLowerCase(); SchemaField countryField = req.getSchema().getField(COUNTRY_FIELD); FieldType countryFieldType = countryField.getType(); Query countryQuery = countryFieldType.getFieldQuery(this, countryField, country); allQuery.add(countryQuery, Occur.MUST); } String latitude = null; String longitude = null; // no location provided, computer user's location via reverse-ip lookup if (allQuery.getClauses().length == 0) { HttpServletRequest httpreq = req.getHttpServletRequest(); String ip = httpreq.getRemoteAddr(); LatLng currLoc = geoTargeter.getCurrentLocation(ip); if (currLoc != null) { latitude = Double.toString(currLoc.getLat()); longitude = Double.toString(currLoc.getLng()); } } else { SolrIndexSearcher searcher = req.getSearcher(); Document geocodeDoc = null; try { Sort s = new Sort(new SortField(POPULATION_FIELD, SortField.LONG, true)); DocList docs = searcher.getDocList(allQuery, new ArrayList<Query>(), s, 0, 1, 0); if (docs == null) return query; DocIterator iter = docs.iterator(); int geocodeDocId = iter.nextDoc(); geocodeDoc = searcher.doc(geocodeDocId); } catch (Exception e) { e.printStackTrace(); return query; } latitude = geocodeDoc.get("latitude"); longitude = geocodeDoc.get("longitude"); } // combine the spatial and free-text queries BooleanQuery finalQuery = new BooleanQuery(); // if no location is provided and user's location cannot be determined, // do not search location if (latitude != null && longitude != null) { String distance = localParams.get(DISTANCE); try { Double.parseDouble(distance); } catch (Exception e) { distance = SEARCH_RADIUS; } SpatialFilterQParserPlugin spatialFilter = new SpatialFilterQParserPlugin(); ModifiableSolrParams spatialParams = new ModifiableSolrParams(); spatialParams.add(SpatialParams.POINT, latitude + "," + longitude); spatialParams.add(SpatialParams.DISTANCE, distance); spatialParams.add(CommonParams.FL, LOCATION_FIELD); Query spatialQuery = spatialFilter.createParser(qstr, spatialParams, spatialParams, req).parse(); finalQuery.add(spatialQuery, Occur.MUST); } // get results from default LuceneQParser Query defQuery = new LuceneQParserPlugin().createParser(qstr, localParams, params, req).parse(); finalQuery.add(defQuery, Occur.MUST); return finalQuery; }
protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException { SolrQueryRequest req = rb.req; SolrQueryResponse rsp = rb.rsp; // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't // currently have an option to return sort field values. Because of this, we // take the documents given and re-derive the sort values. boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES, false); if (fsv) { Sort sort = rb.getSortSpec().getSort(); SortField[] sortFields = sort == null ? new SortField[] {SortField.FIELD_SCORE} : sort.getSort(); NamedList sortVals = new NamedList(); // order is important for the sort fields Field field = new Field("dummy", "", Field.Store.YES, Field.Index.NO); // a dummy Field SolrIndexReader reader = searcher.getReader(); SolrIndexReader[] readers = reader.getLeafReaders(); SolrIndexReader subReader = reader; if (readers.length == 1) { // if there is a single segment, use that subReader and avoid looking up each time subReader = readers[0]; readers = null; } int[] offsets = reader.getLeafOffsets(); for (SortField sortField : sortFields) { int type = sortField.getType(); if (type == SortField.SCORE || type == SortField.DOC) continue; FieldComparator comparator = null; FieldComparator comparators[] = (readers == null) ? null : new FieldComparator[readers.length]; String fieldname = sortField.getField(); FieldType ft = fieldname == null ? null : req.getSchema().getFieldTypeNoEx(fieldname); DocList docList = rb.getResults().docList; ArrayList<Object> vals = new ArrayList<Object>(docList.size()); DocIterator it = rb.getResults().docList.iterator(); int offset = 0; int idx = 0; while (it.hasNext()) { int doc = it.nextDoc(); if (readers != null) { idx = SolrIndexReader.readerIndex(doc, offsets); subReader = readers[idx]; offset = offsets[idx]; comparator = comparators[idx]; } if (comparator == null) { comparator = sortField.getComparator(1, 0); comparator = comparator.setNextReader(subReader, offset); if (comparators != null) comparators[idx] = comparator; } doc -= offset; // adjust for what segment this is in comparator.copy(0, doc); Object val = comparator.value(0); // Sortable float, double, int, long types all just use a string // comparator. For these, we need to put the type into a readable // format. One reason for this is that XML can't represent all // string values (or even all unicode code points). // indexedToReadable() should be a no-op and should // thus be harmless anyway (for all current ways anyway) if (val instanceof String) { field.setValue((String) val); val = ft.toObject(field); } // Must do the same conversion when sorting by a // String field in Lucene, which returns the terms // data as BytesRef: if (val instanceof BytesRef) { field.setValue(((BytesRef) val).utf8ToString()); val = ft.toObject(field); } vals.add(val); } sortVals.add(fieldname, vals); } rsp.add("sort_values", sortVals); } }
/** Actually run the query */ @Override public void process(ResponseBuilder rb) throws IOException { SolrQueryRequest req = rb.req; SolrQueryResponse rsp = rb.rsp; SolrParams params = req.getParams(); if (!params.getBool(COMPONENT_NAME, true)) { return; } SolrIndexSearcher searcher = req.getSearcher(); if (rb.getQueryCommand().getOffset() < 0) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "'start' parameter cannot be negative"); } // -1 as flag if not set. long timeAllowed = (long) params.getInt(CommonParams.TIME_ALLOWED, -1); // Optional: This could also be implemented by the top-level searcher sending // a filter that lists the ids... that would be transparent to // the request handler, but would be more expensive (and would preserve score // too if desired). String ids = params.get(ShardParams.IDS); if (ids != null) { SchemaField idField = req.getSchema().getUniqueKeyField(); List<String> idArr = StrUtils.splitSmart(ids, ",", true); int[] luceneIds = new int[idArr.size()]; int docs = 0; for (int i = 0; i < idArr.size(); i++) { int id = req.getSearcher() .getFirstMatch( new Term(idField.getName(), idField.getType().toInternal(idArr.get(i)))); if (id >= 0) luceneIds[docs++] = id; } DocListAndSet res = new DocListAndSet(); res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0); if (rb.isNeedDocSet()) { List<Query> queries = new ArrayList<Query>(); queries.add(rb.getQuery()); List<Query> filters = rb.getFilters(); if (filters != null) queries.addAll(filters); res.docSet = searcher.getDocSet(queries); } rb.setResults(res); rsp.add("response", rb.getResults().docList); return; } SolrIndexSearcher.QueryCommand cmd = rb.getQueryCommand(); cmd.setTimeAllowed(timeAllowed); SolrIndexSearcher.QueryResult result = new SolrIndexSearcher.QueryResult(); // // grouping / field collapsing // boolean doGroup = params.getBool(GroupParams.GROUP, false); if (doGroup) { try { cmd.groupCommands = new ArrayList<Grouping.Command>(); String[] fields = params.getParams(GroupParams.GROUP_FIELD); String[] funcs = params.getParams(GroupParams.GROUP_FUNC); String[] queries = params.getParams(GroupParams.GROUP_QUERY); String groupSortStr = params.get(GroupParams.GROUP_SORT); Sort groupSort = groupSortStr != null ? QueryParsing.parseSort(groupSortStr, req.getSchema()) : null; int limitDefault = cmd.getLen(); // this is normally from "rows" int docsPerGroupDefault = params.getInt(GroupParams.GROUP_LIMIT, 1); // temporary: implement all group-by-field as group-by-func if (funcs == null) { funcs = fields; } else if (fields != null) { // catenate functions and fields String[] both = new String[fields.length + funcs.length]; System.arraycopy(fields, 0, both, 0, fields.length); System.arraycopy(funcs, 0, both, fields.length, funcs.length); funcs = both; } if (funcs != null) { for (String groupByStr : funcs) { QParser parser = QParser.getParser(groupByStr, "func", rb.req); Query q = parser.getQuery(); Grouping.CommandFunc gc = new Grouping.CommandFunc(); gc.groupSort = groupSort; if (q instanceof FunctionQuery) { gc.groupBy = ((FunctionQuery) q).getValueSource(); } else { gc.groupBy = new QueryValueSource(q, 0.0f); } gc.key = groupByStr; gc.groupLimit = limitDefault; gc.docsPerGroup = docsPerGroupDefault; cmd.groupCommands.add(gc); } } if (cmd.groupCommands.size() == 0) cmd.groupCommands = null; if (cmd.groupCommands != null) { if (rb.doHighlights || rb.isDebug()) { // we need a single list of the returned docs cmd.setFlags(SolrIndexSearcher.GET_DOCLIST); } searcher.search(result, cmd); rb.setResult(result); rsp.add("grouped", result.groupedResults); // TODO: get "hits" a different way to log return; } } catch (ParseException e) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); } } // normal search result searcher.search(result, cmd); rb.setResult(result); rsp.add("response", rb.getResults().docList); rsp.getToLog().add("hits", rb.getResults().docList.matches()); doFieldSortValues(rb, searcher); doPrefetch(rb); }
public void handleMergeFields(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException { SolrQueryRequest req = rb.req; SolrQueryResponse rsp = rb.rsp; // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't // currently have an option to return sort field values. Because of this, we // take the documents given and re-derive the sort values. // // TODO: See SOLR-5595 boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES, false); if (fsv) { NamedList<Object[]> sortVals = new NamedList<>(); // order is important for the sort fields IndexReaderContext topReaderContext = searcher.getTopReaderContext(); List<LeafReaderContext> leaves = topReaderContext.leaves(); LeafReaderContext currentLeaf = null; if (leaves.size() == 1) { // if there is a single segment, use that subReader and avoid looking up each time currentLeaf = leaves.get(0); leaves = null; } DocList docList = rb.getResults().docList; // sort ids from lowest to highest so we can access them in order int nDocs = docList.size(); final long[] sortedIds = new long[nDocs]; final float[] scores = new float[nDocs]; // doc scores, parallel to sortedIds DocList docs = rb.getResults().docList; DocIterator it = docs.iterator(); for (int i = 0; i < nDocs; i++) { sortedIds[i] = (((long) it.nextDoc()) << 32) | i; scores[i] = docs.hasScores() ? it.score() : Float.NaN; } // sort ids and scores together new InPlaceMergeSorter() { @Override protected void swap(int i, int j) { long tmpId = sortedIds[i]; float tmpScore = scores[i]; sortedIds[i] = sortedIds[j]; scores[i] = scores[j]; sortedIds[j] = tmpId; scores[j] = tmpScore; } @Override protected int compare(int i, int j) { return Long.compare(sortedIds[i], sortedIds[j]); } }.sort(0, sortedIds.length); SortSpec sortSpec = rb.getSortSpec(); Sort sort = searcher.weightSort(sortSpec.getSort()); SortField[] sortFields = sort == null ? new SortField[] {SortField.FIELD_SCORE} : sort.getSort(); List<SchemaField> schemaFields = sortSpec.getSchemaFields(); for (int fld = 0; fld < schemaFields.size(); fld++) { SchemaField schemaField = schemaFields.get(fld); FieldType ft = null == schemaField ? null : schemaField.getType(); SortField sortField = sortFields[fld]; SortField.Type type = sortField.getType(); // :TODO: would be simpler to always serialize every position of SortField[] if (type == SortField.Type.SCORE || type == SortField.Type.DOC) continue; FieldComparator<?> comparator = null; LeafFieldComparator leafComparator = null; Object[] vals = new Object[nDocs]; int lastIdx = -1; int idx = 0; for (int i = 0; i < sortedIds.length; ++i) { long idAndPos = sortedIds[i]; float score = scores[i]; int doc = (int) (idAndPos >>> 32); int position = (int) idAndPos; if (leaves != null) { idx = ReaderUtil.subIndex(doc, leaves); currentLeaf = leaves.get(idx); if (idx != lastIdx) { // we switched segments. invalidate comparator. comparator = null; } } if (comparator == null) { comparator = sortField.getComparator(1, 0); leafComparator = comparator.getLeafComparator(currentLeaf); } doc -= currentLeaf.docBase; // adjust for what segment this is in leafComparator.setScorer(new FakeScorer(doc, score)); leafComparator.copy(0, doc); Object val = comparator.value(0); if (null != ft) val = ft.marshalSortValue(val); vals[position] = val; } sortVals.add(sortField.getField(), vals); } rsp.add("merge_values", sortVals); } }