@SuppressWarnings({"rawtypes", "unchecked"}) private Iterator<Integer> docIterator(ResponseBuilder rb) { if (rb.grouping()) { List<Integer> docList = new ArrayList<>(); NamedList values = rb.rsp.getValues(); NamedList grouped = (NamedList) values.get("grouped"); for (String field : rb.getGroupingSpec().getFields()) { NamedList fieldResults = (NamedList) grouped.get(field); if (rb.getGroupingSpec().getResponseFormat() == Grouping.Format.grouped) { List<NamedList> groups = (List<NamedList>) fieldResults.get("groups"); for (NamedList group : groups) { for (DocIterator it = ((DocList) group.get("doclist")).iterator(); it.hasNext(); ) { docList.add(it.nextDoc()); } } } else { for (DocIterator it = ((DocList) fieldResults.get("doclist")).iterator(); it.hasNext(); ) { docList.add(it.nextDoc()); } } } return docList.iterator(); } else { return rb.getResults().docList.iterator(); } }
public static SolrReaderSetScorer createReaderSetScorer( Weight weight, AtomicReaderContext context, Bits acceptDocs, SolrIndexSearcher searcher, String authorities, AtomicReader reader) throws IOException { DocSet readableDocSet = (DocSet) searcher.cacheLookup(CacheConstants.ALFRESCO_READER_CACHE, authorities); if (readableDocSet == null) { String[] auths = authorities.substring(1).split(authorities.substring(0, 1)); readableDocSet = new BitDocSet(new FixedBitSet(searcher.maxDoc())); BooleanQuery bQuery = new BooleanQuery(); for (String current : auths) { bQuery.add(new TermQuery(new Term(QueryConstants.FIELD_READER, current)), Occur.SHOULD); } DocSet aclDocs = searcher.getDocSet(bQuery); BooleanQuery aQuery = new BooleanQuery(); for (DocIterator it = aclDocs.iterator(); it.hasNext(); /**/ ) { int docID = it.nextDoc(); // Obtain the ACL ID for this ACL doc. long aclID = searcher.getAtomicReader().getNumericDocValues(QueryConstants.FIELD_ACLID).get(docID); SchemaField schemaField = searcher.getSchema().getField(QueryConstants.FIELD_ACLID); Query query = schemaField.getType().getFieldQuery(null, schemaField, Long.toString(aclID)); aQuery.add(query, Occur.SHOULD); if ((aQuery.clauses().size() > 999) || !it.hasNext()) { DocSet docsForAclId = searcher.getDocSet(aQuery); readableDocSet = readableDocSet.union(docsForAclId); aQuery = new BooleanQuery(); } } // Exclude the ACL docs from the results, we only want real docs that match. // Probably not very efficient, what we really want is remove(docID) readableDocSet = readableDocSet.andNot(aclDocs); searcher.cacheInsert(CacheConstants.ALFRESCO_READER_CACHE, authorities, readableDocSet); } // TODO: cache the full set? e.g. searcher.cacheInsert(CacheConstants.ALFRESCO_READERSET_CACHE, // authorities, readableDocSet) // plus check of course, for presence in cache at start of method. return new SolrReaderSetScorer(weight, readableDocSet, context, acceptDocs, searcher); }
public InvertResult invertScan(IndexSchema schema, InvertParams params) throws Exception { InvertResult rtn = new InvertResult(); rtn.setParams(schema, params); DocSet docset = params.getDocset(); DocSet[] subdocset = new DocSet[subReaders.length]; if (subdocset.length == 1) { subdocset[0] = docset; } else { for (int i = 0; i < subReaders.length; i++) { subdocset[i] = new BitDocSet(); } int index = 0; int end = this.getend(index); DocIterator iter = docset.iterator(); while (iter.hasNext()) { int doc = iter.nextDoc(); if (doc >= end) { index = this.readerIndex(doc); end = this.getend(index); } subdocset[index].add(doc - this.starts[index]); } } for (int i = 0; i < subReaders.length; i++) { params.setDocset(subdocset[i]); rtn.merge(subReaders[i].invertScan(schema, params)); } return rtn; }
int collect(DocSet docs, int slot) throws IOException { int count = 0; SolrIndexSearcher searcher = fcontext.searcher; final List<LeafReaderContext> leaves = searcher.getIndexReader().leaves(); final Iterator<LeafReaderContext> ctxIt = leaves.iterator(); LeafReaderContext ctx = null; int segBase = 0; int segMax; int adjustedMax = 0; for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) { final int doc = docsIt.nextDoc(); if (doc >= adjustedMax) { do { ctx = ctxIt.next(); if (ctx == null) { // should be impossible throw new RuntimeException("INTERNAL FACET ERROR"); } segBase = ctx.docBase; segMax = ctx.reader().maxDoc(); adjustedMax = segBase + segMax; } while (doc >= adjustedMax); assert doc >= ctx.docBase; setNextReader(ctx); } count++; collect(doc - segBase, slot); // per-seg collectors } return count; }
protected void processIds( ResponseBuilder rb, DocList dl, IndexSchema schema, SolrIndexSearcher searcher) throws IOException { StringBuilder sb = new StringBuilder(); Set<String> fields = Collections.singleton(schema.getUniqueKeyField().getName()); for (DocIterator iter = dl.iterator(); iter.hasNext(); ) { sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))).append(','); } if (sb.length() > 0) { rb.rsp.addToLog("responseLog", sb.substring(0, sb.length() - 1)); } }
public static SolrCachingAuxDocScorer createAuxDocScorer( SolrIndexSearcher searcher, Similarity similarity, Query query, SolrIndexReader reader) throws IOException { // Get hold of solr top level searcher // Execute query with caching // translate reults to leaf docs // build ordered doc list DocSet auxDocSet = searcher.getDocSet(query); CacheEntry[] indexedByDocId = (CacheEntry[]) searcher.cacheLookup( AlfrescoSolrEventListener.ALFRESCO_CACHE, AlfrescoSolrEventListener.KEY_DBID_LEAF_PATH_BY_DOC_ID); // List<ScoreDoc> auxDocs = pathCollector.getDocs(); OpenBitSet translated = new OpenBitSet(); if (auxDocSet instanceof BitDocSet) { BitDocSet source = (BitDocSet) auxDocSet; OpenBitSet openBitSet = source.getBits(); int current = -1; while ((current = openBitSet.nextSetBit(current + 1)) != -1) { CacheEntry entry = indexedByDocId[current]; translated.set(entry.getLeaf()); } } else { for (DocIterator it = auxDocSet.iterator(); it.hasNext(); /* */ ) { CacheEntry entry = indexedByDocId[it.nextDoc()]; translated.set(entry.getLeaf()); } } return new SolrCachingAuxDocScorer(similarity, new BitDocSet(translated), reader); }
public NamedList get(String[] fields, DocSet baseDocs) throws IOException, ParseException { if (this.crcget == null) { this.container = this.parse.createContainer(fields, baseDocs, this.reader, this.searcher, this.req); DocIterator iter = baseDocs.iterator(); this.recordCount.inc(baseDocs.size()); Doclist res = new Doclist(this.parse.limit_offset); int doc = -1; while (iter.hasNext()) { doc = iter.nextDoc(); res.add(doc); if (res.index >= this.parse.limit_offset) { break; } } PriorityQueue<SelectDetailRow> topItems = this.transGroupValue(res, fields); this.container.free(); return this.toNameList(topItems); } String hostkey = String.valueOf(this.getkeyCrc()); ConcurrentHashMap<Long, String> cache = MdrillUtils.CRC_CACHE_SIZE.remove(crcget + "@" + hostkey); NamedList rtn = new NamedList(); Map<Long, String> crcvalue = new HashMap<Long, String>(); rtn.add("fdtcre", crcvalue); if (cache != null) { MapFieldSelector selector = new MapFieldSelector(fields); FieldType[] ftlist = new FieldType[fields.length]; IndexSchema schema = this.searcher.getSchema(); for (int j = 0; j < fields.length; j++) { ftlist[j] = schema.getFieldType(fields[j]); } String crcliststr = params.get("mdrill.crc.key.get.crclist"); if (crcliststr != null) { String[] crclist = crcliststr.split(","); for (String s : crclist) { Long crc = Long.parseLong(s); String v = cache.get(crc); if (v != null) { String cols[] = v.split(UniqConfig.GroupJoinString(), -1); if (cols.length >= 2) { int doc = Integer.parseInt(cols[0]); SortGroupVal buff = new SortGroupVal(); buff.groupbuff.append("-"); buff.groupbuff.append(UniqConfig.GroupJoinString()); buff.groupbuff.append("-"); Document docfields = this.reader.document(doc, selector); if (docfields == null) { for (int j = 0; j < fields.length; j++) { buff.groupbuff.append(UniqConfig.GroupJoinString()); buff.groupbuff.append(EncodeUtils.encode("-")); } if (!crcvalue.containsKey(crc)) { crcvalue.put(crc, buff.groupbuff.toString()); } } else { for (int j = 0; j < fields.length; j++) { buff.groupbuff.append(UniqConfig.GroupJoinString()); Fieldable fv = docfields.getFieldable(fields[j]); if (fv != null) { buff.groupbuff.append(ftlist[j].toExternal(fv)); } else { buff.groupbuff.append(EncodeUtils.encode("-")); } } crcvalue.put(crc, buff.groupbuff.toString()); } } } } } } return rtn; }
protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException { SolrQueryRequest req = rb.req; SolrQueryResponse rsp = rb.rsp; // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't // currently have an option to return sort field values. Because of this, we // take the documents given and re-derive the sort values. boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES, false); if (fsv) { Sort sort = rb.getSortSpec().getSort(); SortField[] sortFields = sort == null ? new SortField[] {SortField.FIELD_SCORE} : sort.getSort(); NamedList sortVals = new NamedList(); // order is important for the sort fields Field field = new Field("dummy", "", Field.Store.YES, Field.Index.NO); // a dummy Field SolrIndexReader reader = searcher.getReader(); SolrIndexReader[] readers = reader.getLeafReaders(); SolrIndexReader subReader = reader; if (readers.length == 1) { // if there is a single segment, use that subReader and avoid looking up each time subReader = readers[0]; readers = null; } int[] offsets = reader.getLeafOffsets(); for (SortField sortField : sortFields) { int type = sortField.getType(); if (type == SortField.SCORE || type == SortField.DOC) continue; FieldComparator comparator = null; FieldComparator comparators[] = (readers == null) ? null : new FieldComparator[readers.length]; String fieldname = sortField.getField(); FieldType ft = fieldname == null ? null : req.getSchema().getFieldTypeNoEx(fieldname); DocList docList = rb.getResults().docList; ArrayList<Object> vals = new ArrayList<Object>(docList.size()); DocIterator it = rb.getResults().docList.iterator(); int offset = 0; int idx = 0; while (it.hasNext()) { int doc = it.nextDoc(); if (readers != null) { idx = SolrIndexReader.readerIndex(doc, offsets); subReader = readers[idx]; offset = offsets[idx]; comparator = comparators[idx]; } if (comparator == null) { comparator = sortField.getComparator(1, 0); comparator = comparator.setNextReader(subReader, offset); if (comparators != null) comparators[idx] = comparator; } doc -= offset; // adjust for what segment this is in comparator.copy(0, doc); Object val = comparator.value(0); // Sortable float, double, int, long types all just use a string // comparator. For these, we need to put the type into a readable // format. One reason for this is that XML can't represent all // string values (or even all unicode code points). // indexedToReadable() should be a no-op and should // thus be harmless anyway (for all current ways anyway) if (val instanceof String) { field.setValue((String) val); val = ft.toObject(field); } // Must do the same conversion when sorting by a // String field in Lucene, which returns the terms // data as BytesRef: if (val instanceof BytesRef) { field.setValue(((BytesRef) val).utf8ToString()); val = ft.toObject(field); } vals.add(val); } sortVals.add(fieldname, vals); } rsp.add("sort_values", sortVals); } }