예제 #1
0
 @SuppressWarnings({"rawtypes", "unchecked"})
 private Iterator<Integer> docIterator(ResponseBuilder rb) {
   if (rb.grouping()) {
     List<Integer> docList = new ArrayList<>();
     NamedList values = rb.rsp.getValues();
     NamedList grouped = (NamedList) values.get("grouped");
     for (String field : rb.getGroupingSpec().getFields()) {
       NamedList fieldResults = (NamedList) grouped.get(field);
       if (rb.getGroupingSpec().getResponseFormat() == Grouping.Format.grouped) {
         List<NamedList> groups = (List<NamedList>) fieldResults.get("groups");
         for (NamedList group : groups) {
           for (DocIterator it = ((DocList) group.get("doclist")).iterator(); it.hasNext(); ) {
             docList.add(it.nextDoc());
           }
         }
       } else {
         for (DocIterator it = ((DocList) fieldResults.get("doclist")).iterator();
             it.hasNext(); ) {
           docList.add(it.nextDoc());
         }
       }
     }
     return docList.iterator();
   } else {
     return rb.getResults().docList.iterator();
   }
 }
  public static SolrReaderSetScorer createReaderSetScorer(
      Weight weight,
      AtomicReaderContext context,
      Bits acceptDocs,
      SolrIndexSearcher searcher,
      String authorities,
      AtomicReader reader)
      throws IOException {

    DocSet readableDocSet =
        (DocSet) searcher.cacheLookup(CacheConstants.ALFRESCO_READER_CACHE, authorities);

    if (readableDocSet == null) {

      String[] auths = authorities.substring(1).split(authorities.substring(0, 1));

      readableDocSet = new BitDocSet(new FixedBitSet(searcher.maxDoc()));

      BooleanQuery bQuery = new BooleanQuery();
      for (String current : auths) {
        bQuery.add(new TermQuery(new Term(QueryConstants.FIELD_READER, current)), Occur.SHOULD);
      }

      DocSet aclDocs = searcher.getDocSet(bQuery);

      BooleanQuery aQuery = new BooleanQuery();
      for (DocIterator it = aclDocs.iterator(); it.hasNext(); /**/ ) {
        int docID = it.nextDoc();
        // Obtain the ACL ID for this ACL doc.
        long aclID =
            searcher.getAtomicReader().getNumericDocValues(QueryConstants.FIELD_ACLID).get(docID);
        SchemaField schemaField = searcher.getSchema().getField(QueryConstants.FIELD_ACLID);
        Query query = schemaField.getType().getFieldQuery(null, schemaField, Long.toString(aclID));
        aQuery.add(query, Occur.SHOULD);

        if ((aQuery.clauses().size() > 999) || !it.hasNext()) {
          DocSet docsForAclId = searcher.getDocSet(aQuery);
          readableDocSet = readableDocSet.union(docsForAclId);

          aQuery = new BooleanQuery();
        }
      }
      // Exclude the ACL docs from the results, we only want real docs that match.
      // Probably not very efficient, what we really want is remove(docID)
      readableDocSet = readableDocSet.andNot(aclDocs);
      searcher.cacheInsert(CacheConstants.ALFRESCO_READER_CACHE, authorities, readableDocSet);
    }

    // TODO: cache the full set? e.g. searcher.cacheInsert(CacheConstants.ALFRESCO_READERSET_CACHE,
    // authorities, readableDocSet)
    // plus check of course, for presence in cache at start of method.
    return new SolrReaderSetScorer(weight, readableDocSet, context, acceptDocs, searcher);
  }
예제 #3
0
  public InvertResult invertScan(IndexSchema schema, InvertParams params) throws Exception {
    InvertResult rtn = new InvertResult();
    rtn.setParams(schema, params);
    DocSet docset = params.getDocset();
    DocSet[] subdocset = new DocSet[subReaders.length];
    if (subdocset.length == 1) {
      subdocset[0] = docset;
    } else {
      for (int i = 0; i < subReaders.length; i++) {
        subdocset[i] = new BitDocSet();
      }

      int index = 0;
      int end = this.getend(index);
      DocIterator iter = docset.iterator();
      while (iter.hasNext()) {
        int doc = iter.nextDoc();
        if (doc >= end) {
          index = this.readerIndex(doc);
          end = this.getend(index);
        }
        subdocset[index].add(doc - this.starts[index]);
      }
    }
    for (int i = 0; i < subReaders.length; i++) {
      params.setDocset(subdocset[i]);
      rtn.merge(subReaders[i].invertScan(schema, params));
    }
    return rtn;
  }
예제 #4
0
  int collect(DocSet docs, int slot) throws IOException {
    int count = 0;
    SolrIndexSearcher searcher = fcontext.searcher;

    final List<LeafReaderContext> leaves = searcher.getIndexReader().leaves();
    final Iterator<LeafReaderContext> ctxIt = leaves.iterator();
    LeafReaderContext ctx = null;
    int segBase = 0;
    int segMax;
    int adjustedMax = 0;
    for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) {
      final int doc = docsIt.nextDoc();
      if (doc >= adjustedMax) {
        do {
          ctx = ctxIt.next();
          if (ctx == null) {
            // should be impossible
            throw new RuntimeException("INTERNAL FACET ERROR");
          }
          segBase = ctx.docBase;
          segMax = ctx.reader().maxDoc();
          adjustedMax = segBase + segMax;
        } while (doc >= adjustedMax);
        assert doc >= ctx.docBase;
        setNextReader(ctx);
      }
      count++;
      collect(doc - segBase, slot); // per-seg collectors
    }
    return count;
  }
예제 #5
0
  protected void processIds(
      ResponseBuilder rb, DocList dl, IndexSchema schema, SolrIndexSearcher searcher)
      throws IOException {

    StringBuilder sb = new StringBuilder();

    Set<String> fields = Collections.singleton(schema.getUniqueKeyField().getName());
    for (DocIterator iter = dl.iterator(); iter.hasNext(); ) {

      sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))).append(',');
    }
    if (sb.length() > 0) {
      rb.rsp.addToLog("responseLog", sb.substring(0, sb.length() - 1));
    }
  }
예제 #6
0
  public static SolrCachingAuxDocScorer createAuxDocScorer(
      SolrIndexSearcher searcher, Similarity similarity, Query query, SolrIndexReader reader)
      throws IOException {
    // Get hold of solr top level searcher
    // Execute query with caching
    // translate reults to leaf docs
    // build ordered doc list

    DocSet auxDocSet = searcher.getDocSet(query);

    CacheEntry[] indexedByDocId =
        (CacheEntry[])
            searcher.cacheLookup(
                AlfrescoSolrEventListener.ALFRESCO_CACHE,
                AlfrescoSolrEventListener.KEY_DBID_LEAF_PATH_BY_DOC_ID);

    // List<ScoreDoc> auxDocs = pathCollector.getDocs();
    OpenBitSet translated = new OpenBitSet();

    if (auxDocSet instanceof BitDocSet) {
      BitDocSet source = (BitDocSet) auxDocSet;
      OpenBitSet openBitSet = source.getBits();
      int current = -1;
      while ((current = openBitSet.nextSetBit(current + 1)) != -1) {
        CacheEntry entry = indexedByDocId[current];
        translated.set(entry.getLeaf());
      }
    } else {
      for (DocIterator it = auxDocSet.iterator(); it.hasNext(); /* */ ) {
        CacheEntry entry = indexedByDocId[it.nextDoc()];
        translated.set(entry.getLeaf());
      }
    }

    return new SolrCachingAuxDocScorer(similarity, new BitDocSet(translated), reader);
  }
예제 #7
0
  public NamedList get(String[] fields, DocSet baseDocs) throws IOException, ParseException {
    if (this.crcget == null) {
      this.container =
          this.parse.createContainer(fields, baseDocs, this.reader, this.searcher, this.req);
      DocIterator iter = baseDocs.iterator();
      this.recordCount.inc(baseDocs.size());
      Doclist res = new Doclist(this.parse.limit_offset);
      int doc = -1;
      while (iter.hasNext()) {
        doc = iter.nextDoc();
        res.add(doc);
        if (res.index >= this.parse.limit_offset) {
          break;
        }
      }
      PriorityQueue<SelectDetailRow> topItems = this.transGroupValue(res, fields);
      this.container.free();
      return this.toNameList(topItems);
    }
    String hostkey = String.valueOf(this.getkeyCrc());

    ConcurrentHashMap<Long, String> cache =
        MdrillUtils.CRC_CACHE_SIZE.remove(crcget + "@" + hostkey);

    NamedList rtn = new NamedList();
    Map<Long, String> crcvalue = new HashMap<Long, String>();
    rtn.add("fdtcre", crcvalue);
    if (cache != null) {
      MapFieldSelector selector = new MapFieldSelector(fields);
      FieldType[] ftlist = new FieldType[fields.length];
      IndexSchema schema = this.searcher.getSchema();

      for (int j = 0; j < fields.length; j++) {
        ftlist[j] = schema.getFieldType(fields[j]);
      }

      String crcliststr = params.get("mdrill.crc.key.get.crclist");
      if (crcliststr != null) {
        String[] crclist = crcliststr.split(",");
        for (String s : crclist) {
          Long crc = Long.parseLong(s);
          String v = cache.get(crc);
          if (v != null) {
            String cols[] = v.split(UniqConfig.GroupJoinString(), -1);
            if (cols.length >= 2) {
              int doc = Integer.parseInt(cols[0]);

              SortGroupVal buff = new SortGroupVal();
              buff.groupbuff.append("-");
              buff.groupbuff.append(UniqConfig.GroupJoinString());
              buff.groupbuff.append("-");
              Document docfields = this.reader.document(doc, selector);
              if (docfields == null) {
                for (int j = 0; j < fields.length; j++) {
                  buff.groupbuff.append(UniqConfig.GroupJoinString());
                  buff.groupbuff.append(EncodeUtils.encode("-"));
                }
                if (!crcvalue.containsKey(crc)) {
                  crcvalue.put(crc, buff.groupbuff.toString());
                }
              } else {

                for (int j = 0; j < fields.length; j++) {
                  buff.groupbuff.append(UniqConfig.GroupJoinString());

                  Fieldable fv = docfields.getFieldable(fields[j]);

                  if (fv != null) {
                    buff.groupbuff.append(ftlist[j].toExternal(fv));
                  } else {
                    buff.groupbuff.append(EncodeUtils.encode("-"));
                  }
                }
                crcvalue.put(crc, buff.groupbuff.toString());
              }
            }
          }
        }
      }
    }
    return rtn;
  }
예제 #8
0
  protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher)
      throws IOException {
    SolrQueryRequest req = rb.req;
    SolrQueryResponse rsp = rb.rsp;

    // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't
    // currently have an option to return sort field values.  Because of this, we
    // take the documents given and re-derive the sort values.
    boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES, false);
    if (fsv) {
      Sort sort = rb.getSortSpec().getSort();
      SortField[] sortFields =
          sort == null ? new SortField[] {SortField.FIELD_SCORE} : sort.getSort();
      NamedList sortVals = new NamedList(); // order is important for the sort fields
      Field field = new Field("dummy", "", Field.Store.YES, Field.Index.NO); // a dummy Field

      SolrIndexReader reader = searcher.getReader();
      SolrIndexReader[] readers = reader.getLeafReaders();
      SolrIndexReader subReader = reader;
      if (readers.length == 1) {
        // if there is a single segment, use that subReader and avoid looking up each time
        subReader = readers[0];
        readers = null;
      }
      int[] offsets = reader.getLeafOffsets();

      for (SortField sortField : sortFields) {
        int type = sortField.getType();
        if (type == SortField.SCORE || type == SortField.DOC) continue;

        FieldComparator comparator = null;
        FieldComparator comparators[] =
            (readers == null) ? null : new FieldComparator[readers.length];

        String fieldname = sortField.getField();
        FieldType ft = fieldname == null ? null : req.getSchema().getFieldTypeNoEx(fieldname);

        DocList docList = rb.getResults().docList;
        ArrayList<Object> vals = new ArrayList<Object>(docList.size());
        DocIterator it = rb.getResults().docList.iterator();

        int offset = 0;
        int idx = 0;

        while (it.hasNext()) {
          int doc = it.nextDoc();
          if (readers != null) {
            idx = SolrIndexReader.readerIndex(doc, offsets);
            subReader = readers[idx];
            offset = offsets[idx];
            comparator = comparators[idx];
          }

          if (comparator == null) {
            comparator = sortField.getComparator(1, 0);
            comparator = comparator.setNextReader(subReader, offset);
            if (comparators != null) comparators[idx] = comparator;
          }

          doc -= offset; // adjust for what segment this is in
          comparator.copy(0, doc);
          Object val = comparator.value(0);

          // Sortable float, double, int, long types all just use a string
          // comparator. For these, we need to put the type into a readable
          // format.  One reason for this is that XML can't represent all
          // string values (or even all unicode code points).
          // indexedToReadable() should be a no-op and should
          // thus be harmless anyway (for all current ways anyway)
          if (val instanceof String) {
            field.setValue((String) val);
            val = ft.toObject(field);
          }

          // Must do the same conversion when sorting by a
          // String field in Lucene, which returns the terms
          // data as BytesRef:
          if (val instanceof BytesRef) {
            field.setValue(((BytesRef) val).utf8ToString());
            val = ft.toObject(field);
          }

          vals.add(val);
        }

        sortVals.add(fieldname, vals);
      }

      rsp.add("sort_values", sortVals);
    }
  }