Пример #1
0
  @Override
  public void transform(SolrDocument doc, int docid) {
    // This is only good for random-access functions

    try {

      // TODO: calculate this stuff just once across diff functions
      int idx = ReaderUtil.subIndex(docid, readerContexts);
      LeafReaderContext rcontext = readerContexts.get(idx);
      FunctionValues values = docValuesArr[idx];
      if (values == null) {
        docValuesArr[idx] = values = valueSource.getValues(fcontext, rcontext);
      }

      int localId = docid - rcontext.docBase;
      setValue(doc, values.objectVal(localId));
    } catch (IOException e) {
      throw new SolrException(
          SolrException.ErrorCode.SERVER_ERROR,
          "exception at docid " + docid + " for valuesource " + valueSource,
          e);
    }
  }
Пример #2
0
  @Override
  public void execute(SearchContext context) {
    FieldsVisitor fieldsVisitor;
    Set<String> fieldNames = null;
    List<String> extractFieldNames = null;

    boolean loadAllStored = false;
    if (!context.hasFieldNames()) {
      // no fields specified, default to return source if no explicit indication
      if (!context.hasScriptFields() && !context.hasFetchSourceContext()) {
        context.fetchSourceContext(new FetchSourceContext(true));
      }
      fieldsVisitor = new FieldsVisitor(context.sourceRequested());
    } else if (context.fieldNames().isEmpty()) {
      fieldsVisitor = new FieldsVisitor(context.sourceRequested());
    } else {
      for (String fieldName : context.fieldNames()) {
        if (fieldName.equals("*")) {
          loadAllStored = true;
          continue;
        }
        if (fieldName.equals(SourceFieldMapper.NAME)) {
          if (context.hasFetchSourceContext()) {
            context.fetchSourceContext().fetchSource(true);
          } else {
            context.fetchSourceContext(new FetchSourceContext(true));
          }
          continue;
        }
        MappedFieldType fieldType = context.smartNameFieldType(fieldName);
        if (fieldType == null) {
          // Only fail if we know it is a object field, missing paths / fields shouldn't fail.
          if (context.getObjectMapper(fieldName) != null) {
            throw new IllegalArgumentException("field [" + fieldName + "] isn't a leaf field");
          }
        } else if (fieldType.stored()) {
          if (fieldNames == null) {
            fieldNames = new HashSet<>();
          }
          fieldNames.add(fieldType.names().indexName());
        } else {
          if (extractFieldNames == null) {
            extractFieldNames = newArrayList();
          }
          extractFieldNames.add(fieldName);
        }
      }
      if (loadAllStored) {
        fieldsVisitor = new AllFieldsVisitor(); // load everything, including _source
      } else if (fieldNames != null) {
        boolean loadSource = extractFieldNames != null || context.sourceRequested();
        fieldsVisitor = new CustomFieldsVisitor(fieldNames, loadSource);
      } else {
        fieldsVisitor = new FieldsVisitor(extractFieldNames != null || context.sourceRequested());
      }
    }

    InternalSearchHit[] hits = new InternalSearchHit[context.docIdsToLoadSize()];
    FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext();
    for (int index = 0; index < context.docIdsToLoadSize(); index++) {
      int docId = context.docIdsToLoad()[context.docIdsToLoadFrom() + index];
      int readerIndex = ReaderUtil.subIndex(docId, context.searcher().getIndexReader().leaves());
      LeafReaderContext subReaderContext =
          context.searcher().getIndexReader().leaves().get(readerIndex);
      int subDocId = docId - subReaderContext.docBase;

      final InternalSearchHit searchHit;
      try {
        int rootDocId = findRootDocumentIfNested(context, subReaderContext, subDocId);
        if (rootDocId != -1) {
          searchHit =
              createNestedSearchHit(
                  context,
                  docId,
                  subDocId,
                  rootDocId,
                  extractFieldNames,
                  loadAllStored,
                  fieldNames,
                  subReaderContext);
        } else {
          searchHit =
              createSearchHit(
                  context, fieldsVisitor, docId, subDocId, extractFieldNames, subReaderContext);
        }
      } catch (IOException e) {
        throw ExceptionsHelper.convertToElastic(e);
      }

      hits[index] = searchHit;
      hitContext.reset(searchHit, subReaderContext, subDocId, context.searcher());
      for (FetchSubPhase fetchSubPhase : fetchSubPhases) {
        if (fetchSubPhase.hitExecutionNeeded(context)) {
          fetchSubPhase.hitExecute(context, hitContext);
        }
      }
    }

    for (FetchSubPhase fetchSubPhase : fetchSubPhases) {
      if (fetchSubPhase.hitsExecutionNeeded(context)) {
        fetchSubPhase.hitsExecute(context, hits);
      }
    }

    context
        .fetchResult()
        .hits(
            new InternalSearchHits(
                hits,
                context.queryResult().topDocs().totalHits,
                context.queryResult().topDocs().getMaxScore()));
  }
Пример #3
0
        @Override
        public PercolateShardResponse doPercolate(
            PercolateShardRequest request, PercolateContext context) {
          Engine.Searcher percolatorSearcher = context.indexShard().acquireSearcher("percolate");
          try {
            MatchAndSort matchAndSort = QueryCollector.matchAndSort(logger, context);
            queryBasedPercolating(percolatorSearcher, context, matchAndSort);
            TopDocs topDocs = matchAndSort.topDocs();
            long count = topDocs.totalHits;
            List<BytesRef> matches = new ArrayList<BytesRef>(topDocs.scoreDocs.length);
            float[] scores = new float[topDocs.scoreDocs.length];
            List<Map<String, HighlightField>> hls = null;
            if (context.highlight() != null) {
              hls = new ArrayList<Map<String, HighlightField>>(topDocs.scoreDocs.length);
            }

            final FieldMapper<?> idMapper =
                context.mapperService().smartNameFieldMapper(IdFieldMapper.NAME);
            final IndexFieldData<?> idFieldData = context.fieldData().getForField(idMapper);
            int i = 0;
            final HashedBytesRef spare = new HashedBytesRef(new BytesRef());
            for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
              int segmentIdx =
                  ReaderUtil.subIndex(scoreDoc.doc, percolatorSearcher.reader().leaves());
              AtomicReaderContext atomicReaderContext =
                  percolatorSearcher.reader().leaves().get(segmentIdx);
              BytesValues values = idFieldData.load(atomicReaderContext).getBytesValues(true);
              final int localDocId = scoreDoc.doc - atomicReaderContext.docBase;
              final int numValues = values.setDocument(localDocId);
              assert numValues == 1;
              spare.bytes = values.nextValue();
              spare.hash = values.currentValueHash();
              matches.add(values.copyShared());
              if (hls != null) {
                Query query = context.percolateQueries().get(spare);
                context.parsedQuery(new ParsedQuery(query, ImmutableMap.<String, Filter>of()));
                context.hitContext().cache().clear();
                highlightPhase.hitExecute(context, context.hitContext());
                hls.add(i, context.hitContext().hit().getHighlightFields());
              }
              scores[i++] = scoreDoc.score;
            }
            if (hls != null) {
              return new PercolateShardResponse(
                  matches.toArray(new BytesRef[matches.size()]),
                  hls,
                  count,
                  scores,
                  context,
                  request.index(),
                  request.shardId());
            } else {
              return new PercolateShardResponse(
                  matches.toArray(new BytesRef[matches.size()]),
                  count,
                  scores,
                  context,
                  request.index(),
                  request.shardId());
            }
          } catch (Throwable e) {
            logger.debug("failed to execute", e);
            throw new PercolateException(context.indexShard().shardId(), "failed to execute", e);
          } finally {
            percolatorSearcher.release();
          }
        }
  private Map<Integer, Object> highlightField(
      String field,
      String contents[],
      BreakIterator bi,
      BytesRef terms[],
      int[] docids,
      List<AtomicReaderContext> leaves,
      int maxPassages,
      Query query)
      throws IOException {
    Map<Integer, Object> highlights = new HashMap<>();

    PassageFormatter fieldFormatter = getFormatter(field);
    if (fieldFormatter == null) {
      throw new NullPointerException("PassageFormatter cannot be null");
    }

    // check if we should do any multiterm processing
    Analyzer analyzer = getIndexAnalyzer(field);
    CharacterRunAutomaton automata[] = new CharacterRunAutomaton[0];
    if (analyzer != null) {
      automata = MultiTermHighlighting.extractAutomata(query, field);
    }

    // resize 'terms', where the last term is the multiterm matcher
    if (automata.length > 0) {
      BytesRef newTerms[] = new BytesRef[terms.length + 1];
      System.arraycopy(terms, 0, newTerms, 0, terms.length);
      terms = newTerms;
    }

    // we are processing in increasing docid order, so we only need to reinitialize stuff on segment
    // changes
    // otherwise, we will just advance() existing enums to the new document in the same segment.
    DocsAndPositionsEnum postings[] = null;
    TermsEnum termsEnum = null;
    int lastLeaf = -1;

    for (int i = 0; i < docids.length; i++) {
      String content = contents[i];
      if (content.length() == 0) {
        continue; // nothing to do
      }
      bi.setText(content);
      int doc = docids[i];
      int leaf = ReaderUtil.subIndex(doc, leaves);
      AtomicReaderContext subContext = leaves.get(leaf);
      AtomicReader r = subContext.reader();

      assert leaf >= lastLeaf; // increasing order

      // if the segment has changed, we must initialize new enums.
      if (leaf != lastLeaf) {
        Terms t = r.terms(field);
        if (t != null) {
          termsEnum = t.iterator(null);
          postings = new DocsAndPositionsEnum[terms.length];
        }
      }
      if (termsEnum == null) {
        continue; // no terms for this field, nothing to do
      }

      // if there are multi-term matches, we have to initialize the "fake" enum for each document
      if (automata.length > 0) {
        DocsAndPositionsEnum dp =
            MultiTermHighlighting.getDocsEnum(analyzer.tokenStream(field, content), automata);
        dp.advance(doc - subContext.docBase);
        postings[terms.length - 1] = dp; // last term is the multiterm matcher
      }

      Passage passages[] =
          highlightDoc(
              field,
              terms,
              content.length(),
              bi,
              doc - subContext.docBase,
              termsEnum,
              postings,
              maxPassages);

      if (passages.length == 0) {
        // no passages were returned, so ask for a default summary
        passages = getEmptyHighlight(field, bi, maxPassages);
      }

      if (passages.length > 0) {
        highlights.put(doc, fieldFormatter.format(passages, content));
      }

      lastLeaf = leaf;
    }

    return highlights;
  }
  int resolveParentDocuments(
      TopDocs topDocs,
      SearchContext context,
      Recycler.V<ObjectObjectOpenHashMap<Object, ParentDoc[]>> parentDocs) {
    int parentHitsResolved = 0;
    Recycler.V<ObjectObjectOpenHashMap<Object, Recycler.V<IntObjectOpenHashMap<ParentDoc>>>>
        parentDocsPerReader =
            cacheRecycler.hashMap(context.searcher().getIndexReader().leaves().size());
    for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
      int readerIndex =
          ReaderUtil.subIndex(scoreDoc.doc, context.searcher().getIndexReader().leaves());
      AtomicReaderContext subContext =
          context.searcher().getIndexReader().leaves().get(readerIndex);
      int subDoc = scoreDoc.doc - subContext.docBase;

      // find the parent id
      HashedBytesArray parentId =
          context.idCache().reader(subContext.reader()).parentIdByDoc(parentType, subDoc);
      if (parentId == null) {
        // no parent found
        continue;
      }
      // now go over and find the parent doc Id and reader tuple
      for (AtomicReaderContext atomicReaderContext : context.searcher().getIndexReader().leaves()) {
        AtomicReader indexReader = atomicReaderContext.reader();
        int parentDocId = context.idCache().reader(indexReader).docById(parentType, parentId);
        Bits liveDocs = indexReader.getLiveDocs();
        if (parentDocId != -1 && (liveDocs == null || liveDocs.get(parentDocId))) {
          // we found a match, add it and break

          Recycler.V<IntObjectOpenHashMap<ParentDoc>> readerParentDocs =
              parentDocsPerReader.v().get(indexReader.getCoreCacheKey());
          if (readerParentDocs == null) {
            readerParentDocs = cacheRecycler.intObjectMap(indexReader.maxDoc());
            parentDocsPerReader.v().put(indexReader.getCoreCacheKey(), readerParentDocs);
          }

          ParentDoc parentDoc = readerParentDocs.v().get(parentDocId);
          if (parentDoc == null) {
            parentHitsResolved++; // we have a hit on a parent
            parentDoc = new ParentDoc();
            parentDoc.docId = parentDocId;
            parentDoc.count = 1;
            parentDoc.maxScore = scoreDoc.score;
            parentDoc.sumScores = scoreDoc.score;
            readerParentDocs.v().put(parentDocId, parentDoc);
          } else {
            parentDoc.count++;
            parentDoc.sumScores += scoreDoc.score;
            if (scoreDoc.score > parentDoc.maxScore) {
              parentDoc.maxScore = scoreDoc.score;
            }
          }
        }
      }
    }
    boolean[] states = parentDocsPerReader.v().allocated;
    Object[] keys = parentDocsPerReader.v().keys;
    Object[] values = parentDocsPerReader.v().values;
    for (int i = 0; i < states.length; i++) {
      if (states[i]) {
        Recycler.V<IntObjectOpenHashMap<ParentDoc>> value =
            (Recycler.V<IntObjectOpenHashMap<ParentDoc>>) values[i];
        ParentDoc[] _parentDocs = value.v().values().toArray(ParentDoc.class);
        Arrays.sort(_parentDocs, PARENT_DOC_COMP);
        parentDocs.v().put(keys[i], _parentDocs);
        Releasables.release(value);
      }
    }
    Releasables.release(parentDocsPerReader);
    return parentHitsResolved;
  }
Пример #6
0
    public void handleMergeFields(ResponseBuilder rb, SolrIndexSearcher searcher)
        throws IOException {
      SolrQueryRequest req = rb.req;
      SolrQueryResponse rsp = rb.rsp;
      // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't
      // currently have an option to return sort field values.  Because of this, we
      // take the documents given and re-derive the sort values.
      //
      // TODO: See SOLR-5595
      boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES, false);
      if (fsv) {
        NamedList<Object[]> sortVals = new NamedList<>(); // order is important for the sort fields
        IndexReaderContext topReaderContext = searcher.getTopReaderContext();
        List<LeafReaderContext> leaves = topReaderContext.leaves();
        LeafReaderContext currentLeaf = null;
        if (leaves.size() == 1) {
          // if there is a single segment, use that subReader and avoid looking up each time
          currentLeaf = leaves.get(0);
          leaves = null;
        }

        DocList docList = rb.getResults().docList;

        // sort ids from lowest to highest so we can access them in order
        int nDocs = docList.size();
        final long[] sortedIds = new long[nDocs];
        final float[] scores = new float[nDocs]; // doc scores, parallel to sortedIds
        DocList docs = rb.getResults().docList;
        DocIterator it = docs.iterator();
        for (int i = 0; i < nDocs; i++) {
          sortedIds[i] = (((long) it.nextDoc()) << 32) | i;
          scores[i] = docs.hasScores() ? it.score() : Float.NaN;
        }

        // sort ids and scores together
        new InPlaceMergeSorter() {
          @Override
          protected void swap(int i, int j) {
            long tmpId = sortedIds[i];
            float tmpScore = scores[i];
            sortedIds[i] = sortedIds[j];
            scores[i] = scores[j];
            sortedIds[j] = tmpId;
            scores[j] = tmpScore;
          }

          @Override
          protected int compare(int i, int j) {
            return Long.compare(sortedIds[i], sortedIds[j]);
          }
        }.sort(0, sortedIds.length);

        SortSpec sortSpec = rb.getSortSpec();
        Sort sort = searcher.weightSort(sortSpec.getSort());
        SortField[] sortFields =
            sort == null ? new SortField[] {SortField.FIELD_SCORE} : sort.getSort();
        List<SchemaField> schemaFields = sortSpec.getSchemaFields();

        for (int fld = 0; fld < schemaFields.size(); fld++) {
          SchemaField schemaField = schemaFields.get(fld);
          FieldType ft = null == schemaField ? null : schemaField.getType();
          SortField sortField = sortFields[fld];

          SortField.Type type = sortField.getType();
          // :TODO: would be simpler to always serialize every position of SortField[]
          if (type == SortField.Type.SCORE || type == SortField.Type.DOC) continue;

          FieldComparator<?> comparator = null;
          LeafFieldComparator leafComparator = null;
          Object[] vals = new Object[nDocs];

          int lastIdx = -1;
          int idx = 0;

          for (int i = 0; i < sortedIds.length; ++i) {
            long idAndPos = sortedIds[i];
            float score = scores[i];
            int doc = (int) (idAndPos >>> 32);
            int position = (int) idAndPos;

            if (leaves != null) {
              idx = ReaderUtil.subIndex(doc, leaves);
              currentLeaf = leaves.get(idx);
              if (idx != lastIdx) {
                // we switched segments.  invalidate comparator.
                comparator = null;
              }
            }

            if (comparator == null) {
              comparator = sortField.getComparator(1, 0);
              leafComparator = comparator.getLeafComparator(currentLeaf);
            }

            doc -= currentLeaf.docBase; // adjust for what segment this is in
            leafComparator.setScorer(new FakeScorer(doc, score));
            leafComparator.copy(0, doc);
            Object val = comparator.value(0);
            if (null != ft) val = ft.marshalSortValue(val);
            vals[position] = val;
          }

          sortVals.add(sortField.getField(), vals);
        }

        rsp.add("merge_values", sortVals);
      }
    }