Ejemplo n.º 1
0
  private Map<String, SearchHitField> getSearchFields(
      SearchContext context,
      int nestedSubDocId,
      boolean loadAllStored,
      Set<String> fieldNames,
      LeafReaderContext subReaderContext) {
    Map<String, SearchHitField> searchFields = null;
    if (context.hasFieldNames() && !context.fieldNames().isEmpty()) {
      FieldsVisitor nestedFieldsVisitor = null;
      if (loadAllStored) {
        nestedFieldsVisitor = new AllFieldsVisitor();
      } else if (fieldNames != null) {
        nestedFieldsVisitor = new CustomFieldsVisitor(fieldNames, false);
      }

      if (nestedFieldsVisitor != null) {
        loadStoredFields(context, subReaderContext, nestedFieldsVisitor, nestedSubDocId);
        nestedFieldsVisitor.postProcess(context.mapperService());
        if (!nestedFieldsVisitor.fields().isEmpty()) {
          searchFields = new HashMap<>(nestedFieldsVisitor.fields().size());
          for (Map.Entry<String, List<Object>> entry : nestedFieldsVisitor.fields().entrySet()) {
            searchFields.put(
                entry.getKey(), new InternalSearchHitField(entry.getKey(), entry.getValue()));
          }
        }
      }
    }
    return searchFields;
  }
Ejemplo n.º 2
0
  private InternalSearchHit createSearchHit(
      SearchContext context,
      FieldsVisitor fieldsVisitor,
      int docId,
      int subDocId,
      List<String> extractFieldNames,
      LeafReaderContext subReaderContext) {
    loadStoredFields(context, subReaderContext, fieldsVisitor, subDocId);
    fieldsVisitor.postProcess(context.mapperService());

    Map<String, SearchHitField> searchFields = null;
    if (!fieldsVisitor.fields().isEmpty()) {
      searchFields = new HashMap<>(fieldsVisitor.fields().size());
      for (Map.Entry<String, List<Object>> entry : fieldsVisitor.fields().entrySet()) {
        searchFields.put(
            entry.getKey(), new InternalSearchHitField(entry.getKey(), entry.getValue()));
      }
    }

    DocumentMapper documentMapper =
        context.mapperService().documentMapper(fieldsVisitor.uid().type());
    Text typeText;
    if (documentMapper == null) {
      typeText = new StringAndBytesText(fieldsVisitor.uid().type());
    } else {
      typeText = documentMapper.typeText();
    }
    InternalSearchHit searchHit =
        new InternalSearchHit(docId, fieldsVisitor.uid().id(), typeText, searchFields);

    // go over and extract fields that are not mapped / stored
    SourceLookup sourceLookup = context.lookup().source();
    sourceLookup.setSegmentAndDocument(subReaderContext, subDocId);
    if (fieldsVisitor.source() != null) {
      sourceLookup.setSource(fieldsVisitor.source());
    }
    if (extractFieldNames != null) {
      for (String extractFieldName : extractFieldNames) {
        List<Object> values = context.lookup().source().extractRawValues(extractFieldName);
        if (!values.isEmpty()) {
          if (searchHit.fieldsOrNull() == null) {
            searchHit.fields(new HashMap<String, SearchHitField>(2));
          }

          SearchHitField hitField = searchHit.fields().get(extractFieldName);
          if (hitField == null) {
            hitField = new InternalSearchHitField(extractFieldName, new ArrayList<>(2));
            searchHit.fields().put(extractFieldName, hitField);
          }
          for (Object value : values) {
            hitField.values().add(value);
          }
        }
      }
    }

    return searchHit;
  }
Ejemplo n.º 3
0
 @Override
 public void collect(int doc) {
   try {
     FieldsVisitor fieldsVisitor = new FieldsVisitor(false);
     context.reader().document(doc, fieldsVisitor);
     Uid uid = fieldsVisitor.uid();
     final long version =
         Versions.loadVersion(context.reader(), new Term(UidFieldMapper.NAME, uid.toBytesRef()));
     docsToPurge.add(new DocToPurge(uid.type(), uid.id(), version, fieldsVisitor.routing()));
   } catch (Exception e) {
     logger.trace("failed to collect doc", e);
   }
 }
Ejemplo n.º 4
0
 private void loadStoredFields(
     SearchContext searchContext,
     LeafReaderContext readerContext,
     FieldsVisitor fieldVisitor,
     int docId) {
   fieldVisitor.reset();
   try {
     readerContext.reader().document(docId, fieldVisitor);
   } catch (IOException e) {
     throw new FetchPhaseExecutionException(
         searchContext, "Failed to fetch doc id [" + docId + "]", e);
   }
 }
Ejemplo n.º 5
0
  private InternalSearchHit createSearchHit(
      SearchContext context,
      FieldsVisitor fieldsVisitor,
      int docId,
      int subDocId,
      LeafReaderContext subReaderContext) {
    if (fieldsVisitor == null) {
      return new InternalSearchHit(docId);
    }
    loadStoredFields(context, subReaderContext, fieldsVisitor, subDocId);
    fieldsVisitor.postProcess(context.mapperService());

    Map<String, SearchHitField> searchFields = null;
    if (!fieldsVisitor.fields().isEmpty()) {
      searchFields = new HashMap<>(fieldsVisitor.fields().size());
      for (Map.Entry<String, List<Object>> entry : fieldsVisitor.fields().entrySet()) {
        searchFields.put(
            entry.getKey(), new InternalSearchHitField(entry.getKey(), entry.getValue()));
      }
    }

    DocumentMapper documentMapper =
        context.mapperService().documentMapper(fieldsVisitor.uid().type());
    Text typeText;
    if (documentMapper == null) {
      typeText = new Text(fieldsVisitor.uid().type());
    } else {
      typeText = documentMapper.typeText();
    }
    InternalSearchHit searchHit =
        new InternalSearchHit(docId, fieldsVisitor.uid().id(), typeText, searchFields);
    // Set _source if requested.
    SourceLookup sourceLookup = context.lookup().source();
    sourceLookup.setSegmentAndDocument(subReaderContext, subDocId);
    if (fieldsVisitor.source() != null) {
      sourceLookup.setSource(fieldsVisitor.source());
    }
    return searchHit;
  }
Ejemplo n.º 6
0
  private InternalSearchHit createNestedSearchHit(
      SearchContext context,
      int nestedTopDocId,
      int nestedSubDocId,
      int rootSubDocId,
      List<String> extractFieldNames,
      boolean loadAllStored,
      Set<String> fieldNames,
      LeafReaderContext subReaderContext)
      throws IOException {
    // Also if highlighting is requested on nested documents we need to fetch the _source from the
    // root document,
    // otherwise highlighting will attempt to fetch the _source from the nested doc, which will
    // fail,
    // because the entire _source is only stored with the root document.
    final FieldsVisitor rootFieldsVisitor =
        new FieldsVisitor(
            context.sourceRequested() || extractFieldNames != null || context.highlight() != null);
    loadStoredFields(context, subReaderContext, rootFieldsVisitor, rootSubDocId);
    rootFieldsVisitor.postProcess(context.mapperService());

    Map<String, SearchHitField> searchFields =
        getSearchFields(context, nestedSubDocId, loadAllStored, fieldNames, subReaderContext);
    DocumentMapper documentMapper =
        context.mapperService().documentMapper(rootFieldsVisitor.uid().type());
    SourceLookup sourceLookup = context.lookup().source();
    sourceLookup.setSegmentAndDocument(subReaderContext, nestedSubDocId);

    ObjectMapper nestedObjectMapper =
        documentMapper.findNestedObjectMapper(nestedSubDocId, context, subReaderContext);
    assert nestedObjectMapper != null;
    InternalSearchHit.InternalNestedIdentity nestedIdentity =
        getInternalNestedIdentity(
            context, nestedSubDocId, subReaderContext, documentMapper, nestedObjectMapper);

    BytesReference source = rootFieldsVisitor.source();
    if (source != null) {
      Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(source, true);
      Map<String, Object> sourceAsMap = tuple.v2();

      List<Map<String, Object>> nestedParsedSource;
      SearchHit.NestedIdentity nested = nestedIdentity;
      do {
        Object extractedValue =
            XContentMapValues.extractValue(nested.getField().string(), sourceAsMap);
        if (extractedValue == null) {
          // The nested objects may not exist in the _source, because it was filtered because of
          // _source filtering
          break;
        } else if (extractedValue instanceof List) {
          // nested field has an array value in the _source
          nestedParsedSource = (List<Map<String, Object>>) extractedValue;
        } else if (extractedValue instanceof Map) {
          // nested field has an object value in the _source. This just means the nested field has
          // just one inner object, which is valid, but uncommon.
          nestedParsedSource = ImmutableList.of((Map<String, Object>) extractedValue);
        } else {
          throw new IllegalStateException("extracted source isn't an object or an array");
        }
        sourceAsMap = nestedParsedSource.get(nested.getOffset());
        nested = nested.getChild();
      } while (nested != null);

      context.lookup().source().setSource(sourceAsMap);
      XContentType contentType = tuple.v1();
      BytesReference nestedSource = contentBuilder(contentType).map(sourceAsMap).bytes();
      context.lookup().source().setSource(nestedSource);
      context.lookup().source().setSourceContentType(contentType);
    }

    InternalSearchHit searchHit =
        new InternalSearchHit(
            nestedTopDocId,
            rootFieldsVisitor.uid().id(),
            documentMapper.typeText(),
            nestedIdentity,
            searchFields);
    if (extractFieldNames != null) {
      for (String extractFieldName : extractFieldNames) {
        List<Object> values = context.lookup().source().extractRawValues(extractFieldName);
        if (!values.isEmpty()) {
          if (searchHit.fieldsOrNull() == null) {
            searchHit.fields(new HashMap<String, SearchHitField>(2));
          }

          SearchHitField hitField = searchHit.fields().get(extractFieldName);
          if (hitField == null) {
            hitField = new InternalSearchHitField(extractFieldName, new ArrayList<>(2));
            searchHit.fields().put(extractFieldName, hitField);
          }
          for (Object value : values) {
            hitField.values().add(value);
          }
        }
      }
    }

    return searchHit;
  }
Ejemplo n.º 7
0
  private InternalSearchHit createNestedSearchHit(
      SearchContext context,
      int nestedTopDocId,
      int nestedSubDocId,
      int rootSubDocId,
      Set<String> fieldNames,
      List<String> fieldNamePatterns,
      LeafReaderContext subReaderContext)
      throws IOException {
    // Also if highlighting is requested on nested documents we need to fetch the _source from the
    // root document,
    // otherwise highlighting will attempt to fetch the _source from the nested doc, which will
    // fail,
    // because the entire _source is only stored with the root document.
    final FieldsVisitor rootFieldsVisitor =
        new FieldsVisitor(context.sourceRequested() || context.highlight() != null);
    loadStoredFields(context, subReaderContext, rootFieldsVisitor, rootSubDocId);
    rootFieldsVisitor.postProcess(context.mapperService());

    Map<String, SearchHitField> searchFields =
        getSearchFields(context, nestedSubDocId, fieldNames, fieldNamePatterns, subReaderContext);
    DocumentMapper documentMapper =
        context.mapperService().documentMapper(rootFieldsVisitor.uid().type());
    SourceLookup sourceLookup = context.lookup().source();
    sourceLookup.setSegmentAndDocument(subReaderContext, nestedSubDocId);

    ObjectMapper nestedObjectMapper =
        documentMapper.findNestedObjectMapper(nestedSubDocId, context, subReaderContext);
    assert nestedObjectMapper != null;
    InternalSearchHit.InternalNestedIdentity nestedIdentity =
        getInternalNestedIdentity(
            context, nestedSubDocId, subReaderContext, documentMapper, nestedObjectMapper);

    BytesReference source = rootFieldsVisitor.source();
    if (source != null) {
      Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(source, true);
      Map<String, Object> sourceAsMap = tuple.v2();

      // Isolate the nested json array object that matches with nested hit and wrap it back into the
      // same json
      // structure with the nested json array object being the actual content. The latter is
      // important, so that
      // features like source filtering and highlighting work consistent regardless of whether the
      // field points
      // to a json object array for consistency reasons on how we refer to fields
      Map<String, Object> nestedSourceAsMap = new HashMap<>();
      Map<String, Object> current = nestedSourceAsMap;
      for (SearchHit.NestedIdentity nested = nestedIdentity;
          nested != null;
          nested = nested.getChild()) {
        String nestedPath = nested.getField().string();
        current.put(nestedPath, new HashMap<>());
        Object extractedValue = XContentMapValues.extractValue(nestedPath, sourceAsMap);
        List<Map<String, Object>> nestedParsedSource;
        if (extractedValue instanceof List) {
          // nested field has an array value in the _source
          nestedParsedSource = (List<Map<String, Object>>) extractedValue;
        } else if (extractedValue instanceof Map) {
          // nested field has an object value in the _source. This just means the nested field has
          // just one inner object, which is valid, but uncommon.
          nestedParsedSource = Collections.singletonList((Map<String, Object>) extractedValue);
        } else {
          throw new IllegalStateException("extracted source isn't an object or an array");
        }
        sourceAsMap = nestedParsedSource.get(nested.getOffset());
        if (nested.getChild() == null) {
          current.put(nestedPath, sourceAsMap);
        } else {
          Map<String, Object> next = new HashMap<>();
          current.put(nestedPath, next);
          current = next;
        }
      }
      context.lookup().source().setSource(nestedSourceAsMap);
      XContentType contentType = tuple.v1();
      BytesReference nestedSource = contentBuilder(contentType).map(sourceAsMap).bytes();
      context.lookup().source().setSource(nestedSource);
      context.lookup().source().setSourceContentType(contentType);
    }

    return new InternalSearchHit(
        nestedTopDocId,
        rootFieldsVisitor.uid().id(),
        documentMapper.typeText(),
        nestedIdentity,
        searchFields);
  }