private Map<String, SearchHitField> getSearchFields(
SearchContext context,
int nestedSubDocId,
boolean loadAllStored,
Set<String> fieldNames,
LeafReaderContext subReaderContext) {
Map<String, SearchHitField> searchFields = null;
if (context.hasFieldNames() && !context.fieldNames().isEmpty()) {
FieldsVisitor nestedFieldsVisitor = null;
if (loadAllStored) {
nestedFieldsVisitor = new AllFieldsVisitor();
} else if (fieldNames != null) {
nestedFieldsVisitor = new CustomFieldsVisitor(fieldNames, false);
}
if (nestedFieldsVisitor != null) {
loadStoredFields(context, subReaderContext, nestedFieldsVisitor, nestedSubDocId);
nestedFieldsVisitor.postProcess(context.mapperService());
if (!nestedFieldsVisitor.fields().isEmpty()) {
searchFields = new HashMap<>(nestedFieldsVisitor.fields().size());
for (Map.Entry<String, List<Object>> entry : nestedFieldsVisitor.fields().entrySet()) {
searchFields.put(
entry.getKey(), new InternalSearchHitField(entry.getKey(), entry.getValue()));
}
}
}
}
return searchFields;
}
private InternalSearchHit createSearchHit(
SearchContext context,
FieldsVisitor fieldsVisitor,
int docId,
int subDocId,
List<String> extractFieldNames,
LeafReaderContext subReaderContext) {
loadStoredFields(context, subReaderContext, fieldsVisitor, subDocId);
fieldsVisitor.postProcess(context.mapperService());
Map<String, SearchHitField> searchFields = null;
if (!fieldsVisitor.fields().isEmpty()) {
searchFields = new HashMap<>(fieldsVisitor.fields().size());
for (Map.Entry<String, List<Object>> entry : fieldsVisitor.fields().entrySet()) {
searchFields.put(
entry.getKey(), new InternalSearchHitField(entry.getKey(), entry.getValue()));
}
}
DocumentMapper documentMapper =
context.mapperService().documentMapper(fieldsVisitor.uid().type());
Text typeText;
if (documentMapper == null) {
typeText = new StringAndBytesText(fieldsVisitor.uid().type());
} else {
typeText = documentMapper.typeText();
}
InternalSearchHit searchHit =
new InternalSearchHit(docId, fieldsVisitor.uid().id(), typeText, searchFields);
// go over and extract fields that are not mapped / stored
SourceLookup sourceLookup = context.lookup().source();
sourceLookup.setSegmentAndDocument(subReaderContext, subDocId);
if (fieldsVisitor.source() != null) {
sourceLookup.setSource(fieldsVisitor.source());
}
if (extractFieldNames != null) {
for (String extractFieldName : extractFieldNames) {
List<Object> values = context.lookup().source().extractRawValues(extractFieldName);
if (!values.isEmpty()) {
if (searchHit.fieldsOrNull() == null) {
searchHit.fields(new HashMap<String, SearchHitField>(2));
}
SearchHitField hitField = searchHit.fields().get(extractFieldName);
if (hitField == null) {
hitField = new InternalSearchHitField(extractFieldName, new ArrayList<>(2));
searchHit.fields().put(extractFieldName, hitField);
}
for (Object value : values) {
hitField.values().add(value);
}
}
}
}
return searchHit;
}
@Override
public void collect(int doc) {
try {
FieldsVisitor fieldsVisitor = new FieldsVisitor(false);
context.reader().document(doc, fieldsVisitor);
Uid uid = fieldsVisitor.uid();
final long version =
Versions.loadVersion(context.reader(), new Term(UidFieldMapper.NAME, uid.toBytesRef()));
docsToPurge.add(new DocToPurge(uid.type(), uid.id(), version, fieldsVisitor.routing()));
} catch (Exception e) {
logger.trace("failed to collect doc", e);
}
}
private void loadStoredFields(
SearchContext searchContext,
LeafReaderContext readerContext,
FieldsVisitor fieldVisitor,
int docId) {
fieldVisitor.reset();
try {
readerContext.reader().document(docId, fieldVisitor);
} catch (IOException e) {
throw new FetchPhaseExecutionException(
searchContext, "Failed to fetch doc id [" + docId + "]", e);
}
}
private InternalSearchHit createSearchHit(
SearchContext context,
FieldsVisitor fieldsVisitor,
int docId,
int subDocId,
LeafReaderContext subReaderContext) {
if (fieldsVisitor == null) {
return new InternalSearchHit(docId);
}
loadStoredFields(context, subReaderContext, fieldsVisitor, subDocId);
fieldsVisitor.postProcess(context.mapperService());
Map<String, SearchHitField> searchFields = null;
if (!fieldsVisitor.fields().isEmpty()) {
searchFields = new HashMap<>(fieldsVisitor.fields().size());
for (Map.Entry<String, List<Object>> entry : fieldsVisitor.fields().entrySet()) {
searchFields.put(
entry.getKey(), new InternalSearchHitField(entry.getKey(), entry.getValue()));
}
}
DocumentMapper documentMapper =
context.mapperService().documentMapper(fieldsVisitor.uid().type());
Text typeText;
if (documentMapper == null) {
typeText = new Text(fieldsVisitor.uid().type());
} else {
typeText = documentMapper.typeText();
}
InternalSearchHit searchHit =
new InternalSearchHit(docId, fieldsVisitor.uid().id(), typeText, searchFields);
// Set _source if requested.
SourceLookup sourceLookup = context.lookup().source();
sourceLookup.setSegmentAndDocument(subReaderContext, subDocId);
if (fieldsVisitor.source() != null) {
sourceLookup.setSource(fieldsVisitor.source());
}
return searchHit;
}
private InternalSearchHit createNestedSearchHit(
SearchContext context,
int nestedTopDocId,
int nestedSubDocId,
int rootSubDocId,
List<String> extractFieldNames,
boolean loadAllStored,
Set<String> fieldNames,
LeafReaderContext subReaderContext)
throws IOException {
// Also if highlighting is requested on nested documents we need to fetch the _source from the
// root document,
// otherwise highlighting will attempt to fetch the _source from the nested doc, which will
// fail,
// because the entire _source is only stored with the root document.
final FieldsVisitor rootFieldsVisitor =
new FieldsVisitor(
context.sourceRequested() || extractFieldNames != null || context.highlight() != null);
loadStoredFields(context, subReaderContext, rootFieldsVisitor, rootSubDocId);
rootFieldsVisitor.postProcess(context.mapperService());
Map<String, SearchHitField> searchFields =
getSearchFields(context, nestedSubDocId, loadAllStored, fieldNames, subReaderContext);
DocumentMapper documentMapper =
context.mapperService().documentMapper(rootFieldsVisitor.uid().type());
SourceLookup sourceLookup = context.lookup().source();
sourceLookup.setSegmentAndDocument(subReaderContext, nestedSubDocId);
ObjectMapper nestedObjectMapper =
documentMapper.findNestedObjectMapper(nestedSubDocId, context, subReaderContext);
assert nestedObjectMapper != null;
InternalSearchHit.InternalNestedIdentity nestedIdentity =
getInternalNestedIdentity(
context, nestedSubDocId, subReaderContext, documentMapper, nestedObjectMapper);
BytesReference source = rootFieldsVisitor.source();
if (source != null) {
Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(source, true);
Map<String, Object> sourceAsMap = tuple.v2();
List<Map<String, Object>> nestedParsedSource;
SearchHit.NestedIdentity nested = nestedIdentity;
do {
Object extractedValue =
XContentMapValues.extractValue(nested.getField().string(), sourceAsMap);
if (extractedValue == null) {
// The nested objects may not exist in the _source, because it was filtered because of
// _source filtering
break;
} else if (extractedValue instanceof List) {
// nested field has an array value in the _source
nestedParsedSource = (List<Map<String, Object>>) extractedValue;
} else if (extractedValue instanceof Map) {
// nested field has an object value in the _source. This just means the nested field has
// just one inner object, which is valid, but uncommon.
nestedParsedSource = ImmutableList.of((Map<String, Object>) extractedValue);
} else {
throw new IllegalStateException("extracted source isn't an object or an array");
}
sourceAsMap = nestedParsedSource.get(nested.getOffset());
nested = nested.getChild();
} while (nested != null);
context.lookup().source().setSource(sourceAsMap);
XContentType contentType = tuple.v1();
BytesReference nestedSource = contentBuilder(contentType).map(sourceAsMap).bytes();
context.lookup().source().setSource(nestedSource);
context.lookup().source().setSourceContentType(contentType);
}
InternalSearchHit searchHit =
new InternalSearchHit(
nestedTopDocId,
rootFieldsVisitor.uid().id(),
documentMapper.typeText(),
nestedIdentity,
searchFields);
if (extractFieldNames != null) {
for (String extractFieldName : extractFieldNames) {
List<Object> values = context.lookup().source().extractRawValues(extractFieldName);
if (!values.isEmpty()) {
if (searchHit.fieldsOrNull() == null) {
searchHit.fields(new HashMap<String, SearchHitField>(2));
}
SearchHitField hitField = searchHit.fields().get(extractFieldName);
if (hitField == null) {
hitField = new InternalSearchHitField(extractFieldName, new ArrayList<>(2));
searchHit.fields().put(extractFieldName, hitField);
}
for (Object value : values) {
hitField.values().add(value);
}
}
}
}
return searchHit;
}
private InternalSearchHit createNestedSearchHit(
SearchContext context,
int nestedTopDocId,
int nestedSubDocId,
int rootSubDocId,
Set<String> fieldNames,
List<String> fieldNamePatterns,
LeafReaderContext subReaderContext)
throws IOException {
// Also if highlighting is requested on nested documents we need to fetch the _source from the
// root document,
// otherwise highlighting will attempt to fetch the _source from the nested doc, which will
// fail,
// because the entire _source is only stored with the root document.
final FieldsVisitor rootFieldsVisitor =
new FieldsVisitor(context.sourceRequested() || context.highlight() != null);
loadStoredFields(context, subReaderContext, rootFieldsVisitor, rootSubDocId);
rootFieldsVisitor.postProcess(context.mapperService());
Map<String, SearchHitField> searchFields =
getSearchFields(context, nestedSubDocId, fieldNames, fieldNamePatterns, subReaderContext);
DocumentMapper documentMapper =
context.mapperService().documentMapper(rootFieldsVisitor.uid().type());
SourceLookup sourceLookup = context.lookup().source();
sourceLookup.setSegmentAndDocument(subReaderContext, nestedSubDocId);
ObjectMapper nestedObjectMapper =
documentMapper.findNestedObjectMapper(nestedSubDocId, context, subReaderContext);
assert nestedObjectMapper != null;
InternalSearchHit.InternalNestedIdentity nestedIdentity =
getInternalNestedIdentity(
context, nestedSubDocId, subReaderContext, documentMapper, nestedObjectMapper);
BytesReference source = rootFieldsVisitor.source();
if (source != null) {
Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(source, true);
Map<String, Object> sourceAsMap = tuple.v2();
// Isolate the nested json array object that matches with nested hit and wrap it back into the
// same json
// structure with the nested json array object being the actual content. The latter is
// important, so that
// features like source filtering and highlighting work consistent regardless of whether the
// field points
// to a json object array for consistency reasons on how we refer to fields
Map<String, Object> nestedSourceAsMap = new HashMap<>();
Map<String, Object> current = nestedSourceAsMap;
for (SearchHit.NestedIdentity nested = nestedIdentity;
nested != null;
nested = nested.getChild()) {
String nestedPath = nested.getField().string();
current.put(nestedPath, new HashMap<>());
Object extractedValue = XContentMapValues.extractValue(nestedPath, sourceAsMap);
List<Map<String, Object>> nestedParsedSource;
if (extractedValue instanceof List) {
// nested field has an array value in the _source
nestedParsedSource = (List<Map<String, Object>>) extractedValue;
} else if (extractedValue instanceof Map) {
// nested field has an object value in the _source. This just means the nested field has
// just one inner object, which is valid, but uncommon.
nestedParsedSource = Collections.singletonList((Map<String, Object>) extractedValue);
} else {
throw new IllegalStateException("extracted source isn't an object or an array");
}
sourceAsMap = nestedParsedSource.get(nested.getOffset());
if (nested.getChild() == null) {
current.put(nestedPath, sourceAsMap);
} else {
Map<String, Object> next = new HashMap<>();
current.put(nestedPath, next);
current = next;
}
}
context.lookup().source().setSource(nestedSourceAsMap);
XContentType contentType = tuple.v1();
BytesReference nestedSource = contentBuilder(contentType).map(sourceAsMap).bytes();
context.lookup().source().setSource(nestedSource);
context.lookup().source().setSourceContentType(contentType);
}
return new InternalSearchHit(
nestedTopDocId,
rootFieldsVisitor.uid().id(),
documentMapper.typeText(),
nestedIdentity,
searchFields);
}
