private Map<String, FieldMappingMetaData> findFieldMappingsByType(
DocumentMapper documentMapper, GetFieldMappingsIndexRequest request) {
MapBuilder<String, FieldMappingMetaData> fieldMappings = new MapBuilder<>();
final DocumentFieldMappers allFieldMappers = documentMapper.mappers();
for (String field : request.fields()) {
if (Regex.isMatchAllPattern(field)) {
for (FieldMapper fieldMapper : allFieldMappers) {
addFieldMapper(
fieldMapper.fieldType().name(),
fieldMapper,
fieldMappings,
request.includeDefaults());
}
} else if (Regex.isSimpleMatchPattern(field)) {
// go through the field mappers 3 times, to make sure we give preference to the resolve
// order: full name, index name, name.
// also make sure we only store each mapper once.
Collection<FieldMapper> remainingFieldMappers = newLinkedList(allFieldMappers);
for (Iterator<FieldMapper> it = remainingFieldMappers.iterator(); it.hasNext(); ) {
final FieldMapper fieldMapper = it.next();
if (Regex.simpleMatch(field, fieldMapper.fieldType().name())) {
addFieldMapper(
fieldMapper.fieldType().name(),
fieldMapper,
fieldMappings,
request.includeDefaults());
it.remove();
}
}
for (Iterator<FieldMapper> it = remainingFieldMappers.iterator(); it.hasNext(); ) {
final FieldMapper fieldMapper = it.next();
if (Regex.simpleMatch(field, fieldMapper.fieldType().name())) {
addFieldMapper(
fieldMapper.fieldType().name(),
fieldMapper,
fieldMappings,
request.includeDefaults());
it.remove();
}
}
} else {
// not a pattern
FieldMapper fieldMapper = allFieldMappers.smartNameFieldMapper(field);
if (fieldMapper != null) {
addFieldMapper(field, fieldMapper, fieldMappings, request.includeDefaults());
} else if (request.probablySingleFieldRequest()) {
fieldMappings.put(field, FieldMappingMetaData.NULL);
}
}
}
return fieldMappings.immutableMap();
}
private Fields generateTermVectorsFromDoc(TermVectorRequest request, boolean doAllFields)
throws IOException {
// parse the document, at the moment we do update the mapping, just like percolate
ParsedDocument parsedDocument =
parseDocument(indexShard.shardId().getIndex(), request.type(), request.doc());
// select the right fields and generate term vectors
ParseContext.Document doc = parsedDocument.rootDoc();
Collection<String> seenFields = new HashSet<>();
Collection<GetField> getFields = new HashSet<>();
for (IndexableField field : doc.getFields()) {
FieldMapper fieldMapper = indexShard.mapperService().smartNameFieldMapper(field.name());
if (seenFields.contains(field.name())) {
continue;
} else {
seenFields.add(field.name());
}
if (!isValidField(fieldMapper)) {
continue;
}
if (request.selectedFields() == null
&& !doAllFields
&& !fieldMapper.fieldType().storeTermVectors()) {
continue;
}
if (request.selectedFields() != null && !request.selectedFields().contains(field.name())) {
continue;
}
String[] values = doc.getValues(field.name());
getFields.add(new GetField(field.name(), Arrays.asList((Object[]) values)));
}
return generateTermVectors(getFields, request.offsets(), request.perFieldAnalyzer());
}
private boolean assertMappersShareSameFieldType() {
for (DocumentMapper mapper : docMappers(false)) {
List<FieldMapper> fieldMappers = new ArrayList<>();
Collections.addAll(fieldMappers, mapper.mapping().metadataMappers);
MapperUtils.collect(mapper.root(), new ArrayList<ObjectMapper>(), fieldMappers);
for (FieldMapper fieldMapper : fieldMappers) {
assert fieldMapper.fieldType() == fieldTypes.get(fieldMapper.name()) : fieldMapper.name();
}
}
return true;
}
private boolean isValidField(FieldMapper field) {
// must be a string
if (!(field instanceof StringFieldMapper)) {
return false;
}
// and must be indexed
if (field.fieldType().indexOptions() == IndexOptions.NONE) {
return false;
}
return true;
}
public void testNumericDetectionEnabled() throws Exception {
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.field("numeric_detection", true)
.endObject()
.endObject()
.string();
IndexService index = createIndex("test");
client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get();
DocumentMapper defaultMapper = index.mapperService().documentMapper("type");
ParsedDocument doc =
defaultMapper.parse(
"test",
"type",
"1",
XContentFactory.jsonBuilder()
.startObject()
.field("s_long", "100")
.field("s_double", "100.0")
.endObject()
.bytes());
assertNotNull(doc.dynamicMappingsUpdate());
client()
.admin()
.indices()
.preparePutMapping("test")
.setType("type")
.setSource(doc.dynamicMappingsUpdate().toString())
.get();
defaultMapper = index.mapperService().documentMapper("type");
FieldMapper mapper = defaultMapper.mappers().smartNameFieldMapper("s_long");
assertThat(mapper.fieldType().typeName(), equalTo("long"));
mapper = defaultMapper.mappers().smartNameFieldMapper("s_double");
assertThat(mapper.fieldType().typeName(), equalTo("float"));
}
@Override
public TerminationHandle warmReader(
final IndexShard indexShard, final Engine.Searcher searcher) {
final MapperService mapperService = indexShard.mapperService();
final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
for (DocumentMapper docMapper : mapperService.docMappers(false)) {
for (FieldMapper fieldMapper : docMapper.mappers()) {
final MappedFieldType fieldType = fieldMapper.fieldType();
final String indexName = fieldType.name();
if (fieldType.eagerGlobalOrdinals() == false) {
continue;
}
warmUpGlobalOrdinals.put(indexName, fieldType);
}
}
final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size());
for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) {
executor.execute(
() -> {
try {
final long start = System.nanoTime();
IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType);
DirectoryReader reader = searcher.getDirectoryReader();
IndexFieldData<?> global = ifd.loadGlobal(reader);
if (reader.leaves().isEmpty() == false) {
global.load(reader.leaves().get(0));
}
if (indexShard.warmerService().logger().isTraceEnabled()) {
indexShard
.warmerService()
.logger()
.trace(
"warmed global ordinals for [{}], took [{}]",
fieldType.name(),
TimeValue.timeValueNanos(System.nanoTime() - start));
}
} catch (Exception e) {
indexShard
.warmerService()
.logger()
.warn("failed to warm-up global ordinals for [{}]", e, fieldType.name());
} finally {
latch.countDown();
}
});
}
return () -> latch.await();
}
public void testSimple() throws Exception {
String mapping =
copyToStringFromClasspath(
"/org/elasticsearch/index/mapper/dynamictemplate/genericstore/test-mapping.json");
IndexService index = createIndex("test");
client().admin().indices().preparePutMapping("test").setType("person").setSource(mapping).get();
DocumentMapper docMapper = index.mapperService().documentMapper("person");
byte[] json =
copyToBytesFromClasspath(
"/org/elasticsearch/index/mapper/dynamictemplate/genericstore/test-data.json");
ParsedDocument parsedDoc = docMapper.parse("test", "person", "1", new BytesArray(json));
client()
.admin()
.indices()
.preparePutMapping("test")
.setType("person")
.setSource(parsedDoc.dynamicMappingsUpdate().toString())
.get();
docMapper = index.mapperService().documentMapper("person");
Document doc = parsedDoc.rootDoc();
IndexableField f = doc.getField("name");
assertThat(f.name(), equalTo("name"));
assertThat(f.stringValue(), equalTo("some name"));
assertThat(f.fieldType().stored(), equalTo(true));
FieldMapper fieldMapper = docMapper.mappers().getMapper("name");
assertThat(fieldMapper.fieldType().stored(), equalTo(true));
f = doc.getField("age");
assertThat(f.name(), equalTo("age"));
assertThat(f.fieldType().stored(), equalTo(true));
fieldMapper = docMapper.mappers().getMapper("age");
assertThat(fieldMapper.fieldType().stored(), equalTo(true));
}
private Fields addGeneratedTermVectors(
Engine.GetResult get,
Fields termVectorsByField,
TermVectorRequest request,
Set<String> selectedFields)
throws IOException {
/* only keep valid fields */
Set<String> validFields = new HashSet<>();
for (String field : selectedFields) {
FieldMapper fieldMapper = indexShard.mapperService().smartNameFieldMapper(field);
if (!isValidField(fieldMapper)) {
continue;
}
// already retrieved, only if the analyzer hasn't been overridden at the field
if (fieldMapper.fieldType().storeTermVectors()
&& (request.perFieldAnalyzer() == null
|| !request.perFieldAnalyzer().containsKey(field))) {
continue;
}
validFields.add(field);
}
if (validFields.isEmpty()) {
return termVectorsByField;
}
/* generate term vectors from fetched document fields */
GetResult getResult =
indexShard
.getService()
.get(
get,
request.id(),
request.type(),
validFields.toArray(Strings.EMPTY_ARRAY),
null,
false);
Fields generatedTermVectors =
generateTermVectors(
getResult.getFields().values(), request.offsets(), request.perFieldAnalyzer());
/* merge with existing Fields */
if (termVectorsByField == null) {
return generatedTermVectors;
} else {
return mergeFields(termVectorsByField, generatedTermVectors);
}
}
private void addFieldMapper(
String field,
FieldMapper fieldMapper,
MapBuilder<String, FieldMappingMetaData> fieldMappings,
boolean includeDefaults) {
if (fieldMappings.containsKey(field)) {
return;
}
try {
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
builder.startObject();
fieldMapper.toXContent(
builder, includeDefaults ? includeDefaultsParams : ToXContent.EMPTY_PARAMS);
builder.endObject();
fieldMappings.put(
field, new FieldMappingMetaData(fieldMapper.fieldType().name(), builder.bytes()));
} catch (IOException e) {
throw new ElasticsearchException("failed to serialize XContent of field [" + field + "]", e);
}
}
@Override
public TerminationHandle warmNewReaders(
final IndexShard indexShard,
IndexMetaData indexMetaData,
final WarmerContext context,
ThreadPool threadPool) {
final Loading defaultLoading =
Loading.parse(indexMetaData.settings().get(NORMS_LOADING_KEY), Loading.LAZY);
final MapperService mapperService = indexShard.mapperService();
final ObjectSet<String> warmUp = new ObjectHashSet<>();
for (DocumentMapper docMapper : mapperService.docMappers(false)) {
for (FieldMapper fieldMapper : docMapper.mappers()) {
final String indexName = fieldMapper.fieldType().names().indexName();
Loading normsLoading = fieldMapper.fieldType().normsLoading();
if (normsLoading == null) {
normsLoading = defaultLoading;
}
if (fieldMapper.fieldType().indexOptions() != IndexOptions.NONE
&& !fieldMapper.fieldType().omitNorms()
&& normsLoading == Loading.EAGER) {
warmUp.add(indexName);
}
}
}
final CountDownLatch latch = new CountDownLatch(1);
// Norms loading may be I/O intensive but is not CPU intensive, so we execute it in a single
// task
threadPool
.executor(executor())
.execute(
new Runnable() {
@Override
public void run() {
try {
for (ObjectCursor<String> stringObjectCursor : warmUp) {
final String indexName = stringObjectCursor.value;
final long start = System.nanoTime();
for (final LeafReaderContext ctx : context.searcher().reader().leaves()) {
final NumericDocValues values = ctx.reader().getNormValues(indexName);
if (values != null) {
values.get(0);
}
}
if (indexShard.warmerService().logger().isTraceEnabled()) {
indexShard
.warmerService()
.logger()
.trace(
"warmed norms for [{}], took [{}]",
indexName,
TimeValue.timeValueNanos(System.nanoTime() - start));
}
}
} catch (Throwable t) {
indexShard.warmerService().logger().warn("failed to warm-up norms", t);
} finally {
latch.countDown();
}
}
});
return new TerminationHandle() {
@Override
public void awaitTermination() throws InterruptedException {
latch.await();
}
};
}
@Override
public TerminationHandle warmTopReader(
final IndexShard indexShard,
IndexMetaData indexMetaData,
final WarmerContext context,
ThreadPool threadPool) {
final MapperService mapperService = indexShard.mapperService();
final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
for (DocumentMapper docMapper : mapperService.docMappers(false)) {
for (FieldMapper fieldMapper : docMapper.mappers()) {
final FieldDataType fieldDataType = fieldMapper.fieldType().fieldDataType();
if (fieldDataType == null) {
continue;
}
if (fieldDataType.getLoading() != Loading.EAGER_GLOBAL_ORDINALS) {
continue;
}
final String indexName = fieldMapper.fieldType().names().indexName();
if (warmUpGlobalOrdinals.containsKey(indexName)) {
continue;
}
warmUpGlobalOrdinals.put(indexName, fieldMapper.fieldType());
}
}
final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
final Executor executor = threadPool.executor(executor());
final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size());
for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) {
executor.execute(
new Runnable() {
@Override
public void run() {
try {
final long start = System.nanoTime();
IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType);
ifd.loadGlobal(context.reader());
if (indexShard.warmerService().logger().isTraceEnabled()) {
indexShard
.warmerService()
.logger()
.trace(
"warmed global ordinals for [{}], took [{}]",
fieldType.names().fullName(),
TimeValue.timeValueNanos(System.nanoTime() - start));
}
} catch (Throwable t) {
indexShard
.warmerService()
.logger()
.warn(
"failed to warm-up global ordinals for [{}]",
t,
fieldType.names().fullName());
} finally {
latch.countDown();
}
}
});
}
return new TerminationHandle() {
@Override
public void awaitTermination() throws InterruptedException {
latch.await();
}
};
}
@Override
public HighlightField highlight(HighlighterContext highlighterContext) {
SearchContextHighlight.Field field = highlighterContext.field;
SearchContext context = highlighterContext.context;
FetchSubPhase.HitContext hitContext = highlighterContext.hitContext;
FieldMapper mapper = highlighterContext.mapper;
if (canHighlight(mapper) == false) {
throw new IllegalArgumentException(
"the field ["
+ highlighterContext.fieldName
+ "] should be indexed with term vector with position offsets to be used with fast vector highlighter");
}
Encoder encoder =
field.fieldOptions().encoder().equals("html")
? HighlightUtils.Encoders.HTML
: HighlightUtils.Encoders.DEFAULT;
if (!hitContext.cache().containsKey(CACHE_KEY)) {
hitContext.cache().put(CACHE_KEY, new HighlighterEntry());
}
HighlighterEntry cache = (HighlighterEntry) hitContext.cache().get(CACHE_KEY);
try {
FieldQuery fieldQuery;
if (field.fieldOptions().requireFieldMatch()) {
if (cache.fieldMatchFieldQuery == null) {
/*
* we use top level reader to rewrite the query against all readers, with use caching it across hits (and across
* readers...)
*/
cache.fieldMatchFieldQuery =
new CustomFieldQuery(
highlighterContext.query,
hitContext.topLevelReader(),
true,
field.fieldOptions().requireFieldMatch());
}
fieldQuery = cache.fieldMatchFieldQuery;
} else {
if (cache.noFieldMatchFieldQuery == null) {
/*
* we use top level reader to rewrite the query against all readers, with use caching it across hits (and across
* readers...)
*/
cache.noFieldMatchFieldQuery =
new CustomFieldQuery(
highlighterContext.query,
hitContext.topLevelReader(),
true,
field.fieldOptions().requireFieldMatch());
}
fieldQuery = cache.noFieldMatchFieldQuery;
}
MapperHighlightEntry entry = cache.mappers.get(mapper);
if (entry == null) {
FragListBuilder fragListBuilder;
BaseFragmentsBuilder fragmentsBuilder;
BoundaryScanner boundaryScanner = DEFAULT_BOUNDARY_SCANNER;
if (field.fieldOptions().boundaryMaxScan() != SimpleBoundaryScanner.DEFAULT_MAX_SCAN
|| field.fieldOptions().boundaryChars()
!= SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS) {
boundaryScanner =
new SimpleBoundaryScanner(
field.fieldOptions().boundaryMaxScan(), field.fieldOptions().boundaryChars());
}
boolean forceSource = context.highlight().forceSource(field);
if (field.fieldOptions().numberOfFragments() == 0) {
fragListBuilder = new SingleFragListBuilder();
if (!forceSource && mapper.fieldType().stored()) {
fragmentsBuilder =
new SimpleFragmentsBuilder(
mapper,
field.fieldOptions().preTags(),
field.fieldOptions().postTags(),
boundaryScanner);
} else {
fragmentsBuilder =
new SourceSimpleFragmentsBuilder(
mapper,
context,
field.fieldOptions().preTags(),
field.fieldOptions().postTags(),
boundaryScanner);
}
} else {
fragListBuilder =
field.fieldOptions().fragmentOffset() == -1
? new SimpleFragListBuilder()
: new SimpleFragListBuilder(field.fieldOptions().fragmentOffset());
if (field.fieldOptions().scoreOrdered()) {
if (!forceSource && mapper.fieldType().stored()) {
fragmentsBuilder =
new ScoreOrderFragmentsBuilder(
field.fieldOptions().preTags(),
field.fieldOptions().postTags(),
boundaryScanner);
} else {
fragmentsBuilder =
new SourceScoreOrderFragmentsBuilder(
mapper,
context,
field.fieldOptions().preTags(),
field.fieldOptions().postTags(),
boundaryScanner);
}
} else {
if (!forceSource && mapper.fieldType().stored()) {
fragmentsBuilder =
new SimpleFragmentsBuilder(
mapper,
field.fieldOptions().preTags(),
field.fieldOptions().postTags(),
boundaryScanner);
} else {
fragmentsBuilder =
new SourceSimpleFragmentsBuilder(
mapper,
context,
field.fieldOptions().preTags(),
field.fieldOptions().postTags(),
boundaryScanner);
}
}
}
fragmentsBuilder.setDiscreteMultiValueHighlighting(termVectorMultiValue);
entry = new MapperHighlightEntry();
entry.fragListBuilder = fragListBuilder;
entry.fragmentsBuilder = fragmentsBuilder;
if (cache.fvh == null) {
// parameters to FVH are not requires since:
// first two booleans are not relevant since they are set on the CustomFieldQuery (phrase
// and fieldMatch)
// fragment builders are used explicitly
cache.fvh = new org.apache.lucene.search.vectorhighlight.FastVectorHighlighter();
}
CustomFieldQuery.highlightFilters.set(field.fieldOptions().highlightFilter());
cache.mappers.put(mapper, entry);
}
cache.fvh.setPhraseLimit(field.fieldOptions().phraseLimit());
String[] fragments;
// a HACK to make highlighter do highlighting, even though its using the single frag list
// builder
int numberOfFragments =
field.fieldOptions().numberOfFragments() == 0
? Integer.MAX_VALUE
: field.fieldOptions().numberOfFragments();
int fragmentCharSize =
field.fieldOptions().numberOfFragments() == 0
? Integer.MAX_VALUE
: field.fieldOptions().fragmentCharSize();
// we highlight against the low level reader and docId, because if we load source, we want to
// reuse it if possible
// Only send matched fields if they were requested to save time.
if (field.fieldOptions().matchedFields() != null
&& !field.fieldOptions().matchedFields().isEmpty()) {
fragments =
cache.fvh.getBestFragments(
fieldQuery,
hitContext.reader(),
hitContext.docId(),
mapper.fieldType().name(),
field.fieldOptions().matchedFields(),
fragmentCharSize,
numberOfFragments,
entry.fragListBuilder,
entry.fragmentsBuilder,
field.fieldOptions().preTags(),
field.fieldOptions().postTags(),
encoder);
} else {
fragments =
cache.fvh.getBestFragments(
fieldQuery,
hitContext.reader(),
hitContext.docId(),
mapper.fieldType().name(),
fragmentCharSize,
numberOfFragments,
entry.fragListBuilder,
entry.fragmentsBuilder,
field.fieldOptions().preTags(),
field.fieldOptions().postTags(),
encoder);
}
if (fragments != null && fragments.length > 0) {
return new HighlightField(
highlighterContext.fieldName, Text.convertFromStringArray(fragments));
}
int noMatchSize = highlighterContext.field.fieldOptions().noMatchSize();
if (noMatchSize > 0) {
// Essentially we just request that a fragment is built from 0 to noMatchSize using the
// normal fragmentsBuilder
FieldFragList fieldFragList = new SimpleFieldFragList(-1 /*ignored*/);
fieldFragList.add(0, noMatchSize, Collections.<WeightedPhraseInfo>emptyList());
fragments =
entry.fragmentsBuilder.createFragments(
hitContext.reader(),
hitContext.docId(),
mapper.fieldType().name(),
fieldFragList,
1,
field.fieldOptions().preTags(),
field.fieldOptions().postTags(),
encoder);
if (fragments != null && fragments.length > 0) {
return new HighlightField(
highlighterContext.fieldName, Text.convertFromStringArray(fragments));
}
}
return null;
} catch (Exception e) {
throw new FetchPhaseExecutionException(
context, "Failed to highlight field [" + highlighterContext.fieldName + "]", e);
}
}
@Override
public boolean canHighlight(FieldMapper fieldMapper) {
return fieldMapper.fieldType().storeTermVectors()
&& fieldMapper.fieldType().storeTermVectorOffsets()
&& fieldMapper.fieldType().storeTermVectorPositions();
}