private Map<String, FieldMappingMetaData> findFieldMappingsByType(
      DocumentMapper documentMapper, GetFieldMappingsIndexRequest request) {
    MapBuilder<String, FieldMappingMetaData> fieldMappings = new MapBuilder<>();
    final DocumentFieldMappers allFieldMappers = documentMapper.mappers();
    for (String field : request.fields()) {
      if (Regex.isMatchAllPattern(field)) {
        for (FieldMapper fieldMapper : allFieldMappers) {
          addFieldMapper(
              fieldMapper.fieldType().name(),
              fieldMapper,
              fieldMappings,
              request.includeDefaults());
        }
      } else if (Regex.isSimpleMatchPattern(field)) {
        // go through the field mappers 3 times, to make sure we give preference to the resolve
        // order: full name, index name, name.
        // also make sure we only store each mapper once.
        Collection<FieldMapper> remainingFieldMappers = newLinkedList(allFieldMappers);
        for (Iterator<FieldMapper> it = remainingFieldMappers.iterator(); it.hasNext(); ) {
          final FieldMapper fieldMapper = it.next();
          if (Regex.simpleMatch(field, fieldMapper.fieldType().name())) {
            addFieldMapper(
                fieldMapper.fieldType().name(),
                fieldMapper,
                fieldMappings,
                request.includeDefaults());
            it.remove();
          }
        }
        for (Iterator<FieldMapper> it = remainingFieldMappers.iterator(); it.hasNext(); ) {
          final FieldMapper fieldMapper = it.next();
          if (Regex.simpleMatch(field, fieldMapper.fieldType().name())) {
            addFieldMapper(
                fieldMapper.fieldType().name(),
                fieldMapper,
                fieldMappings,
                request.includeDefaults());
            it.remove();
          }
        }

      } else {
        // not a pattern
        FieldMapper fieldMapper = allFieldMappers.smartNameFieldMapper(field);
        if (fieldMapper != null) {
          addFieldMapper(field, fieldMapper, fieldMappings, request.includeDefaults());
        } else if (request.probablySingleFieldRequest()) {
          fieldMappings.put(field, FieldMappingMetaData.NULL);
        }
      }
    }
    return fieldMappings.immutableMap();
  }
  private Fields generateTermVectorsFromDoc(TermVectorRequest request, boolean doAllFields)
      throws IOException {
    // parse the document, at the moment we do update the mapping, just like percolate
    ParsedDocument parsedDocument =
        parseDocument(indexShard.shardId().getIndex(), request.type(), request.doc());

    // select the right fields and generate term vectors
    ParseContext.Document doc = parsedDocument.rootDoc();
    Collection<String> seenFields = new HashSet<>();
    Collection<GetField> getFields = new HashSet<>();
    for (IndexableField field : doc.getFields()) {
      FieldMapper fieldMapper = indexShard.mapperService().smartNameFieldMapper(field.name());
      if (seenFields.contains(field.name())) {
        continue;
      } else {
        seenFields.add(field.name());
      }
      if (!isValidField(fieldMapper)) {
        continue;
      }
      if (request.selectedFields() == null
          && !doAllFields
          && !fieldMapper.fieldType().storeTermVectors()) {
        continue;
      }
      if (request.selectedFields() != null && !request.selectedFields().contains(field.name())) {
        continue;
      }
      String[] values = doc.getValues(field.name());
      getFields.add(new GetField(field.name(), Arrays.asList((Object[]) values)));
    }
    return generateTermVectors(getFields, request.offsets(), request.perFieldAnalyzer());
  }
 private boolean assertMappersShareSameFieldType() {
   for (DocumentMapper mapper : docMappers(false)) {
     List<FieldMapper> fieldMappers = new ArrayList<>();
     Collections.addAll(fieldMappers, mapper.mapping().metadataMappers);
     MapperUtils.collect(mapper.root(), new ArrayList<ObjectMapper>(), fieldMappers);
     for (FieldMapper fieldMapper : fieldMappers) {
       assert fieldMapper.fieldType() == fieldTypes.get(fieldMapper.name()) : fieldMapper.name();
     }
   }
   return true;
 }
 private boolean isValidField(FieldMapper field) {
   // must be a string
   if (!(field instanceof StringFieldMapper)) {
     return false;
   }
   // and must be indexed
   if (field.fieldType().indexOptions() == IndexOptions.NONE) {
     return false;
   }
   return true;
 }
  public void testNumericDetectionEnabled() throws Exception {
    String mapping =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type")
            .field("numeric_detection", true)
            .endObject()
            .endObject()
            .string();

    IndexService index = createIndex("test");
    client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get();
    DocumentMapper defaultMapper = index.mapperService().documentMapper("type");

    ParsedDocument doc =
        defaultMapper.parse(
            "test",
            "type",
            "1",
            XContentFactory.jsonBuilder()
                .startObject()
                .field("s_long", "100")
                .field("s_double", "100.0")
                .endObject()
                .bytes());
    assertNotNull(doc.dynamicMappingsUpdate());
    client()
        .admin()
        .indices()
        .preparePutMapping("test")
        .setType("type")
        .setSource(doc.dynamicMappingsUpdate().toString())
        .get();

    defaultMapper = index.mapperService().documentMapper("type");
    FieldMapper mapper = defaultMapper.mappers().smartNameFieldMapper("s_long");
    assertThat(mapper.fieldType().typeName(), equalTo("long"));

    mapper = defaultMapper.mappers().smartNameFieldMapper("s_double");
    assertThat(mapper.fieldType().typeName(), equalTo("float"));
  }
    @Override
    public TerminationHandle warmReader(
        final IndexShard indexShard, final Engine.Searcher searcher) {
      final MapperService mapperService = indexShard.mapperService();
      final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
      for (DocumentMapper docMapper : mapperService.docMappers(false)) {
        for (FieldMapper fieldMapper : docMapper.mappers()) {
          final MappedFieldType fieldType = fieldMapper.fieldType();
          final String indexName = fieldType.name();
          if (fieldType.eagerGlobalOrdinals() == false) {
            continue;
          }
          warmUpGlobalOrdinals.put(indexName, fieldType);
        }
      }
      final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
      final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size());
      for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) {
        executor.execute(
            () -> {
              try {
                final long start = System.nanoTime();
                IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType);
                DirectoryReader reader = searcher.getDirectoryReader();
                IndexFieldData<?> global = ifd.loadGlobal(reader);
                if (reader.leaves().isEmpty() == false) {
                  global.load(reader.leaves().get(0));
                }

                if (indexShard.warmerService().logger().isTraceEnabled()) {
                  indexShard
                      .warmerService()
                      .logger()
                      .trace(
                          "warmed global ordinals for [{}], took [{}]",
                          fieldType.name(),
                          TimeValue.timeValueNanos(System.nanoTime() - start));
                }
              } catch (Exception e) {
                indexShard
                    .warmerService()
                    .logger()
                    .warn("failed to warm-up global ordinals for [{}]", e, fieldType.name());
              } finally {
                latch.countDown();
              }
            });
      }
      return () -> latch.await();
    }
  public void testSimple() throws Exception {
    String mapping =
        copyToStringFromClasspath(
            "/org/elasticsearch/index/mapper/dynamictemplate/genericstore/test-mapping.json");
    IndexService index = createIndex("test");
    client().admin().indices().preparePutMapping("test").setType("person").setSource(mapping).get();
    DocumentMapper docMapper = index.mapperService().documentMapper("person");
    byte[] json =
        copyToBytesFromClasspath(
            "/org/elasticsearch/index/mapper/dynamictemplate/genericstore/test-data.json");
    ParsedDocument parsedDoc = docMapper.parse("test", "person", "1", new BytesArray(json));
    client()
        .admin()
        .indices()
        .preparePutMapping("test")
        .setType("person")
        .setSource(parsedDoc.dynamicMappingsUpdate().toString())
        .get();
    docMapper = index.mapperService().documentMapper("person");
    Document doc = parsedDoc.rootDoc();

    IndexableField f = doc.getField("name");
    assertThat(f.name(), equalTo("name"));
    assertThat(f.stringValue(), equalTo("some name"));
    assertThat(f.fieldType().stored(), equalTo(true));

    FieldMapper fieldMapper = docMapper.mappers().getMapper("name");
    assertThat(fieldMapper.fieldType().stored(), equalTo(true));

    f = doc.getField("age");
    assertThat(f.name(), equalTo("age"));
    assertThat(f.fieldType().stored(), equalTo(true));

    fieldMapper = docMapper.mappers().getMapper("age");
    assertThat(fieldMapper.fieldType().stored(), equalTo(true));
  }
  private Fields addGeneratedTermVectors(
      Engine.GetResult get,
      Fields termVectorsByField,
      TermVectorRequest request,
      Set<String> selectedFields)
      throws IOException {
    /* only keep valid fields */
    Set<String> validFields = new HashSet<>();
    for (String field : selectedFields) {
      FieldMapper fieldMapper = indexShard.mapperService().smartNameFieldMapper(field);
      if (!isValidField(fieldMapper)) {
        continue;
      }
      // already retrieved, only if the analyzer hasn't been overridden at the field
      if (fieldMapper.fieldType().storeTermVectors()
          && (request.perFieldAnalyzer() == null
              || !request.perFieldAnalyzer().containsKey(field))) {
        continue;
      }
      validFields.add(field);
    }

    if (validFields.isEmpty()) {
      return termVectorsByField;
    }

    /* generate term vectors from fetched document fields */
    GetResult getResult =
        indexShard
            .getService()
            .get(
                get,
                request.id(),
                request.type(),
                validFields.toArray(Strings.EMPTY_ARRAY),
                null,
                false);
    Fields generatedTermVectors =
        generateTermVectors(
            getResult.getFields().values(), request.offsets(), request.perFieldAnalyzer());

    /* merge with existing Fields */
    if (termVectorsByField == null) {
      return generatedTermVectors;
    } else {
      return mergeFields(termVectorsByField, generatedTermVectors);
    }
  }
 private void addFieldMapper(
     String field,
     FieldMapper fieldMapper,
     MapBuilder<String, FieldMappingMetaData> fieldMappings,
     boolean includeDefaults) {
   if (fieldMappings.containsKey(field)) {
     return;
   }
   try {
     XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
     builder.startObject();
     fieldMapper.toXContent(
         builder, includeDefaults ? includeDefaultsParams : ToXContent.EMPTY_PARAMS);
     builder.endObject();
     fieldMappings.put(
         field, new FieldMappingMetaData(fieldMapper.fieldType().name(), builder.bytes()));
   } catch (IOException e) {
     throw new ElasticsearchException("failed to serialize XContent of field [" + field + "]", e);
   }
 }
    @Override
    public TerminationHandle warmNewReaders(
        final IndexShard indexShard,
        IndexMetaData indexMetaData,
        final WarmerContext context,
        ThreadPool threadPool) {
      final Loading defaultLoading =
          Loading.parse(indexMetaData.settings().get(NORMS_LOADING_KEY), Loading.LAZY);
      final MapperService mapperService = indexShard.mapperService();
      final ObjectSet<String> warmUp = new ObjectHashSet<>();
      for (DocumentMapper docMapper : mapperService.docMappers(false)) {
        for (FieldMapper fieldMapper : docMapper.mappers()) {
          final String indexName = fieldMapper.fieldType().names().indexName();
          Loading normsLoading = fieldMapper.fieldType().normsLoading();
          if (normsLoading == null) {
            normsLoading = defaultLoading;
          }
          if (fieldMapper.fieldType().indexOptions() != IndexOptions.NONE
              && !fieldMapper.fieldType().omitNorms()
              && normsLoading == Loading.EAGER) {
            warmUp.add(indexName);
          }
        }
      }

      final CountDownLatch latch = new CountDownLatch(1);
      // Norms loading may be I/O intensive but is not CPU intensive, so we execute it in a single
      // task
      threadPool
          .executor(executor())
          .execute(
              new Runnable() {
                @Override
                public void run() {
                  try {
                    for (ObjectCursor<String> stringObjectCursor : warmUp) {
                      final String indexName = stringObjectCursor.value;
                      final long start = System.nanoTime();
                      for (final LeafReaderContext ctx : context.searcher().reader().leaves()) {
                        final NumericDocValues values = ctx.reader().getNormValues(indexName);
                        if (values != null) {
                          values.get(0);
                        }
                      }
                      if (indexShard.warmerService().logger().isTraceEnabled()) {
                        indexShard
                            .warmerService()
                            .logger()
                            .trace(
                                "warmed norms for [{}], took [{}]",
                                indexName,
                                TimeValue.timeValueNanos(System.nanoTime() - start));
                      }
                    }
                  } catch (Throwable t) {
                    indexShard.warmerService().logger().warn("failed to warm-up norms", t);
                  } finally {
                    latch.countDown();
                  }
                }
              });

      return new TerminationHandle() {
        @Override
        public void awaitTermination() throws InterruptedException {
          latch.await();
        }
      };
    }
 @Override
 public TerminationHandle warmTopReader(
     final IndexShard indexShard,
     IndexMetaData indexMetaData,
     final WarmerContext context,
     ThreadPool threadPool) {
   final MapperService mapperService = indexShard.mapperService();
   final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
   for (DocumentMapper docMapper : mapperService.docMappers(false)) {
     for (FieldMapper fieldMapper : docMapper.mappers()) {
       final FieldDataType fieldDataType = fieldMapper.fieldType().fieldDataType();
       if (fieldDataType == null) {
         continue;
       }
       if (fieldDataType.getLoading() != Loading.EAGER_GLOBAL_ORDINALS) {
         continue;
       }
       final String indexName = fieldMapper.fieldType().names().indexName();
       if (warmUpGlobalOrdinals.containsKey(indexName)) {
         continue;
       }
       warmUpGlobalOrdinals.put(indexName, fieldMapper.fieldType());
     }
   }
   final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
   final Executor executor = threadPool.executor(executor());
   final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size());
   for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) {
     executor.execute(
         new Runnable() {
           @Override
           public void run() {
             try {
               final long start = System.nanoTime();
               IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType);
               ifd.loadGlobal(context.reader());
               if (indexShard.warmerService().logger().isTraceEnabled()) {
                 indexShard
                     .warmerService()
                     .logger()
                     .trace(
                         "warmed global ordinals for [{}], took [{}]",
                         fieldType.names().fullName(),
                         TimeValue.timeValueNanos(System.nanoTime() - start));
               }
             } catch (Throwable t) {
               indexShard
                   .warmerService()
                   .logger()
                   .warn(
                       "failed to warm-up global ordinals for [{}]",
                       t,
                       fieldType.names().fullName());
             } finally {
               latch.countDown();
             }
           }
         });
   }
   return new TerminationHandle() {
     @Override
     public void awaitTermination() throws InterruptedException {
       latch.await();
     }
   };
 }
  @Override
  public HighlightField highlight(HighlighterContext highlighterContext) {
    SearchContextHighlight.Field field = highlighterContext.field;
    SearchContext context = highlighterContext.context;
    FetchSubPhase.HitContext hitContext = highlighterContext.hitContext;
    FieldMapper mapper = highlighterContext.mapper;

    if (canHighlight(mapper) == false) {
      throw new IllegalArgumentException(
          "the field ["
              + highlighterContext.fieldName
              + "] should be indexed with term vector with position offsets to be used with fast vector highlighter");
    }

    Encoder encoder =
        field.fieldOptions().encoder().equals("html")
            ? HighlightUtils.Encoders.HTML
            : HighlightUtils.Encoders.DEFAULT;

    if (!hitContext.cache().containsKey(CACHE_KEY)) {
      hitContext.cache().put(CACHE_KEY, new HighlighterEntry());
    }
    HighlighterEntry cache = (HighlighterEntry) hitContext.cache().get(CACHE_KEY);

    try {
      FieldQuery fieldQuery;
      if (field.fieldOptions().requireFieldMatch()) {
        if (cache.fieldMatchFieldQuery == null) {
          /*
           * we use top level reader to rewrite the query against all readers, with use caching it across hits (and across
           * readers...)
           */
          cache.fieldMatchFieldQuery =
              new CustomFieldQuery(
                  highlighterContext.query,
                  hitContext.topLevelReader(),
                  true,
                  field.fieldOptions().requireFieldMatch());
        }
        fieldQuery = cache.fieldMatchFieldQuery;
      } else {
        if (cache.noFieldMatchFieldQuery == null) {
          /*
           * we use top level reader to rewrite the query against all readers, with use caching it across hits (and across
           * readers...)
           */
          cache.noFieldMatchFieldQuery =
              new CustomFieldQuery(
                  highlighterContext.query,
                  hitContext.topLevelReader(),
                  true,
                  field.fieldOptions().requireFieldMatch());
        }
        fieldQuery = cache.noFieldMatchFieldQuery;
      }

      MapperHighlightEntry entry = cache.mappers.get(mapper);
      if (entry == null) {
        FragListBuilder fragListBuilder;
        BaseFragmentsBuilder fragmentsBuilder;

        BoundaryScanner boundaryScanner = DEFAULT_BOUNDARY_SCANNER;
        if (field.fieldOptions().boundaryMaxScan() != SimpleBoundaryScanner.DEFAULT_MAX_SCAN
            || field.fieldOptions().boundaryChars()
                != SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS) {
          boundaryScanner =
              new SimpleBoundaryScanner(
                  field.fieldOptions().boundaryMaxScan(), field.fieldOptions().boundaryChars());
        }
        boolean forceSource = context.highlight().forceSource(field);
        if (field.fieldOptions().numberOfFragments() == 0) {
          fragListBuilder = new SingleFragListBuilder();

          if (!forceSource && mapper.fieldType().stored()) {
            fragmentsBuilder =
                new SimpleFragmentsBuilder(
                    mapper,
                    field.fieldOptions().preTags(),
                    field.fieldOptions().postTags(),
                    boundaryScanner);
          } else {
            fragmentsBuilder =
                new SourceSimpleFragmentsBuilder(
                    mapper,
                    context,
                    field.fieldOptions().preTags(),
                    field.fieldOptions().postTags(),
                    boundaryScanner);
          }
        } else {
          fragListBuilder =
              field.fieldOptions().fragmentOffset() == -1
                  ? new SimpleFragListBuilder()
                  : new SimpleFragListBuilder(field.fieldOptions().fragmentOffset());
          if (field.fieldOptions().scoreOrdered()) {
            if (!forceSource && mapper.fieldType().stored()) {
              fragmentsBuilder =
                  new ScoreOrderFragmentsBuilder(
                      field.fieldOptions().preTags(),
                      field.fieldOptions().postTags(),
                      boundaryScanner);
            } else {
              fragmentsBuilder =
                  new SourceScoreOrderFragmentsBuilder(
                      mapper,
                      context,
                      field.fieldOptions().preTags(),
                      field.fieldOptions().postTags(),
                      boundaryScanner);
            }
          } else {
            if (!forceSource && mapper.fieldType().stored()) {
              fragmentsBuilder =
                  new SimpleFragmentsBuilder(
                      mapper,
                      field.fieldOptions().preTags(),
                      field.fieldOptions().postTags(),
                      boundaryScanner);
            } else {
              fragmentsBuilder =
                  new SourceSimpleFragmentsBuilder(
                      mapper,
                      context,
                      field.fieldOptions().preTags(),
                      field.fieldOptions().postTags(),
                      boundaryScanner);
            }
          }
        }
        fragmentsBuilder.setDiscreteMultiValueHighlighting(termVectorMultiValue);
        entry = new MapperHighlightEntry();
        entry.fragListBuilder = fragListBuilder;
        entry.fragmentsBuilder = fragmentsBuilder;
        if (cache.fvh == null) {
          // parameters to FVH are not requires since:
          // first two booleans are not relevant since they are set on the CustomFieldQuery (phrase
          // and fieldMatch)
          // fragment builders are used explicitly
          cache.fvh = new org.apache.lucene.search.vectorhighlight.FastVectorHighlighter();
        }
        CustomFieldQuery.highlightFilters.set(field.fieldOptions().highlightFilter());
        cache.mappers.put(mapper, entry);
      }
      cache.fvh.setPhraseLimit(field.fieldOptions().phraseLimit());

      String[] fragments;

      // a HACK to make highlighter do highlighting, even though its using the single frag list
      // builder
      int numberOfFragments =
          field.fieldOptions().numberOfFragments() == 0
              ? Integer.MAX_VALUE
              : field.fieldOptions().numberOfFragments();
      int fragmentCharSize =
          field.fieldOptions().numberOfFragments() == 0
              ? Integer.MAX_VALUE
              : field.fieldOptions().fragmentCharSize();
      // we highlight against the low level reader and docId, because if we load source, we want to
      // reuse it if possible
      // Only send matched fields if they were requested to save time.
      if (field.fieldOptions().matchedFields() != null
          && !field.fieldOptions().matchedFields().isEmpty()) {
        fragments =
            cache.fvh.getBestFragments(
                fieldQuery,
                hitContext.reader(),
                hitContext.docId(),
                mapper.fieldType().name(),
                field.fieldOptions().matchedFields(),
                fragmentCharSize,
                numberOfFragments,
                entry.fragListBuilder,
                entry.fragmentsBuilder,
                field.fieldOptions().preTags(),
                field.fieldOptions().postTags(),
                encoder);
      } else {
        fragments =
            cache.fvh.getBestFragments(
                fieldQuery,
                hitContext.reader(),
                hitContext.docId(),
                mapper.fieldType().name(),
                fragmentCharSize,
                numberOfFragments,
                entry.fragListBuilder,
                entry.fragmentsBuilder,
                field.fieldOptions().preTags(),
                field.fieldOptions().postTags(),
                encoder);
      }

      if (fragments != null && fragments.length > 0) {
        return new HighlightField(
            highlighterContext.fieldName, Text.convertFromStringArray(fragments));
      }

      int noMatchSize = highlighterContext.field.fieldOptions().noMatchSize();
      if (noMatchSize > 0) {
        // Essentially we just request that a fragment is built from 0 to noMatchSize using the
        // normal fragmentsBuilder
        FieldFragList fieldFragList = new SimpleFieldFragList(-1 /*ignored*/);
        fieldFragList.add(0, noMatchSize, Collections.<WeightedPhraseInfo>emptyList());
        fragments =
            entry.fragmentsBuilder.createFragments(
                hitContext.reader(),
                hitContext.docId(),
                mapper.fieldType().name(),
                fieldFragList,
                1,
                field.fieldOptions().preTags(),
                field.fieldOptions().postTags(),
                encoder);
        if (fragments != null && fragments.length > 0) {
          return new HighlightField(
              highlighterContext.fieldName, Text.convertFromStringArray(fragments));
        }
      }

      return null;

    } catch (Exception e) {
      throw new FetchPhaseExecutionException(
          context, "Failed to highlight field [" + highlighterContext.fieldName + "]", e);
    }
  }
 @Override
 public boolean canHighlight(FieldMapper fieldMapper) {
   return fieldMapper.fieldType().storeTermVectors()
       && fieldMapper.fieldType().storeTermVectorOffsets()
       && fieldMapper.fieldType().storeTermVectorPositions();
 }