private Fields generateTermVectorsFromDoc(TermVectorRequest request, boolean doAllFields)
      throws IOException {
    // parse the document, at the moment we do update the mapping, just like percolate
    ParsedDocument parsedDocument =
        parseDocument(indexShard.shardId().getIndex(), request.type(), request.doc());

    // select the right fields and generate term vectors
    ParseContext.Document doc = parsedDocument.rootDoc();
    Collection<String> seenFields = new HashSet<>();
    Collection<GetField> getFields = new HashSet<>();
    for (IndexableField field : doc.getFields()) {
      FieldMapper fieldMapper = indexShard.mapperService().smartNameFieldMapper(field.name());
      if (seenFields.contains(field.name())) {
        continue;
      } else {
        seenFields.add(field.name());
      }
      if (!isValidField(fieldMapper)) {
        continue;
      }
      if (request.selectedFields() == null
          && !doAllFields
          && !fieldMapper.fieldType().storeTermVectors()) {
        continue;
      }
      if (request.selectedFields() != null && !request.selectedFields().contains(field.name())) {
        continue;
      }
      String[] values = doc.getValues(field.name());
      getFields.add(new GetField(field.name(), Arrays.asList((Object[]) values)));
    }
    return generateTermVectors(getFields, request.offsets(), request.perFieldAnalyzer());
  }
 private boolean isValidField(FieldMapper field) {
   // must be a string
   if (!(field instanceof StringFieldMapper)) {
     return false;
   }
   // and must be indexed
   if (field.fieldType().indexOptions() == IndexOptions.NONE) {
     return false;
   }
   return true;
 }
  private Fields addGeneratedTermVectors(
      Engine.GetResult get,
      Fields termVectorsByField,
      TermVectorRequest request,
      Set<String> selectedFields)
      throws IOException {
    /* only keep valid fields */
    Set<String> validFields = new HashSet<>();
    for (String field : selectedFields) {
      FieldMapper fieldMapper = indexShard.mapperService().smartNameFieldMapper(field);
      if (!isValidField(fieldMapper)) {
        continue;
      }
      // already retrieved, only if the analyzer hasn't been overridden at the field
      if (fieldMapper.fieldType().storeTermVectors()
          && (request.perFieldAnalyzer() == null
              || !request.perFieldAnalyzer().containsKey(field))) {
        continue;
      }
      validFields.add(field);
    }

    if (validFields.isEmpty()) {
      return termVectorsByField;
    }

    /* generate term vectors from fetched document fields */
    GetResult getResult =
        indexShard
            .getService()
            .get(
                get,
                request.id(),
                request.type(),
                validFields.toArray(Strings.EMPTY_ARRAY),
                null,
                false);
    Fields generatedTermVectors =
        generateTermVectors(
            getResult.getFields().values(), request.offsets(), request.perFieldAnalyzer());

    /* merge with existing Fields */
    if (termVectorsByField == null) {
      return generatedTermVectors;
    } else {
      return mergeFields(termVectorsByField, generatedTermVectors);
    }
  }