public Filter getInnerFilter() throws IOException {
    if (filterParsed) {
      return innerFilter;
    } else {
      if (path == null) {
        throw new QueryParsingException(parseContext.index(), "[nested] requires 'path' field");
      }
      if (!filterFound) {
        throw new QueryParsingException(
            parseContext.index(), "[nested] requires either 'query' or 'filter' field");
      }

      setPathLevel();
      XContentParser old = parseContext.parser();
      try {
        XContentParser innerParser = XContentHelper.createParser(source);
        parseContext.parser(innerParser);
        innerFilter = parseContext.parseInnerFilter();
        filterParsed = true;
        return innerFilter;
      } finally {
        resetPathLevel();
        parseContext.parser(old);
      }
    }
  }
  public void testGeoDistanceSortCanBeParsedFromGeoHash() throws IOException {
    String json =
        "{\n"
            + "    \"VDcvDuFjE\" : [ \"7umzzv8eychg\", \"dmdgmt5z13uw\", "
            + "    \"ezu09wxw6v4c\", \"kc7s3515p6k6\", \"jgeuvjwrmfzn\", \"kcpcfj7ruyf8\" ],\n"
            + "    \"unit\" : \"m\",\n"
            + "    \"distance_type\" : \"sloppy_arc\",\n"
            + "    \"mode\" : \"MAX\",\n"
            + "    \"nested_filter\" : {\n"
            + "      \"ids\" : {\n"
            + "        \"type\" : [ ],\n"
            + "        \"values\" : [ ],\n"
            + "        \"boost\" : 5.711116\n"
            + "      }\n"
            + "    },\n"
            + "    \"validation_method\" : \"STRICT\"\n"
            + "  }";
    XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
    itemParser.nextToken();

    QueryParseContext context =
        new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);

    GeoDistanceSortBuilder result = GeoDistanceSortBuilder.fromXContent(context, json);
    assertEquals(
        "[-19.700583312660456, -2.8225036337971687, "
            + "31.537466906011105, -74.63590376079082, "
            + "43.71844606474042, -5.548660643398762, "
            + "-37.20467280596495, 38.71751043945551, "
            + "-69.44606635719538, 84.25200328230858, "
            + "-39.03717711567879, 44.74099852144718]",
        Arrays.toString(result.points()));
  }
 private static GeoDistanceSortBuilder parse(XContentBuilder sortBuilder) throws Exception {
   XContentParser parser = XContentHelper.createParser(sortBuilder.bytes());
   QueryParseContext parseContext =
       new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
   parser.nextToken();
   return GeoDistanceSortBuilder.fromXContent(parseContext, null);
 }
 static Query parseQuery(QueryBuilder queryBuilder) throws IOException {
   QueryParseContext context =
       new QueryParseContext(new Index("test"), SearchContext.current().queryParserService());
   XContentParser parser = XContentHelper.createParser(queryBuilder.buildAsBytes());
   context.reset(parser);
   return context.parseInnerQuery();
 }
  public void testRegisteredQueries() throws IOException {
    SearchModule module = new SearchModule(Settings.EMPTY, false, emptyList());
    List<String> allSupportedQueries = new ArrayList<>();
    Collections.addAll(allSupportedQueries, NON_DEPRECATED_QUERIES);
    Collections.addAll(allSupportedQueries, DEPRECATED_QUERIES);
    String[] supportedQueries = allSupportedQueries.toArray(new String[allSupportedQueries.size()]);
    assertThat(module.getQueryParserRegistry().getNames(), containsInAnyOrder(supportedQueries));

    IndicesQueriesRegistry indicesQueriesRegistry = module.getQueryParserRegistry();
    XContentParser dummyParser = XContentHelper.createParser(new BytesArray("{}"));
    for (String queryName : supportedQueries) {
      indicesQueriesRegistry.lookup(
          queryName, ParseFieldMatcher.EMPTY, dummyParser.getTokenLocation());
    }

    for (String queryName : NON_DEPRECATED_QUERIES) {
      QueryParser<?> queryParser =
          indicesQueriesRegistry.lookup(
              queryName, ParseFieldMatcher.STRICT, dummyParser.getTokenLocation());
      assertThat(queryParser, notNullValue());
    }
    for (String queryName : DEPRECATED_QUERIES) {
      try {
        indicesQueriesRegistry.lookup(
            queryName, ParseFieldMatcher.STRICT, dummyParser.getTokenLocation());
        fail("query is deprecated, getQueryParser should have failed in strict mode");
      } catch (IllegalArgumentException e) {
        assertThat(e.getMessage(), containsString("Deprecated field [" + queryName + "] used"));
      }
    }
  }
 private MetaData readMetaData(byte[] data) throws IOException {
   XContentParser parser = null;
   try {
     parser = XContentHelper.createParser(data, 0, data.length);
     return MetaData.Builder.fromXContent(parser);
   } finally {
     if (parser != null) {
       parser.close();
     }
   }
 }
Exemple #7
0
 /** Sets the aliases that will be associated with the index when it gets created */
 public CreateIndexRequest aliases(BytesReference source) {
   try (XContentParser parser = XContentHelper.createParser(source)) {
     // move to the first alias
     parser.nextToken();
     while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) {
       alias(Alias.fromXContent(parser));
     }
     return this;
   } catch (IOException e) {
     throw new ElasticsearchParseException("Failed to parse aliases", e);
   }
 }
 private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XContentBuilder builder)
     throws Exception {
   Settings settings =
       Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
   ParseContext.InternalParseContext ctx =
       new ParseContext.InternalParseContext("test", settings, parser, mapper, new ContentPath(0));
   SourceToParse source = SourceToParse.source(builder.bytes());
   ctx.reset(XContentHelper.createParser(source.source()), new ParseContext.Document(), source);
   assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken());
   ctx.parser().nextToken();
   return DocumentParser.parseObject(ctx, mapper.root());
 }
 @Nullable
 private IndexMetaData loadIndex(String index) {
   long highestVersion = -1;
   IndexMetaData indexMetaData = null;
   for (File indexLocation : nodeEnv.indexLocations(new Index(index))) {
     File stateDir = new File(indexLocation, "_state");
     if (!stateDir.exists() || !stateDir.isDirectory()) {
       continue;
     }
     // now, iterate over the current versions, and find latest one
     File[] stateFiles = stateDir.listFiles();
     if (stateFiles == null) {
       continue;
     }
     for (File stateFile : stateFiles) {
       if (!stateFile.getName().startsWith("state-")) {
         continue;
       }
       try {
         long version = Long.parseLong(stateFile.getName().substring("state-".length()));
         if (version > highestVersion) {
           byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile));
           if (data.length == 0) {
             logger.debug(
                 "[{}]: no data for [" + stateFile.getAbsolutePath() + "], ignoring...", index);
             continue;
           }
           XContentParser parser = null;
           try {
             parser = XContentHelper.createParser(data, 0, data.length);
             parser.nextToken(); // move to START_OBJECT
             indexMetaData = IndexMetaData.Builder.fromXContent(parser);
             highestVersion = version;
           } finally {
             if (parser != null) {
               parser.close();
             }
           }
         }
       } catch (Exception e) {
         logger.debug(
             "[{}]: failed to read [" + stateFile.getAbsolutePath() + "], ignoring...", e, index);
       }
     }
   }
   return indexMetaData;
 }
Exemple #10
0
  Query parsePercolatorDocument(String id, BytesReference source) {
    String type = null;
    BytesReference querySource = null;

    XContentParser parser = null;
    try {
      parser = XContentHelper.createParser(source);
      String currentFieldName = null;
      XContentParser.Token token = parser.nextToken(); // move the START_OBJECT
      if (token != XContentParser.Token.START_OBJECT) {
        throw new ElasticsearchException(
            "failed to parse query [" + id + "], not starting with OBJECT");
      }
      while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
        if (token == XContentParser.Token.FIELD_NAME) {
          currentFieldName = parser.currentName();
        } else if (token == XContentParser.Token.START_OBJECT) {
          if ("query".equals(currentFieldName)) {
            if (type != null) {
              return parseQuery(type, null, parser);
            } else {
              XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType());
              builder.copyCurrentStructure(parser);
              querySource = builder.bytes();
              builder.close();
            }
          } else {
            parser.skipChildren();
          }
        } else if (token == XContentParser.Token.START_ARRAY) {
          parser.skipChildren();
        } else if (token.isValue()) {
          if ("type".equals(currentFieldName)) {
            type = parser.text();
          }
        }
      }
      return parseQuery(type, querySource, null);
    } catch (Exception e) {
      throw new PercolatorException(shardId().index(), "failed to parse query [" + id + "]", e);
    } finally {
      if (parser != null) {
        parser.close();
      }
    }
  }
  private MetaData loadGlobalState() {
    long highestVersion = -1;
    MetaData metaData = null;
    for (File dataLocation : nodeEnv.nodeDataLocations()) {
      File stateLocation = new File(dataLocation, "_state");
      if (!stateLocation.exists()) {
        continue;
      }
      File[] stateFiles = stateLocation.listFiles();
      if (stateFiles == null) {
        continue;
      }
      for (File stateFile : stateFiles) {
        String name = stateFile.getName();
        if (!name.startsWith("global-")) {
          continue;
        }
        try {
          long version = Long.parseLong(stateFile.getName().substring("global-".length()));
          if (version > highestVersion) {
            byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile));
            if (data.length == 0) {
              logger.debug(
                  "[_global] no data for [" + stateFile.getAbsolutePath() + "], ignoring...");
              continue;
            }

            XContentParser parser = null;
            try {
              parser = XContentHelper.createParser(data, 0, data.length);
              metaData = MetaData.Builder.fromXContent(parser);
              highestVersion = version;
            } finally {
              if (parser != null) {
                parser.close();
              }
            }
          }
        } catch (Exception e) {
          logger.debug("failed to load global state from [{}]", e, stateFile.getAbsolutePath());
        }
      }
    }

    return metaData;
  }
 public void testFromXContent() throws Exception {
   SliceBuilder sliceBuilder = randomSliceBuilder();
   XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
   if (randomBoolean()) {
     builder.prettyPrint();
   }
   builder.startObject();
   sliceBuilder.innerToXContent(builder);
   builder.endObject();
   XContentParser parser = XContentHelper.createParser(shuffleXContent(builder).bytes());
   QueryParseContext context =
       new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
   SliceBuilder secondSliceBuilder = SliceBuilder.fromXContent(context);
   assertNotSame(sliceBuilder, secondSliceBuilder);
   assertEquals(sliceBuilder, secondSliceBuilder);
   assertEquals(sliceBuilder.hashCode(), secondSliceBuilder.hashCode());
 }
 /**
  * Test that creates new smoothing model from a random test smoothing model and checks both for
  * equality
  */
 public void testFromXContent() throws IOException {
   SmoothingModel testModel = createTestModel();
   XContentBuilder contentBuilder =
       XContentFactory.contentBuilder(randomFrom(XContentType.values()));
   if (randomBoolean()) {
     contentBuilder.prettyPrint();
   }
   contentBuilder.startObject();
   testModel.innerToXContent(contentBuilder, ToXContent.EMPTY_PARAMS);
   contentBuilder.endObject();
   XContentParser parser = XContentHelper.createParser(shuffleXContent(contentBuilder).bytes());
   QueryParseContext context =
       new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
   parser.nextToken(); // go to start token, real parsing would do that in the outer element parser
   SmoothingModel parsedModel = fromXContent(context);
   assertNotSame(testModel, parsedModel);
   assertEquals(testModel, parsedModel);
   assertEquals(testModel.hashCode(), parsedModel.hashCode());
 }
  public void testReverseOptionFailsBuildWhenInvalidGeoHashString() throws IOException {
    String json = "{\n" + "  \"reverse\" : \"false\"\n" + "}";
    XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
    itemParser.nextToken();

    QueryParseContext context =
        new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);

    try {
      GeoDistanceSortBuilder item = GeoDistanceSortBuilder.fromXContent(context, "");
      item.validation(GeoValidationMethod.STRICT);
      item.build(createMockShardContext());

      fail("adding reverse sorting option should fail with an exception");
    } catch (ElasticsearchParseException e) {
      assertEquals(
          "illegal latitude value [269.384765625] for [GeoDistanceSort] for field [reverse].",
          e.getMessage());
    }
  }
  /**
   * creates random suggestion builder, renders it to xContent and back to new instance that should
   * be equal to original
   */
  public void testFromXContent() throws IOException {
    Suggesters suggesters = new Suggesters(Collections.emptyMap());
    QueryParseContext context = new QueryParseContext(null);
    context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY));
    for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) {
      SuggestBuilder suggestBuilder = createTestModel();
      XContentBuilder xContentBuilder =
          XContentFactory.contentBuilder(randomFrom(XContentType.values()));
      if (randomBoolean()) {
        xContentBuilder.prettyPrint();
      }
      suggestBuilder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
      XContentParser parser = XContentHelper.createParser(xContentBuilder.bytes());
      context.reset(parser);

      SuggestBuilder secondSuggestBuilder = SuggestBuilder.fromXContent(context, suggesters);
      assertNotSame(suggestBuilder, secondSuggestBuilder);
      assertEquals(suggestBuilder, secondSuggestBuilder);
      assertEquals(suggestBuilder.hashCode(), secondSuggestBuilder.hashCode());
    }
  }
  public void testSortModeSumIsRejectedInJSON() throws IOException {
    String json =
        "{\n"
            + "  \"testname\" : [ {\n"
            + "    \"lat\" : -6.046997540714173,\n"
            + "    \"lon\" : -51.94128329747579\n"
            + "  } ],\n"
            + "  \"unit\" : \"m\",\n"
            + "  \"distance_type\" : \"sloppy_arc\",\n"
            + "  \"mode\" : \"SUM\"\n"
            + "}";
    XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
    itemParser.nextToken();

    QueryParseContext context =
        new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);

    IllegalArgumentException e =
        expectThrows(
            IllegalArgumentException.class, () -> GeoDistanceSortBuilder.fromXContent(context, ""));
    assertEquals("sort_mode [sum] isn't supported for sorting by geo distance", e.getMessage());
  }
  public void testReverseOptionFailsWhenNonStringField() throws IOException {
    String json =
        "{\n"
            + "  \"testname\" : [ {\n"
            + "    \"lat\" : -6.046997540714173,\n"
            + "    \"lon\" : -51.94128329747579\n"
            + "  } ],\n"
            + "  \"reverse\" : true\n"
            + "}";
    XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
    itemParser.nextToken();

    QueryParseContext context =
        new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);

    try {
      GeoDistanceSortBuilder.fromXContent(context, "");
      fail("adding reverse sorting option should fail with an exception");
    } catch (ParsingException e) {
      assertEquals("Only geohashes of type string supported for field [reverse]", e.getMessage());
    }
  }
  public void testIgnoreMalformedIsDeprecated() throws IOException {
    String json =
        "{\n"
            + "  \"testname\" : [ {\n"
            + "    \"lat\" : -6.046997540714173,\n"
            + "    \"lon\" : -51.94128329747579\n"
            + "  } ],\n"
            + "  \"unit\" : \"m\",\n"
            + "  \"distance_type\" : \"sloppy_arc\",\n"
            + "  \"mode\" : \"SUM\",\n"
            + "  \"ignore_malformed\" : true\n"
            + "}";
    XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
    itemParser.nextToken();

    QueryParseContext context =
        new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);

    IllegalArgumentException e =
        expectThrows(
            IllegalArgumentException.class, () -> GeoDistanceSortBuilder.fromXContent(context, ""));
    assertTrue(e.getMessage().startsWith("Deprecated field "));
  }
  private ParsedQuery parseQuery(ExplainRequest request, IndexService indexService) {
    try {
      XContentParser parser = XContentHelper.createParser(request.getSource());
      for (XContentParser.Token token = parser.nextToken();
          token != XContentParser.Token.END_OBJECT;
          token = parser.nextToken()) {
        if (token == XContentParser.Token.FIELD_NAME) {
          String fieldName = parser.currentName();
          if ("query".equals(fieldName)) {
            return indexService.queryParserService().parse(parser);
          } else if ("query_binary".equals(fieldName)) {
            byte[] querySource = parser.binaryValue();
            XContentParser qSourceParser =
                XContentFactory.xContent(querySource).createParser(querySource);
            return indexService.queryParserService().parse(qSourceParser);
          }
        }
      }
    } catch (Exception e) {
      throw new ElasticSearchException("Couldn't parse query from source.", e);
    }

    throw new ElasticSearchException("No query specified");
  }
  public void process(
      MetaData metaData,
      String aliasOrIndex,
      @Nullable MappingMetaData mappingMd,
      boolean allowIdGeneration)
      throws ElasticsearchException {
    // resolve the routing if needed
    routing(metaData.resolveIndexRouting(routing, aliasOrIndex));
    // resolve timestamp if provided externally
    if (timestamp != null) {
      timestamp =
          MappingMetaData.Timestamp.parseStringTimestamp(
              timestamp,
              mappingMd != null
                  ? mappingMd.timestamp().dateTimeFormatter()
                  : TimestampFieldMapper.Defaults.DATE_TIME_FORMATTER);
    }
    // extract values if needed
    if (mappingMd != null) {
      MappingMetaData.ParseContext parseContext =
          mappingMd.createParseContext(id, routing, timestamp);

      if (parseContext.shouldParse()) {
        XContentParser parser = null;
        try {
          parser = XContentHelper.createParser(source);
          mappingMd.parse(parser, parseContext);
          if (parseContext.shouldParseId()) {
            id = parseContext.id();
          }
          if (parseContext.shouldParseRouting()) {
            routing = parseContext.routing();
          }
          if (parseContext.shouldParseTimestamp()) {
            timestamp = parseContext.timestamp();
            timestamp =
                MappingMetaData.Timestamp.parseStringTimestamp(
                    timestamp, mappingMd.timestamp().dateTimeFormatter());
          }
        } catch (Exception e) {
          throw new ElasticsearchParseException(
              "failed to parse doc to extract routing/timestamp", e);
        } finally {
          if (parser != null) {
            parser.close();
          }
        }
      }

      // might as well check for routing here
      if (mappingMd.routing().required() && routing == null) {
        throw new RoutingMissingException(index, type, id);
      }

      if (parent != null && !mappingMd.hasParentField()) {
        throw new ElasticsearchIllegalArgumentException(
            "Can't specify parent if no parent field has been configured");
      }
    } else {
      if (parent != null) {
        throw new ElasticsearchIllegalArgumentException(
            "Can't specify parent if no parent field has been configured");
      }
    }

    // generate id if not already provided and id generation is allowed
    if (allowIdGeneration) {
      if (id == null) {
        id(Strings.randomBase64UUID());
        // since we generate the id, change it to CREATE
        opType(IndexRequest.OpType.CREATE);
      }
    }

    // generate timestamp if not provided, we always have one post this stage...
    if (timestamp == null) {
      timestamp = Long.toString(System.currentTimeMillis());
    }
  }
  public ParsedDocument parse(SourceToParse source, @Nullable ParseListener listener)
      throws MapperParsingException {
    ParseContext context = cache.get();

    if (source.type() != null && !source.type().equals(this.type)) {
      throw new MapperParsingException(
          "Type mismatch, provide type ["
              + source.type()
              + "] but mapper is of type ["
              + this.type
              + "]");
    }
    source.type(this.type);

    XContentParser parser = source.parser();
    try {
      if (parser == null) {
        parser = XContentHelper.createParser(source.source());
      }
      context.reset(parser, new Document(), source, listener);
      // on a newly created instance of document mapper, we always consider it as new mappers that
      // have been added
      if (initMappersAdded) {
        context.setMappingsModified();
        initMappersAdded = false;
      }

      // will result in START_OBJECT
      int countDownTokens = 0;
      XContentParser.Token token = parser.nextToken();
      if (token != XContentParser.Token.START_OBJECT) {
        throw new MapperParsingException("Malformed content, must start with an object");
      }
      boolean emptyDoc = false;
      token = parser.nextToken();
      if (token == XContentParser.Token.END_OBJECT) {
        // empty doc, we can handle it...
        emptyDoc = true;
      } else if (token != XContentParser.Token.FIELD_NAME) {
        throw new MapperParsingException(
            "Malformed content, after first object, either the type field or the actual properties should exist");
      }
      if (type.equals(parser.currentName())) {
        // first field is the same as the type, this might be because the type is provided, and the
        // object exists within it
        // or because there is a valid field that by chance is named as the type

        // Note, in this case, we only handle plain value types, an object type will be analyzed as
        // if it was the type itself
        // and other same level fields will be ignored
        token = parser.nextToken();
        countDownTokens++;
        // commented out, allow for same type with START_OBJECT, we do our best to handle it except
        // for the above corner case
        //                if (token != XContentParser.Token.START_OBJECT) {
        //                    throw new MapperException("Malformed content, a field with the same
        // name as the type must be an object with the properties/fields within it");
        //                }
      }

      for (RootMapper rootMapper : rootMappersOrdered) {
        rootMapper.preParse(context);
      }

      if (!emptyDoc) {
        rootObjectMapper.parse(context);
      }

      for (int i = 0; i < countDownTokens; i++) {
        parser.nextToken();
      }

      // fire up any new mappers if exists
      if (!context.newFieldMappers().mappers.isEmpty()) {
        addFieldMappers(context.newFieldMappers().mappers);
        context.newFieldMappers().mappers.clear();
      }
      if (!context.newObjectMappers().mappers.isEmpty()) {
        addObjectMappers(context.newObjectMappers().mappers);
        context.newObjectMappers().mappers.clear();
      }

      for (RootMapper rootMapper : rootMappersOrdered) {
        rootMapper.postParse(context);
      }

      for (RootMapper rootMapper : rootMappersOrdered) {
        rootMapper.validate(context);
      }
    } catch (Throwable e) {
      // we have to fire up any new mappers even on a failure, because they
      // have been added internally to each compound mapper...
      // ... we have no option to "rollback" a change, which is very tricky in our copy on change
      // system...
      if (!context.newFieldMappers().mappers.isEmpty()) {
        addFieldMappers(context.newFieldMappers().mappers);
        context.newFieldMappers().mappers.clear();
      }
      if (!context.newObjectMappers().mappers.isEmpty()) {
        addObjectMappers(context.newObjectMappers().mappers);
        context.newObjectMappers().mappers.clear();
      }

      // if its already a mapper parsing exception, no need to wrap it...
      if (e instanceof MapperParsingException) {
        throw (MapperParsingException) e;
      }

      throw new MapperParsingException("failed to parse", e);
    } finally {
      // only close the parser when its not provided externally
      if (source.parser() == null && parser != null) {
        parser.close();
      }
    }
    // reverse the order of docs for nested docs support, parent should be last
    if (context.docs().size() > 1) {
      Collections.reverse(context.docs());
    }
    // apply doc boost
    if (context.docBoost() != 1.0f) {
      Set<String> encounteredFields = Sets.newHashSet();
      for (Document doc : context.docs()) {
        encounteredFields.clear();
        for (IndexableField field : doc) {
          if (field.fieldType().indexed() && !field.fieldType().omitNorms()) {
            if (!encounteredFields.contains(field.name())) {
              ((Field) field).setBoost(context.docBoost() * field.boost());
              encounteredFields.add(field.name());
            }
          }
        }
      }
    }

    ParsedDocument doc =
        new ParsedDocument(
                context.uid(),
                context.id(),
                context.type(),
                source.routing(),
                source.timestamp(),
                source.ttl(),
                context.docs(),
                context.analyzer(),
                context.source(),
                context.mappingsModified())
            .parent(source.parent());
    // reset the context to free up memory
    context.reset(null, null, null, null);
    return doc;
  }
  private ParsedDocument innerParseDocument(SourceToParse source) throws MapperParsingException {
    if (docMapper.type().equals(MapperService.DEFAULT_MAPPING)) {
      throw new IllegalArgumentException(
          "It is forbidden to index into the default mapping ["
              + MapperService.DEFAULT_MAPPING
              + "]");
    }

    ParseContext.InternalParseContext context = cache.get();

    final Mapping mapping = docMapper.mapping();
    if (source.type() != null && !source.type().equals(docMapper.type())) {
      throw new MapperParsingException(
          "Type mismatch, provide type ["
              + source.type()
              + "] but mapper is of type ["
              + docMapper.type()
              + "]");
    }
    source.type(docMapper.type());

    XContentParser parser = source.parser();
    try {
      if (parser == null) {
        parser = XContentHelper.createParser(source.source());
      }
      context.reset(parser, new ParseContext.Document(), source);

      // will result in START_OBJECT
      XContentParser.Token token = parser.nextToken();
      if (token != XContentParser.Token.START_OBJECT) {
        throw new MapperParsingException("Malformed content, must start with an object");
      }

      boolean emptyDoc = false;
      if (mapping.root.isEnabled()) {
        token = parser.nextToken();
        if (token == XContentParser.Token.END_OBJECT) {
          // empty doc, we can handle it...
          emptyDoc = true;
        } else if (token != XContentParser.Token.FIELD_NAME) {
          throw new MapperParsingException(
              "Malformed content, after first object, either the type field or the actual properties should exist");
        }
      }

      for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) {
        metadataMapper.preParse(context);
      }

      if (mapping.root.isEnabled() == false) {
        // entire type is disabled
        parser.skipChildren();
      } else if (emptyDoc == false) {
        Mapper update = parseObject(context, mapping.root, true);
        if (update != null) {
          context.addDynamicMappingsUpdate(update);
        }
      }

      for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) {
        metadataMapper.postParse(context);
      }

      // try to parse the next token, this should be null if the object is ended properly
      // but will throw a JSON exception if the extra tokens is not valid JSON (this will be handled
      // by the catch)
      if (Version.indexCreated(indexSettings).onOrAfter(Version.V_2_0_0_beta1)
          && source.parser() == null
          && parser != null) {
        // only check for end of tokens if we created the parser here
        token = parser.nextToken();
        if (token != null) {
          throw new IllegalArgumentException(
              "Malformed content, found extra data after parsing: " + token);
        }
      }

    } catch (Throwable e) {
      // if its already a mapper parsing exception, no need to wrap it...
      if (e instanceof MapperParsingException) {
        throw (MapperParsingException) e;
      }

      // Throw a more meaningful message if the document is empty.
      if (source.source() != null && source.source().length() == 0) {
        throw new MapperParsingException("failed to parse, document is empty");
      }

      throw new MapperParsingException("failed to parse", e);
    } finally {
      // only close the parser when its not provided externally
      if (source.parser() == null && parser != null) {
        parser.close();
      }
    }
    // reverse the order of docs for nested docs support, parent should be last
    if (context.docs().size() > 1) {
      Collections.reverse(context.docs());
    }
    // apply doc boost
    if (context.docBoost() != 1.0f) {
      Set<String> encounteredFields = new HashSet<>();
      for (ParseContext.Document doc : context.docs()) {
        encounteredFields.clear();
        for (IndexableField field : doc) {
          if (field.fieldType().indexOptions() != IndexOptions.NONE
              && !field.fieldType().omitNorms()) {
            if (!encounteredFields.contains(field.name())) {
              ((Field) field).setBoost(context.docBoost() * field.boost());
              encounteredFields.add(field.name());
            }
          }
        }
      }
    }

    Mapper rootDynamicUpdate = context.dynamicMappingsUpdate();
    Mapping update = null;
    if (rootDynamicUpdate != null) {
      update = mapping.mappingUpdate(rootDynamicUpdate);
    }

    ParsedDocument doc =
        new ParsedDocument(
                context.uid(),
                context.version(),
                context.id(),
                context.type(),
                source.routing(),
                source.timestamp(),
                source.ttl(),
                context.docs(),
                context.source(),
                update)
            .parent(source.parent());
    // reset the context to free up memory
    context.reset(null, null, null);
    return doc;
  }
  private void pre019Upgrade() throws Exception {
    long index = -1;
    File metaDataFile = null;
    MetaData metaData = null;
    long version = -1;
    for (File dataLocation : nodeEnv.nodeDataLocations()) {
      File stateLocation = new File(dataLocation, "_state");
      if (!stateLocation.exists()) {
        continue;
      }
      File[] stateFiles = stateLocation.listFiles();
      if (stateFiles == null) {
        continue;
      }
      for (File stateFile : stateFiles) {
        if (logger.isTraceEnabled()) {
          logger.trace("[upgrade]: processing [" + stateFile.getName() + "]");
        }
        String name = stateFile.getName();
        if (!name.startsWith("metadata-")) {
          continue;
        }
        long fileIndex = Long.parseLong(name.substring(name.indexOf('-') + 1));
        if (fileIndex >= index) {
          // try and read the meta data
          try {
            byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile));
            if (data.length == 0) {
              continue;
            }
            XContentParser parser = XContentHelper.createParser(data, 0, data.length);
            try {
              String currentFieldName = null;
              XContentParser.Token token = parser.nextToken();
              if (token != null) {
                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                  if (token == XContentParser.Token.FIELD_NAME) {
                    currentFieldName = parser.currentName();
                  } else if (token == XContentParser.Token.START_OBJECT) {
                    if ("meta-data".equals(currentFieldName)) {
                      metaData = MetaData.Builder.fromXContent(parser);
                    }
                  } else if (token.isValue()) {
                    if ("version".equals(currentFieldName)) {
                      version = parser.longValue();
                    }
                  }
                }
              }
            } finally {
              parser.close();
            }
            index = fileIndex;
            metaDataFile = stateFile;
          } catch (IOException e) {
            logger.warn("failed to read pre 0.19 state from [" + name + "], ignoring...", e);
          }
        }
      }
    }
    if (metaData == null) {
      return;
    }

    logger.info(
        "found old metadata state, loading metadata from [{}] and converting to new metadata location and strucutre...",
        metaDataFile.getAbsolutePath());

    writeGlobalState(
        "upgrade", MetaData.builder().metaData(metaData).version(version).build(), null);
    for (IndexMetaData indexMetaData : metaData) {
      IndexMetaData.Builder indexMetaDataBuilder =
          IndexMetaData.newIndexMetaDataBuilder(indexMetaData).version(version);
      // set the created version to 0.18
      indexMetaDataBuilder.settings(
          ImmutableSettings.settingsBuilder()
              .put(indexMetaData.settings())
              .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_18_0));
      writeIndex("upgrade", indexMetaDataBuilder.build(), null);
    }

    // rename shards state to backup state
    File backupFile = new File(metaDataFile.getParentFile(), "backup-" + metaDataFile.getName());
    if (!metaDataFile.renameTo(backupFile)) {
      throw new IOException(
          "failed to rename old state to backup state [" + metaDataFile.getAbsolutePath() + "]");
    }

    // delete all other shards state files
    for (File dataLocation : nodeEnv.nodeDataLocations()) {
      File stateLocation = new File(dataLocation, "_state");
      if (!stateLocation.exists()) {
        continue;
      }
      File[] stateFiles = stateLocation.listFiles();
      if (stateFiles == null) {
        continue;
      }
      for (File stateFile : stateFiles) {
        String name = stateFile.getName();
        if (!name.startsWith("metadata-")) {
          continue;
        }
        stateFile.delete();
      }
    }

    logger.info(
        "conversion to new metadata location and format done, backup create at [{}]",
        backupFile.getAbsolutePath());
  }