public Filter getInnerFilter() throws IOException { if (filterParsed) { return innerFilter; } else { if (path == null) { throw new QueryParsingException(parseContext.index(), "[nested] requires 'path' field"); } if (!filterFound) { throw new QueryParsingException( parseContext.index(), "[nested] requires either 'query' or 'filter' field"); } setPathLevel(); XContentParser old = parseContext.parser(); try { XContentParser innerParser = XContentHelper.createParser(source); parseContext.parser(innerParser); innerFilter = parseContext.parseInnerFilter(); filterParsed = true; return innerFilter; } finally { resetPathLevel(); parseContext.parser(old); } } }
public void testGeoDistanceSortCanBeParsedFromGeoHash() throws IOException { String json = "{\n" + " \"VDcvDuFjE\" : [ \"7umzzv8eychg\", \"dmdgmt5z13uw\", " + " \"ezu09wxw6v4c\", \"kc7s3515p6k6\", \"jgeuvjwrmfzn\", \"kcpcfj7ruyf8\" ],\n" + " \"unit\" : \"m\",\n" + " \"distance_type\" : \"sloppy_arc\",\n" + " \"mode\" : \"MAX\",\n" + " \"nested_filter\" : {\n" + " \"ids\" : {\n" + " \"type\" : [ ],\n" + " \"values\" : [ ],\n" + " \"boost\" : 5.711116\n" + " }\n" + " },\n" + " \"validation_method\" : \"STRICT\"\n" + " }"; XContentParser itemParser = XContentHelper.createParser(new BytesArray(json)); itemParser.nextToken(); QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT); GeoDistanceSortBuilder result = GeoDistanceSortBuilder.fromXContent(context, json); assertEquals( "[-19.700583312660456, -2.8225036337971687, " + "31.537466906011105, -74.63590376079082, " + "43.71844606474042, -5.548660643398762, " + "-37.20467280596495, 38.71751043945551, " + "-69.44606635719538, 84.25200328230858, " + "-39.03717711567879, 44.74099852144718]", Arrays.toString(result.points())); }
private static GeoDistanceSortBuilder parse(XContentBuilder sortBuilder) throws Exception { XContentParser parser = XContentHelper.createParser(sortBuilder.bytes()); QueryParseContext parseContext = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT); parser.nextToken(); return GeoDistanceSortBuilder.fromXContent(parseContext, null); }
static Query parseQuery(QueryBuilder queryBuilder) throws IOException { QueryParseContext context = new QueryParseContext(new Index("test"), SearchContext.current().queryParserService()); XContentParser parser = XContentHelper.createParser(queryBuilder.buildAsBytes()); context.reset(parser); return context.parseInnerQuery(); }
public void testRegisteredQueries() throws IOException { SearchModule module = new SearchModule(Settings.EMPTY, false, emptyList()); List<String> allSupportedQueries = new ArrayList<>(); Collections.addAll(allSupportedQueries, NON_DEPRECATED_QUERIES); Collections.addAll(allSupportedQueries, DEPRECATED_QUERIES); String[] supportedQueries = allSupportedQueries.toArray(new String[allSupportedQueries.size()]); assertThat(module.getQueryParserRegistry().getNames(), containsInAnyOrder(supportedQueries)); IndicesQueriesRegistry indicesQueriesRegistry = module.getQueryParserRegistry(); XContentParser dummyParser = XContentHelper.createParser(new BytesArray("{}")); for (String queryName : supportedQueries) { indicesQueriesRegistry.lookup( queryName, ParseFieldMatcher.EMPTY, dummyParser.getTokenLocation()); } for (String queryName : NON_DEPRECATED_QUERIES) { QueryParser<?> queryParser = indicesQueriesRegistry.lookup( queryName, ParseFieldMatcher.STRICT, dummyParser.getTokenLocation()); assertThat(queryParser, notNullValue()); } for (String queryName : DEPRECATED_QUERIES) { try { indicesQueriesRegistry.lookup( queryName, ParseFieldMatcher.STRICT, dummyParser.getTokenLocation()); fail("query is deprecated, getQueryParser should have failed in strict mode"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), containsString("Deprecated field [" + queryName + "] used")); } } }
private MetaData readMetaData(byte[] data) throws IOException { XContentParser parser = null; try { parser = XContentHelper.createParser(data, 0, data.length); return MetaData.Builder.fromXContent(parser); } finally { if (parser != null) { parser.close(); } } }
/** Sets the aliases that will be associated with the index when it gets created */ public CreateIndexRequest aliases(BytesReference source) { try (XContentParser parser = XContentHelper.createParser(source)) { // move to the first alias parser.nextToken(); while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { alias(Alias.fromXContent(parser)); } return this; } catch (IOException e) { throw new ElasticsearchParseException("Failed to parse aliases", e); } }
private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XContentBuilder builder) throws Exception { Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); ParseContext.InternalParseContext ctx = new ParseContext.InternalParseContext("test", settings, parser, mapper, new ContentPath(0)); SourceToParse source = SourceToParse.source(builder.bytes()); ctx.reset(XContentHelper.createParser(source.source()), new ParseContext.Document(), source); assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken()); ctx.parser().nextToken(); return DocumentParser.parseObject(ctx, mapper.root()); }
@Nullable private IndexMetaData loadIndex(String index) { long highestVersion = -1; IndexMetaData indexMetaData = null; for (File indexLocation : nodeEnv.indexLocations(new Index(index))) { File stateDir = new File(indexLocation, "_state"); if (!stateDir.exists() || !stateDir.isDirectory()) { continue; } // now, iterate over the current versions, and find latest one File[] stateFiles = stateDir.listFiles(); if (stateFiles == null) { continue; } for (File stateFile : stateFiles) { if (!stateFile.getName().startsWith("state-")) { continue; } try { long version = Long.parseLong(stateFile.getName().substring("state-".length())); if (version > highestVersion) { byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); if (data.length == 0) { logger.debug( "[{}]: no data for [" + stateFile.getAbsolutePath() + "], ignoring...", index); continue; } XContentParser parser = null; try { parser = XContentHelper.createParser(data, 0, data.length); parser.nextToken(); // move to START_OBJECT indexMetaData = IndexMetaData.Builder.fromXContent(parser); highestVersion = version; } finally { if (parser != null) { parser.close(); } } } } catch (Exception e) { logger.debug( "[{}]: failed to read [" + stateFile.getAbsolutePath() + "], ignoring...", e, index); } } } return indexMetaData; }
Query parsePercolatorDocument(String id, BytesReference source) { String type = null; BytesReference querySource = null; XContentParser parser = null; try { parser = XContentHelper.createParser(source); String currentFieldName = null; XContentParser.Token token = parser.nextToken(); // move the START_OBJECT if (token != XContentParser.Token.START_OBJECT) { throw new ElasticsearchException( "failed to parse query [" + id + "], not starting with OBJECT"); } while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { if ("query".equals(currentFieldName)) { if (type != null) { return parseQuery(type, null, parser); } else { XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType()); builder.copyCurrentStructure(parser); querySource = builder.bytes(); builder.close(); } } else { parser.skipChildren(); } } else if (token == XContentParser.Token.START_ARRAY) { parser.skipChildren(); } else if (token.isValue()) { if ("type".equals(currentFieldName)) { type = parser.text(); } } } return parseQuery(type, querySource, null); } catch (Exception e) { throw new PercolatorException(shardId().index(), "failed to parse query [" + id + "]", e); } finally { if (parser != null) { parser.close(); } } }
private MetaData loadGlobalState() { long highestVersion = -1; MetaData metaData = null; for (File dataLocation : nodeEnv.nodeDataLocations()) { File stateLocation = new File(dataLocation, "_state"); if (!stateLocation.exists()) { continue; } File[] stateFiles = stateLocation.listFiles(); if (stateFiles == null) { continue; } for (File stateFile : stateFiles) { String name = stateFile.getName(); if (!name.startsWith("global-")) { continue; } try { long version = Long.parseLong(stateFile.getName().substring("global-".length())); if (version > highestVersion) { byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); if (data.length == 0) { logger.debug( "[_global] no data for [" + stateFile.getAbsolutePath() + "], ignoring..."); continue; } XContentParser parser = null; try { parser = XContentHelper.createParser(data, 0, data.length); metaData = MetaData.Builder.fromXContent(parser); highestVersion = version; } finally { if (parser != null) { parser.close(); } } } } catch (Exception e) { logger.debug("failed to load global state from [{}]", e, stateFile.getAbsolutePath()); } } } return metaData; }
public void testFromXContent() throws Exception { SliceBuilder sliceBuilder = randomSliceBuilder(); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); } builder.startObject(); sliceBuilder.innerToXContent(builder); builder.endObject(); XContentParser parser = XContentHelper.createParser(shuffleXContent(builder).bytes()); QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT); SliceBuilder secondSliceBuilder = SliceBuilder.fromXContent(context); assertNotSame(sliceBuilder, secondSliceBuilder); assertEquals(sliceBuilder, secondSliceBuilder); assertEquals(sliceBuilder.hashCode(), secondSliceBuilder.hashCode()); }
/** * Test that creates new smoothing model from a random test smoothing model and checks both for * equality */ public void testFromXContent() throws IOException { SmoothingModel testModel = createTestModel(); XContentBuilder contentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { contentBuilder.prettyPrint(); } contentBuilder.startObject(); testModel.innerToXContent(contentBuilder, ToXContent.EMPTY_PARAMS); contentBuilder.endObject(); XContentParser parser = XContentHelper.createParser(shuffleXContent(contentBuilder).bytes()); QueryParseContext context = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT); parser.nextToken(); // go to start token, real parsing would do that in the outer element parser SmoothingModel parsedModel = fromXContent(context); assertNotSame(testModel, parsedModel); assertEquals(testModel, parsedModel); assertEquals(testModel.hashCode(), parsedModel.hashCode()); }
public void testReverseOptionFailsBuildWhenInvalidGeoHashString() throws IOException { String json = "{\n" + " \"reverse\" : \"false\"\n" + "}"; XContentParser itemParser = XContentHelper.createParser(new BytesArray(json)); itemParser.nextToken(); QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT); try { GeoDistanceSortBuilder item = GeoDistanceSortBuilder.fromXContent(context, ""); item.validation(GeoValidationMethod.STRICT); item.build(createMockShardContext()); fail("adding reverse sorting option should fail with an exception"); } catch (ElasticsearchParseException e) { assertEquals( "illegal latitude value [269.384765625] for [GeoDistanceSort] for field [reverse].", e.getMessage()); } }
/** * creates random suggestion builder, renders it to xContent and back to new instance that should * be equal to original */ public void testFromXContent() throws IOException { Suggesters suggesters = new Suggesters(Collections.emptyMap()); QueryParseContext context = new QueryParseContext(null); context.parseFieldMatcher(new ParseFieldMatcher(Settings.EMPTY)); for (int runs = 0; runs < NUMBER_OF_RUNS; runs++) { SuggestBuilder suggestBuilder = createTestModel(); XContentBuilder xContentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { xContentBuilder.prettyPrint(); } suggestBuilder.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS); XContentParser parser = XContentHelper.createParser(xContentBuilder.bytes()); context.reset(parser); SuggestBuilder secondSuggestBuilder = SuggestBuilder.fromXContent(context, suggesters); assertNotSame(suggestBuilder, secondSuggestBuilder); assertEquals(suggestBuilder, secondSuggestBuilder); assertEquals(suggestBuilder.hashCode(), secondSuggestBuilder.hashCode()); } }
public void testSortModeSumIsRejectedInJSON() throws IOException { String json = "{\n" + " \"testname\" : [ {\n" + " \"lat\" : -6.046997540714173,\n" + " \"lon\" : -51.94128329747579\n" + " } ],\n" + " \"unit\" : \"m\",\n" + " \"distance_type\" : \"sloppy_arc\",\n" + " \"mode\" : \"SUM\"\n" + "}"; XContentParser itemParser = XContentHelper.createParser(new BytesArray(json)); itemParser.nextToken(); QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> GeoDistanceSortBuilder.fromXContent(context, "")); assertEquals("sort_mode [sum] isn't supported for sorting by geo distance", e.getMessage()); }
public void testReverseOptionFailsWhenNonStringField() throws IOException { String json = "{\n" + " \"testname\" : [ {\n" + " \"lat\" : -6.046997540714173,\n" + " \"lon\" : -51.94128329747579\n" + " } ],\n" + " \"reverse\" : true\n" + "}"; XContentParser itemParser = XContentHelper.createParser(new BytesArray(json)); itemParser.nextToken(); QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT); try { GeoDistanceSortBuilder.fromXContent(context, ""); fail("adding reverse sorting option should fail with an exception"); } catch (ParsingException e) { assertEquals("Only geohashes of type string supported for field [reverse]", e.getMessage()); } }
public void testIgnoreMalformedIsDeprecated() throws IOException { String json = "{\n" + " \"testname\" : [ {\n" + " \"lat\" : -6.046997540714173,\n" + " \"lon\" : -51.94128329747579\n" + " } ],\n" + " \"unit\" : \"m\",\n" + " \"distance_type\" : \"sloppy_arc\",\n" + " \"mode\" : \"SUM\",\n" + " \"ignore_malformed\" : true\n" + "}"; XContentParser itemParser = XContentHelper.createParser(new BytesArray(json)); itemParser.nextToken(); QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> GeoDistanceSortBuilder.fromXContent(context, "")); assertTrue(e.getMessage().startsWith("Deprecated field ")); }
private ParsedQuery parseQuery(ExplainRequest request, IndexService indexService) { try { XContentParser parser = XContentHelper.createParser(request.getSource()); for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) { if (token == XContentParser.Token.FIELD_NAME) { String fieldName = parser.currentName(); if ("query".equals(fieldName)) { return indexService.queryParserService().parse(parser); } else if ("query_binary".equals(fieldName)) { byte[] querySource = parser.binaryValue(); XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(querySource); return indexService.queryParserService().parse(qSourceParser); } } } } catch (Exception e) { throw new ElasticSearchException("Couldn't parse query from source.", e); } throw new ElasticSearchException("No query specified"); }
public void process( MetaData metaData, String aliasOrIndex, @Nullable MappingMetaData mappingMd, boolean allowIdGeneration) throws ElasticsearchException { // resolve the routing if needed routing(metaData.resolveIndexRouting(routing, aliasOrIndex)); // resolve timestamp if provided externally if (timestamp != null) { timestamp = MappingMetaData.Timestamp.parseStringTimestamp( timestamp, mappingMd != null ? mappingMd.timestamp().dateTimeFormatter() : TimestampFieldMapper.Defaults.DATE_TIME_FORMATTER); } // extract values if needed if (mappingMd != null) { MappingMetaData.ParseContext parseContext = mappingMd.createParseContext(id, routing, timestamp); if (parseContext.shouldParse()) { XContentParser parser = null; try { parser = XContentHelper.createParser(source); mappingMd.parse(parser, parseContext); if (parseContext.shouldParseId()) { id = parseContext.id(); } if (parseContext.shouldParseRouting()) { routing = parseContext.routing(); } if (parseContext.shouldParseTimestamp()) { timestamp = parseContext.timestamp(); timestamp = MappingMetaData.Timestamp.parseStringTimestamp( timestamp, mappingMd.timestamp().dateTimeFormatter()); } } catch (Exception e) { throw new ElasticsearchParseException( "failed to parse doc to extract routing/timestamp", e); } finally { if (parser != null) { parser.close(); } } } // might as well check for routing here if (mappingMd.routing().required() && routing == null) { throw new RoutingMissingException(index, type, id); } if (parent != null && !mappingMd.hasParentField()) { throw new ElasticsearchIllegalArgumentException( "Can't specify parent if no parent field has been configured"); } } else { if (parent != null) { throw new ElasticsearchIllegalArgumentException( "Can't specify parent if no parent field has been configured"); } } // generate id if not already provided and id generation is allowed if (allowIdGeneration) { if (id == null) { id(Strings.randomBase64UUID()); // since we generate the id, change it to CREATE opType(IndexRequest.OpType.CREATE); } } // generate timestamp if not provided, we always have one post this stage... if (timestamp == null) { timestamp = Long.toString(System.currentTimeMillis()); } }
public ParsedDocument parse(SourceToParse source, @Nullable ParseListener listener) throws MapperParsingException { ParseContext context = cache.get(); if (source.type() != null && !source.type().equals(this.type)) { throw new MapperParsingException( "Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + this.type + "]"); } source.type(this.type); XContentParser parser = source.parser(); try { if (parser == null) { parser = XContentHelper.createParser(source.source()); } context.reset(parser, new Document(), source, listener); // on a newly created instance of document mapper, we always consider it as new mappers that // have been added if (initMappersAdded) { context.setMappingsModified(); initMappersAdded = false; } // will result in START_OBJECT int countDownTokens = 0; XContentParser.Token token = parser.nextToken(); if (token != XContentParser.Token.START_OBJECT) { throw new MapperParsingException("Malformed content, must start with an object"); } boolean emptyDoc = false; token = parser.nextToken(); if (token == XContentParser.Token.END_OBJECT) { // empty doc, we can handle it... emptyDoc = true; } else if (token != XContentParser.Token.FIELD_NAME) { throw new MapperParsingException( "Malformed content, after first object, either the type field or the actual properties should exist"); } if (type.equals(parser.currentName())) { // first field is the same as the type, this might be because the type is provided, and the // object exists within it // or because there is a valid field that by chance is named as the type // Note, in this case, we only handle plain value types, an object type will be analyzed as // if it was the type itself // and other same level fields will be ignored token = parser.nextToken(); countDownTokens++; // commented out, allow for same type with START_OBJECT, we do our best to handle it except // for the above corner case // if (token != XContentParser.Token.START_OBJECT) { // throw new MapperException("Malformed content, a field with the same // name as the type must be an object with the properties/fields within it"); // } } for (RootMapper rootMapper : rootMappersOrdered) { rootMapper.preParse(context); } if (!emptyDoc) { rootObjectMapper.parse(context); } for (int i = 0; i < countDownTokens; i++) { parser.nextToken(); } // fire up any new mappers if exists if (!context.newFieldMappers().mappers.isEmpty()) { addFieldMappers(context.newFieldMappers().mappers); context.newFieldMappers().mappers.clear(); } if (!context.newObjectMappers().mappers.isEmpty()) { addObjectMappers(context.newObjectMappers().mappers); context.newObjectMappers().mappers.clear(); } for (RootMapper rootMapper : rootMappersOrdered) { rootMapper.postParse(context); } for (RootMapper rootMapper : rootMappersOrdered) { rootMapper.validate(context); } } catch (Throwable e) { // we have to fire up any new mappers even on a failure, because they // have been added internally to each compound mapper... // ... we have no option to "rollback" a change, which is very tricky in our copy on change // system... if (!context.newFieldMappers().mappers.isEmpty()) { addFieldMappers(context.newFieldMappers().mappers); context.newFieldMappers().mappers.clear(); } if (!context.newObjectMappers().mappers.isEmpty()) { addObjectMappers(context.newObjectMappers().mappers); context.newObjectMappers().mappers.clear(); } // if its already a mapper parsing exception, no need to wrap it... if (e instanceof MapperParsingException) { throw (MapperParsingException) e; } throw new MapperParsingException("failed to parse", e); } finally { // only close the parser when its not provided externally if (source.parser() == null && parser != null) { parser.close(); } } // reverse the order of docs for nested docs support, parent should be last if (context.docs().size() > 1) { Collections.reverse(context.docs()); } // apply doc boost if (context.docBoost() != 1.0f) { Set<String> encounteredFields = Sets.newHashSet(); for (Document doc : context.docs()) { encounteredFields.clear(); for (IndexableField field : doc) { if (field.fieldType().indexed() && !field.fieldType().omitNorms()) { if (!encounteredFields.contains(field.name())) { ((Field) field).setBoost(context.docBoost() * field.boost()); encounteredFields.add(field.name()); } } } } } ParsedDocument doc = new ParsedDocument( context.uid(), context.id(), context.type(), source.routing(), source.timestamp(), source.ttl(), context.docs(), context.analyzer(), context.source(), context.mappingsModified()) .parent(source.parent()); // reset the context to free up memory context.reset(null, null, null, null); return doc; }
private ParsedDocument innerParseDocument(SourceToParse source) throws MapperParsingException { if (docMapper.type().equals(MapperService.DEFAULT_MAPPING)) { throw new IllegalArgumentException( "It is forbidden to index into the default mapping [" + MapperService.DEFAULT_MAPPING + "]"); } ParseContext.InternalParseContext context = cache.get(); final Mapping mapping = docMapper.mapping(); if (source.type() != null && !source.type().equals(docMapper.type())) { throw new MapperParsingException( "Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + docMapper.type() + "]"); } source.type(docMapper.type()); XContentParser parser = source.parser(); try { if (parser == null) { parser = XContentHelper.createParser(source.source()); } context.reset(parser, new ParseContext.Document(), source); // will result in START_OBJECT XContentParser.Token token = parser.nextToken(); if (token != XContentParser.Token.START_OBJECT) { throw new MapperParsingException("Malformed content, must start with an object"); } boolean emptyDoc = false; if (mapping.root.isEnabled()) { token = parser.nextToken(); if (token == XContentParser.Token.END_OBJECT) { // empty doc, we can handle it... emptyDoc = true; } else if (token != XContentParser.Token.FIELD_NAME) { throw new MapperParsingException( "Malformed content, after first object, either the type field or the actual properties should exist"); } } for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) { metadataMapper.preParse(context); } if (mapping.root.isEnabled() == false) { // entire type is disabled parser.skipChildren(); } else if (emptyDoc == false) { Mapper update = parseObject(context, mapping.root, true); if (update != null) { context.addDynamicMappingsUpdate(update); } } for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) { metadataMapper.postParse(context); } // try to parse the next token, this should be null if the object is ended properly // but will throw a JSON exception if the extra tokens is not valid JSON (this will be handled // by the catch) if (Version.indexCreated(indexSettings).onOrAfter(Version.V_2_0_0_beta1) && source.parser() == null && parser != null) { // only check for end of tokens if we created the parser here token = parser.nextToken(); if (token != null) { throw new IllegalArgumentException( "Malformed content, found extra data after parsing: " + token); } } } catch (Throwable e) { // if its already a mapper parsing exception, no need to wrap it... if (e instanceof MapperParsingException) { throw (MapperParsingException) e; } // Throw a more meaningful message if the document is empty. if (source.source() != null && source.source().length() == 0) { throw new MapperParsingException("failed to parse, document is empty"); } throw new MapperParsingException("failed to parse", e); } finally { // only close the parser when its not provided externally if (source.parser() == null && parser != null) { parser.close(); } } // reverse the order of docs for nested docs support, parent should be last if (context.docs().size() > 1) { Collections.reverse(context.docs()); } // apply doc boost if (context.docBoost() != 1.0f) { Set<String> encounteredFields = new HashSet<>(); for (ParseContext.Document doc : context.docs()) { encounteredFields.clear(); for (IndexableField field : doc) { if (field.fieldType().indexOptions() != IndexOptions.NONE && !field.fieldType().omitNorms()) { if (!encounteredFields.contains(field.name())) { ((Field) field).setBoost(context.docBoost() * field.boost()); encounteredFields.add(field.name()); } } } } } Mapper rootDynamicUpdate = context.dynamicMappingsUpdate(); Mapping update = null; if (rootDynamicUpdate != null) { update = mapping.mappingUpdate(rootDynamicUpdate); } ParsedDocument doc = new ParsedDocument( context.uid(), context.version(), context.id(), context.type(), source.routing(), source.timestamp(), source.ttl(), context.docs(), context.source(), update) .parent(source.parent()); // reset the context to free up memory context.reset(null, null, null); return doc; }
private void pre019Upgrade() throws Exception { long index = -1; File metaDataFile = null; MetaData metaData = null; long version = -1; for (File dataLocation : nodeEnv.nodeDataLocations()) { File stateLocation = new File(dataLocation, "_state"); if (!stateLocation.exists()) { continue; } File[] stateFiles = stateLocation.listFiles(); if (stateFiles == null) { continue; } for (File stateFile : stateFiles) { if (logger.isTraceEnabled()) { logger.trace("[upgrade]: processing [" + stateFile.getName() + "]"); } String name = stateFile.getName(); if (!name.startsWith("metadata-")) { continue; } long fileIndex = Long.parseLong(name.substring(name.indexOf('-') + 1)); if (fileIndex >= index) { // try and read the meta data try { byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); if (data.length == 0) { continue; } XContentParser parser = XContentHelper.createParser(data, 0, data.length); try { String currentFieldName = null; XContentParser.Token token = parser.nextToken(); if (token != null) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { if ("meta-data".equals(currentFieldName)) { metaData = MetaData.Builder.fromXContent(parser); } } else if (token.isValue()) { if ("version".equals(currentFieldName)) { version = parser.longValue(); } } } } } finally { parser.close(); } index = fileIndex; metaDataFile = stateFile; } catch (IOException e) { logger.warn("failed to read pre 0.19 state from [" + name + "], ignoring...", e); } } } } if (metaData == null) { return; } logger.info( "found old metadata state, loading metadata from [{}] and converting to new metadata location and strucutre...", metaDataFile.getAbsolutePath()); writeGlobalState( "upgrade", MetaData.builder().metaData(metaData).version(version).build(), null); for (IndexMetaData indexMetaData : metaData) { IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.newIndexMetaDataBuilder(indexMetaData).version(version); // set the created version to 0.18 indexMetaDataBuilder.settings( ImmutableSettings.settingsBuilder() .put(indexMetaData.settings()) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_0_18_0)); writeIndex("upgrade", indexMetaDataBuilder.build(), null); } // rename shards state to backup state File backupFile = new File(metaDataFile.getParentFile(), "backup-" + metaDataFile.getName()); if (!metaDataFile.renameTo(backupFile)) { throw new IOException( "failed to rename old state to backup state [" + metaDataFile.getAbsolutePath() + "]"); } // delete all other shards state files for (File dataLocation : nodeEnv.nodeDataLocations()) { File stateLocation = new File(dataLocation, "_state"); if (!stateLocation.exists()) { continue; } File[] stateFiles = stateLocation.listFiles(); if (stateFiles == null) { continue; } for (File stateFile : stateFiles) { String name = stateFile.getName(); if (!name.startsWith("metadata-")) { continue; } stateFile.delete(); } } logger.info( "conversion to new metadata location and format done, backup create at [{}]", backupFile.getAbsolutePath()); }