@Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { ValueAndBoost valueAndBoost = parseCreateFieldForString(context, fieldType().nullValueAsString(), fieldType().boost()); if (valueAndBoost.value() == null) { return; } if (ignoreAbove > 0 && valueAndBoost.value().length() > ignoreAbove) { return; } if (context.includeInAll(includeInAll, this)) { context .allEntries() .addText(fieldType().name(), valueAndBoost.value(), valueAndBoost.boost()); } if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { Field field = new Field(fieldType().name(), valueAndBoost.value(), fieldType()); field.setBoost(valueAndBoost.boost()); fields.add(field); } if (fieldType().hasDocValues()) { fields.add( new SortedSetDocValuesField(fieldType().name(), new BytesRef(valueAndBoost.value()))); } }
/** * Parse a field as though it were a string. * * @param context parse context used during parsing * @param nullValue value to use for null * @param defaultBoost default boost value returned unless overwritten in the field * @return the parsed field and the boost either parsed or defaulted * @throws IOException if thrown while parsing */ public static ValueAndBoost parseCreateFieldForString( ParseContext context, String nullValue, float defaultBoost) throws IOException { if (context.externalValueSet()) { return new ValueAndBoost(context.externalValue().toString(), defaultBoost); } XContentParser parser = context.parser(); if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { return new ValueAndBoost(nullValue, defaultBoost); } if (parser.currentToken() == XContentParser.Token.START_OBJECT && Version.indexCreated(context.indexSettings()).before(Version.V_3_0_0)) { XContentParser.Token token; String currentFieldName = null; String value = nullValue; float boost = defaultBoost; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { value = parser.textOrNull(); } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); } else { throw new IllegalArgumentException("unknown property [" + currentFieldName + "]"); } } } return new ValueAndBoost(value, boost); } return new ValueAndBoost(parser.textOrNull(), defaultBoost); }
@Override public void postParse(ParseContext context) throws IOException { if (context.id() == null && !context.sourceToParse().flyweight()) { throw new MapperParsingException("No id found while parsing the content source"); } // it either get built in the preParse phase, or get parsed... }
@Override public void validate(ParseContext context) throws MapperParsingException { String routing = context.sourceToParse().routing(); if (path != null && routing != null) { // we have a path, check if we can validate we have the same routing value as the one in the // doc... String value = null; Fieldable field = context.doc().getFieldable(path); if (field != null) { value = field.stringValue(); if (value == null) { // maybe its a numeric field... if (field instanceof NumberFieldMapper.CustomNumericField) { value = ((NumberFieldMapper.CustomNumericField) field).numericAsString(); } } } if (value == null) { value = context.ignoredValue(path); } if (value == null) { // maybe its a numeric field } if (!routing.equals(value)) { throw new MapperParsingException( "External routing [" + routing + "] and document path routing [" + value + "] mismatch"); } } }
@Override public void preParse(ParseContext context) throws IOException { if (context.sourceToParse().id() != null) { context.id(context.sourceToParse().id()); super.parse(context); } }
@Override public Mapper parse(ParseContext context) throws IOException { QueryShardContext queryShardContext = new QueryShardContext(this.queryShardContext); if (context.doc().getField(queryBuilderField.name()) != null) { // If a percolator query has been defined in an array object then multiple percolator queries // could be provided. In order to prevent this we fail if we try to parse more than one query // for the current document. throw new IllegalArgumentException("a document can only contain one percolator query"); } XContentParser parser = context.parser(); QueryBuilder queryBuilder = parseQueryBuilder(queryShardContext.newParseContext(parser), parser.getTokenLocation()); verifyQuery(queryBuilder); // Fetching of terms, shapes and indexed scripts happen during this rewrite: queryBuilder = queryBuilder.rewrite(queryShardContext); try (XContentBuilder builder = XContentFactory.contentBuilder(QUERY_BUILDER_CONTENT_TYPE)) { queryBuilder.toXContent(builder, new MapParams(Collections.emptyMap())); builder.flush(); byte[] queryBuilderAsBytes = BytesReference.toBytes(builder.bytes()); context .doc() .add( new Field( queryBuilderField.name(), queryBuilderAsBytes, queryBuilderField.fieldType())); } Query query = toQuery(queryShardContext, mapUnmappedFieldAsString, queryBuilder); processQuery(query, context); return null; }
@Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { if (!fieldType().stored()) { return; } byte[] value; if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { return; } else { value = context.parser().binaryValue(); if (compress != null && compress && !CompressorFactory.isCompressed(value, 0, value.length)) { if (compressThreshold == -1 || value.length > compressThreshold) { BytesStreamOutput bStream = new BytesStreamOutput(); StreamOutput stream = CompressorFactory.defaultCompressor().streamOutput(bStream); stream.writeBytes(value, 0, value.length); stream.close(); value = bStream.bytes().toBytes(); } } } if (value == null) { return; } fields.add(new Field(names.indexName(), value, fieldType)); }
@Override protected Field parseCreateField(ParseContext context) throws IOException { if (context.parser().currentName() != null && context.parser().currentName().equals(Defaults.NAME)) { // we are in the parsing of _parent phase String parentId = context.parser().text(); context.sourceToParse().parent(parentId); return new Field( names.indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType); } // otherwise, we are running it post processing of the xcontent String parsedParentId = context.doc().get(Defaults.NAME); if (context.sourceToParse().parent() != null) { String parentId = context.sourceToParse().parent(); if (parsedParentId == null) { if (parentId == null) { throw new MapperParsingException( "No parent id provided, not within the document, and not externally"); } // we did not add it in the parsing phase, add it now return new Field( names.indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType); } else if (parentId != null && !parsedParentId.equals(Uid.createUid(context.stringBuilder(), type, parentId))) { throw new MapperParsingException( "Parent id mismatch, document value is [" + Uid.createUid(parsedParentId).id() + "], while external value is [" + parentId + "]"); } } // we have parent mapping, yet no value was set, ignore it... return null; }
@Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { if (fieldType().indexOptions() == IndexOptions.NONE && !fieldType().stored() && !fieldType().hasDocValues()) { return; } Boolean value = context.parseExternalValue(Boolean.class); if (value == null) { XContentParser.Token token = context.parser().currentToken(); if (token == XContentParser.Token.VALUE_NULL) { if (fieldType().nullValue() != null) { value = fieldType().nullValue(); } } else { value = context.parser().booleanValue(); } } if (value == null) { return; } fields.add(new Field(fieldType().names().indexName(), value ? "T" : "F", fieldType())); if (fieldType().hasDocValues()) { fields.add(new SortedNumericDocValuesField(fieldType().names().indexName(), value ? 1 : 0)); } }
private void addFieldData( ParseContext context, FieldMapper<String> mapper, Collection<String> data) throws IOException { if (data != null && !data.isEmpty()) { if (mappers.get(mapper.names().indexName()) == null) { // New mapper context.setWithinNewMapper(); try { parseData(context, mapper, data); FieldMapperListener.Aggregator newFields = new FieldMapperListener.Aggregator(); ObjectMapperListener.Aggregator newObjects = new ObjectMapperListener.Aggregator(); mapper.traverse(newFields); mapper.traverse(newObjects); // callback on adding those fields! context.docMapper().addFieldMappers(newFields.mappers); context.docMapper().addObjectMappers(newObjects.mappers); context.setMappingsModified(); synchronized (mutex) { UpdateInPlaceMap<String, FieldMapper<String>>.Mutator mappingMutator = this.mappers.mutator(); mappingMutator.put(mapper.names().indexName(), mapper); mappingMutator.close(); } } finally { context.clearWithinNewMapper(); } } else { // Mapper already added parseData(context, mapper, data); } } }
@Override public void preParse(ParseContext context) throws IOException { // if we have the id provided, fill it, and parse now if (context.sourceToParse().id() != null) { context.id(context.sourceToParse().id()); super.parse(context); } }
@Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { if (!fieldType.indexed() && !fieldType.stored()) { return; } fields.add(new Field(names.indexName(), context.type(), fieldType)); if (hasDocValues()) { fields.add(new SortedSetDocValuesField(names.indexName(), new BytesRef(context.type()))); } }
@Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { if (context.sourceToParse().token() != null) { Long token = context.sourceToParse().token(); if (token != null) { fields.add(new LongFieldMapper.CustomLongNumericField(token, fieldType())); fields.add(new SortedNumericDocValuesField(fieldType().names().indexName(), token)); } } }
@Override protected Field parseCreateField(ParseContext context) throws IOException { // so, caching uid stream and field is fine // since we don't do any mapping parsing without immediate indexing // and, when percolating, we don't index the uid UidField field = fieldCache.get(); field.setUid(Uid.createUid(context.stringBuilder(), context.type(), context.id())); context.uid(field); return field; // version get updated by the engine }
@Override protected Fieldable parseCreateField(ParseContext context) throws IOException { if (!enabled) { return null; } // reset the entries context.allEntries().reset(); Analyzer analyzer = findAnalyzer(context); return new AllField(names.indexName(), store, termVector, context.allEntries(), analyzer); }
@Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { if (!enabledState.enabled) { return; } if (context.flyweight()) { return; } fields.add( new IntegerFieldMapper.CustomIntegerNumericField(context.source().length(), fieldType())); }
private void parse(ParseContext context, GeoPoint point, String geohash) throws IOException { if (fieldType().ignoreMalformed == false) { if (point.lat() > 90.0 || point.lat() < -90.0) { throw new IllegalArgumentException( "illegal latitude value [" + point.lat() + "] for " + name()); } if (point.lon() > 180.0 || point.lon() < -180) { throw new IllegalArgumentException( "illegal longitude value [" + point.lon() + "] for " + name()); } } if (fieldType().coerce) { // by setting coerce to false we are assuming all geopoints are already in a valid coordinate // system // thus this extra step can be skipped // LUCENE WATCH: This will be folded back into Lucene's GeoPointField GeoUtils.normalizePoint(point, true, true); } if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { Field field = new Field( fieldType().names().indexName(), Double.toString(point.lat()) + ',' + Double.toString(point.lon()), fieldType()); context.doc().add(field); } if (fieldType().isGeohashEnabled()) { if (geohash == null) { geohash = GeoHashUtils.encode(point.lat(), point.lon()); } addGeohashField(context, geohash); } if (fieldType().isLatLonEnabled()) { latMapper.parse(context.createExternalValueContext(point.lat())); lonMapper.parse(context.createExternalValueContext(point.lon())); } if (fieldType().hasDocValues()) { CustomGeoPointDocValuesField field = (CustomGeoPointDocValuesField) context.doc().getByKey(fieldType().names().indexName()); if (field == null) { field = new CustomGeoPointDocValuesField( fieldType().names().indexName(), point.lat(), point.lon()); context.doc().addWithKey(fieldType().names().indexName(), field); } else { field.add(point.lat(), point.lon()); } } multiFields.parse(this, context); }
@Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { Field uid = new Field( NAME, Uid.createUid(context.stringBuilder(), context.type(), context.id()), Defaults.FIELD_TYPE); context.uid(uid); fields.add(uid); if (fieldType().hasDocValues()) { fields.add(new BinaryDocValuesField(NAME, new BytesRef(uid.stringValue()))); } }
@Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { if (context.sourceToParse().routing() != null) { String routing = context.sourceToParse().routing(); if (routing != null) { if (fieldType.indexOptions() == IndexOptions.NONE && !fieldType.stored()) { context.ignoredValue(fieldType.names().indexName(), routing); return; } fields.add(new Field(fieldType.names().indexName(), routing, fieldType)); } } }
private void serializeObject(final ParseContext context, String currentFieldName) throws IOException { if (currentFieldName == null) { throw new MapperParsingException( "object mapping [" + name + "] trying to serialize an object with no field associated with it, current value [" + context.parser().textOrNull() + "]"); } context.path().add(currentFieldName); Mapper objectMapper = mappers.get(currentFieldName); if (objectMapper != null) { objectMapper.parse(context); } else { Dynamic dynamic = this.dynamic; if (dynamic == null) { dynamic = context.root().dynamic(); } if (dynamic == Dynamic.STRICT) { throw new StrictDynamicMappingException(fullPath, currentFieldName); } else if (dynamic == Dynamic.TRUE) { // we sync here just so we won't add it twice. Its not the end of the world // to sync here since next operations will get it before synchronized (mutex) { objectMapper = mappers.get(currentFieldName); if (objectMapper == null) { // remove the current field name from path, since template search and the object builder // add it as well... context.path().remove(); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "object"); if (builder == null) { builder = MapperBuilders.object(currentFieldName).enabled(true).pathType(pathType); // if this is a non root object, then explicitly set the dynamic behavior if set if (!(this instanceof RootObjectMapper) && this.dynamic != Defaults.DYNAMIC) { ((Builder) builder).dynamic(this.dynamic); } } BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); objectMapper = builder.build(builderContext); putDynamicMapper(context, currentFieldName, objectMapper); } else { objectMapper.parse(context); } } } else { // not dynamic, read everything up to end object context.parser().skipChildren(); } } context.path().remove(); }
private void serializeArray(ParseContext context, String lastFieldName) throws IOException { String arrayFieldName = lastFieldName; Mapper mapper = mappers.get(lastFieldName); if (mapper != null) { // There is a concrete mapper for this field already. Need to check if the mapper // expects an array, if so we pass the context straight to the mapper and if not // we serialize the array components if (mapper instanceof ArrayValueMapperParser) { mapper.parse(context); } else { serializeNonDynamicArray(context, lastFieldName, arrayFieldName); } } else { Dynamic dynamic = this.dynamic; if (dynamic == null) { dynamic = context.root().dynamic(); } if (dynamic == Dynamic.STRICT) { throw new StrictDynamicMappingException(fullPath, arrayFieldName); } else if (dynamic == Dynamic.TRUE) { // we sync here just so we won't add it twice. Its not the end of the world // to sync here since next operations will get it before synchronized (mutex) { mapper = mappers.get(arrayFieldName); if (mapper == null) { Mapper.Builder builder = context.root().findTemplateBuilder(context, arrayFieldName, "object"); if (builder == null) { serializeNonDynamicArray(context, lastFieldName, arrayFieldName); return; } BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); mapper = builder.build(builderContext); if (mapper != null && mapper instanceof ArrayValueMapperParser) { putDynamicMapper(context, arrayFieldName, mapper); } else { serializeNonDynamicArray(context, lastFieldName, arrayFieldName); } } else { serializeNonDynamicArray(context, lastFieldName, arrayFieldName); } } } else { serializeNonDynamicArray(context, lastFieldName, arrayFieldName); } } }
private Analyzer findAnalyzer(ParseContext context) { Analyzer analyzer = indexAnalyzer; if (analyzer == null) { analyzer = context.analyzer(); if (analyzer == null) { analyzer = context.docMapper().indexAnalyzer(); if (analyzer == null) { // This should not happen, should we log warn it? analyzer = Lucene.STANDARD_ANALYZER; } } } return analyzer; }
@Override protected Field parseCreateField(ParseContext context) throws IOException { if (context.sourceToParse().routing() != null) { String routing = context.sourceToParse().routing(); if (routing != null) { if (!indexed() && !stored()) { context.ignoredValue(names.indexName(), routing); return null; } return new Field(names.indexName(), routing, store, index); } } return null; }
@Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { Object addressAsObject; if (context.externalValueSet()) { addressAsObject = context.externalValue(); } else { addressAsObject = context.parser().textOrNull(); } if (addressAsObject == null) { addressAsObject = fieldType().nullValue(); } if (addressAsObject == null) { return; } String addressAsString = addressAsObject.toString(); InetAddress address; if (addressAsObject instanceof InetAddress) { address = (InetAddress) addressAsObject; } else { try { address = InetAddresses.forString(addressAsString); } catch (IllegalArgumentException e) { if (ignoreMalformed.value()) { return; } else { throw e; } } } if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(fieldType().name(), addressAsString, fieldType().boost()); } if (fieldType().indexOptions() != IndexOptions.NONE) { fields.add(new InetAddressPoint(fieldType().name(), address)); } if (fieldType().hasDocValues()) { fields.add( new SortedSetDocValuesField( fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); } if (fieldType().stored()) { fields.add( new StoredField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); } }
public Mapper.Builder findTemplateBuilder( ParseContext context, String name, String dynamicType, String matchType) { DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType); if (dynamicTemplate == null) { return null; } Mapper.TypeParser.ParserContext parserContext = context.docMapperParser().parserContext(); String mappingType = dynamicTemplate.mappingType(dynamicType); Mapper.TypeParser typeParser = parserContext.typeParser(mappingType); if (typeParser == null) { throw new MapperParsingException( "failed to find type parsed [" + mappingType + "] for [" + name + "]"); } return typeParser.parse(name, dynamicTemplate.mappingForName(name, dynamicType), parserContext); }
@Override protected Field parseCreateField(ParseContext context) throws IOException { if (!enabled) { return null; } return new Field(names.indexName(), context.index(), fieldType); }
@Override protected Fieldable innerParseCreateField(ParseContext context) throws IOException { if (!enabled) { return null; } return new CustomIntegerNumericField(this, context.source().length()); }
void processQuery(Query query, ParseContext context) { ParseContext.Document doc = context.doc(); FieldType pft = (FieldType) this.fieldType(); QueryAnalyzer.Result result; try { result = QueryAnalyzer.analyze(query); } catch (QueryAnalyzer.UnsupportedQueryException e) { doc.add( new Field( pft.extractionResultField.name(), EXTRACTION_FAILED, extractionResultField.fieldType())); return; } for (Term term : result.terms) { BytesRefBuilder builder = new BytesRefBuilder(); builder.append(new BytesRef(term.field())); builder.append(FIELD_VALUE_SEPARATOR); builder.append(term.bytes()); doc.add(new Field(queryTermsField.name(), builder.toBytesRef(), queryTermsField.fieldType())); } if (result.verified) { doc.add( new Field( extractionResultField.name(), EXTRACTION_COMPLETE, extractionResultField.fieldType())); } else { doc.add( new Field( extractionResultField.name(), EXTRACTION_PARTIAL, extractionResultField.fieldType())); } }
private void serializeArray(ParseContext context, String lastFieldName) throws IOException { String arrayFieldName = lastFieldName; Mapper mapper = mappers.get(lastFieldName); if (mapper != null && mapper instanceof ArrayValueMapperParser) { mapper.parse(context); } else { XContentParser parser = context.parser(); XContentParser.Token token; while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.START_OBJECT) { serializeObject(context, lastFieldName); } else if (token == XContentParser.Token.START_ARRAY) { serializeArray(context, lastFieldName); } else if (token == XContentParser.Token.FIELD_NAME) { lastFieldName = parser.currentName(); } else if (token == XContentParser.Token.VALUE_NULL) { serializeNullValue(context, lastFieldName); } else if (token == null) { throw new MapperParsingException( "object mapping for [" + name + "] with array for [" + arrayFieldName + "] tried to parse as array, but got EOF, is there a mismatch in types for the same field?"); } else { serializeValue(context, lastFieldName, token); } } } }
@Override public Mapper parse(ParseContext context) throws IOException, MapperParsingException { if (context.sourceToParse().ttl() < 0) { // no ttl has been provided externally long ttl; if (context.parser().currentToken() == XContentParser.Token.VALUE_STRING) { ttl = TimeValue.parseTimeValue(context.parser().text(), null, "ttl").millis(); } else { ttl = context.parser().longValue(coerce.value()); } if (ttl <= 0) { throw new MapperParsingException( "TTL value must be > 0. Illegal value provided [" + ttl + "]"); } context.sourceToParse().ttl(ttl); } return null; }