@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
ValueAndBoost valueAndBoost =
parseCreateFieldForString(context, fieldType().nullValueAsString(), fieldType().boost());
if (valueAndBoost.value() == null) {
return;
}
if (ignoreAbove > 0 && valueAndBoost.value().length() > ignoreAbove) {
return;
}
if (context.includeInAll(includeInAll, this)) {
context
.allEntries()
.addText(fieldType().name(), valueAndBoost.value(), valueAndBoost.boost());
}
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
Field field = new Field(fieldType().name(), valueAndBoost.value(), fieldType());
field.setBoost(valueAndBoost.boost());
fields.add(field);
}
if (fieldType().hasDocValues()) {
fields.add(
new SortedSetDocValuesField(fieldType().name(), new BytesRef(valueAndBoost.value())));
}
}
/**
* Parse a field as though it were a string.
*
* @param context parse context used during parsing
* @param nullValue value to use for null
* @param defaultBoost default boost value returned unless overwritten in the field
* @return the parsed field and the boost either parsed or defaulted
* @throws IOException if thrown while parsing
*/
public static ValueAndBoost parseCreateFieldForString(
ParseContext context, String nullValue, float defaultBoost) throws IOException {
if (context.externalValueSet()) {
return new ValueAndBoost(context.externalValue().toString(), defaultBoost);
}
XContentParser parser = context.parser();
if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
return new ValueAndBoost(nullValue, defaultBoost);
}
if (parser.currentToken() == XContentParser.Token.START_OBJECT
&& Version.indexCreated(context.indexSettings()).before(Version.V_3_0_0)) {
XContentParser.Token token;
String currentFieldName = null;
String value = nullValue;
float boost = defaultBoost;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
value = parser.textOrNull();
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
} else {
throw new IllegalArgumentException("unknown property [" + currentFieldName + "]");
}
}
}
return new ValueAndBoost(value, boost);
}
return new ValueAndBoost(parser.textOrNull(), defaultBoost);
}
@Override
public void postParse(ParseContext context) throws IOException {
if (context.id() == null && !context.sourceToParse().flyweight()) {
throw new MapperParsingException("No id found while parsing the content source");
}
// it either get built in the preParse phase, or get parsed...
}
@Override
public void validate(ParseContext context) throws MapperParsingException {
String routing = context.sourceToParse().routing();
if (path != null && routing != null) {
// we have a path, check if we can validate we have the same routing value as the one in the
// doc...
String value = null;
Fieldable field = context.doc().getFieldable(path);
if (field != null) {
value = field.stringValue();
if (value == null) {
// maybe its a numeric field...
if (field instanceof NumberFieldMapper.CustomNumericField) {
value = ((NumberFieldMapper.CustomNumericField) field).numericAsString();
}
}
}
if (value == null) {
value = context.ignoredValue(path);
}
if (value == null) {
// maybe its a numeric field
}
if (!routing.equals(value)) {
throw new MapperParsingException(
"External routing ["
+ routing
+ "] and document path routing ["
+ value
+ "] mismatch");
}
}
}
@Override
public void preParse(ParseContext context) throws IOException {
if (context.sourceToParse().id() != null) {
context.id(context.sourceToParse().id());
super.parse(context);
}
}
@Override
public Mapper parse(ParseContext context) throws IOException {
QueryShardContext queryShardContext = new QueryShardContext(this.queryShardContext);
if (context.doc().getField(queryBuilderField.name()) != null) {
// If a percolator query has been defined in an array object then multiple percolator queries
// could be provided. In order to prevent this we fail if we try to parse more than one query
// for the current document.
throw new IllegalArgumentException("a document can only contain one percolator query");
}
XContentParser parser = context.parser();
QueryBuilder queryBuilder =
parseQueryBuilder(queryShardContext.newParseContext(parser), parser.getTokenLocation());
verifyQuery(queryBuilder);
// Fetching of terms, shapes and indexed scripts happen during this rewrite:
queryBuilder = queryBuilder.rewrite(queryShardContext);
try (XContentBuilder builder = XContentFactory.contentBuilder(QUERY_BUILDER_CONTENT_TYPE)) {
queryBuilder.toXContent(builder, new MapParams(Collections.emptyMap()));
builder.flush();
byte[] queryBuilderAsBytes = BytesReference.toBytes(builder.bytes());
context
.doc()
.add(
new Field(
queryBuilderField.name(), queryBuilderAsBytes, queryBuilderField.fieldType()));
}
Query query = toQuery(queryShardContext, mapUnmappedFieldAsString, queryBuilder);
processQuery(query, context);
return null;
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
if (!fieldType().stored()) {
return;
}
byte[] value;
if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) {
return;
} else {
value = context.parser().binaryValue();
if (compress != null && compress && !CompressorFactory.isCompressed(value, 0, value.length)) {
if (compressThreshold == -1 || value.length > compressThreshold) {
BytesStreamOutput bStream = new BytesStreamOutput();
StreamOutput stream = CompressorFactory.defaultCompressor().streamOutput(bStream);
stream.writeBytes(value, 0, value.length);
stream.close();
value = bStream.bytes().toBytes();
}
}
}
if (value == null) {
return;
}
fields.add(new Field(names.indexName(), value, fieldType));
}
@Override
protected Field parseCreateField(ParseContext context) throws IOException {
if (context.parser().currentName() != null
&& context.parser().currentName().equals(Defaults.NAME)) {
// we are in the parsing of _parent phase
String parentId = context.parser().text();
context.sourceToParse().parent(parentId);
return new Field(
names.indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType);
}
// otherwise, we are running it post processing of the xcontent
String parsedParentId = context.doc().get(Defaults.NAME);
if (context.sourceToParse().parent() != null) {
String parentId = context.sourceToParse().parent();
if (parsedParentId == null) {
if (parentId == null) {
throw new MapperParsingException(
"No parent id provided, not within the document, and not externally");
}
// we did not add it in the parsing phase, add it now
return new Field(
names.indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType);
} else if (parentId != null
&& !parsedParentId.equals(Uid.createUid(context.stringBuilder(), type, parentId))) {
throw new MapperParsingException(
"Parent id mismatch, document value is ["
+ Uid.createUid(parsedParentId).id()
+ "], while external value is ["
+ parentId
+ "]");
}
}
// we have parent mapping, yet no value was set, ignore it...
return null;
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
if (fieldType().indexOptions() == IndexOptions.NONE
&& !fieldType().stored()
&& !fieldType().hasDocValues()) {
return;
}
Boolean value = context.parseExternalValue(Boolean.class);
if (value == null) {
XContentParser.Token token = context.parser().currentToken();
if (token == XContentParser.Token.VALUE_NULL) {
if (fieldType().nullValue() != null) {
value = fieldType().nullValue();
}
} else {
value = context.parser().booleanValue();
}
}
if (value == null) {
return;
}
fields.add(new Field(fieldType().names().indexName(), value ? "T" : "F", fieldType()));
if (fieldType().hasDocValues()) {
fields.add(new SortedNumericDocValuesField(fieldType().names().indexName(), value ? 1 : 0));
}
}
private void addFieldData(
ParseContext context, FieldMapper<String> mapper, Collection<String> data)
throws IOException {
if (data != null && !data.isEmpty()) {
if (mappers.get(mapper.names().indexName()) == null) {
// New mapper
context.setWithinNewMapper();
try {
parseData(context, mapper, data);
FieldMapperListener.Aggregator newFields = new FieldMapperListener.Aggregator();
ObjectMapperListener.Aggregator newObjects = new ObjectMapperListener.Aggregator();
mapper.traverse(newFields);
mapper.traverse(newObjects);
// callback on adding those fields!
context.docMapper().addFieldMappers(newFields.mappers);
context.docMapper().addObjectMappers(newObjects.mappers);
context.setMappingsModified();
synchronized (mutex) {
UpdateInPlaceMap<String, FieldMapper<String>>.Mutator mappingMutator =
this.mappers.mutator();
mappingMutator.put(mapper.names().indexName(), mapper);
mappingMutator.close();
}
} finally {
context.clearWithinNewMapper();
}
} else {
// Mapper already added
parseData(context, mapper, data);
}
}
}
@Override
public void preParse(ParseContext context) throws IOException {
// if we have the id provided, fill it, and parse now
if (context.sourceToParse().id() != null) {
context.id(context.sourceToParse().id());
super.parse(context);
}
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
if (!fieldType.indexed() && !fieldType.stored()) {
return;
}
fields.add(new Field(names.indexName(), context.type(), fieldType));
if (hasDocValues()) {
fields.add(new SortedSetDocValuesField(names.indexName(), new BytesRef(context.type())));
}
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
if (context.sourceToParse().token() != null) {
Long token = context.sourceToParse().token();
if (token != null) {
fields.add(new LongFieldMapper.CustomLongNumericField(token, fieldType()));
fields.add(new SortedNumericDocValuesField(fieldType().names().indexName(), token));
}
}
}
@Override
protected Field parseCreateField(ParseContext context) throws IOException {
// so, caching uid stream and field is fine
// since we don't do any mapping parsing without immediate indexing
// and, when percolating, we don't index the uid
UidField field = fieldCache.get();
field.setUid(Uid.createUid(context.stringBuilder(), context.type(), context.id()));
context.uid(field);
return field; // version get updated by the engine
}
@Override
protected Fieldable parseCreateField(ParseContext context) throws IOException {
if (!enabled) {
return null;
}
// reset the entries
context.allEntries().reset();
Analyzer analyzer = findAnalyzer(context);
return new AllField(names.indexName(), store, termVector, context.allEntries(), analyzer);
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
if (!enabledState.enabled) {
return;
}
if (context.flyweight()) {
return;
}
fields.add(
new IntegerFieldMapper.CustomIntegerNumericField(context.source().length(), fieldType()));
}
private void parse(ParseContext context, GeoPoint point, String geohash) throws IOException {
if (fieldType().ignoreMalformed == false) {
if (point.lat() > 90.0 || point.lat() < -90.0) {
throw new IllegalArgumentException(
"illegal latitude value [" + point.lat() + "] for " + name());
}
if (point.lon() > 180.0 || point.lon() < -180) {
throw new IllegalArgumentException(
"illegal longitude value [" + point.lon() + "] for " + name());
}
}
if (fieldType().coerce) {
// by setting coerce to false we are assuming all geopoints are already in a valid coordinate
// system
// thus this extra step can be skipped
// LUCENE WATCH: This will be folded back into Lucene's GeoPointField
GeoUtils.normalizePoint(point, true, true);
}
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
Field field =
new Field(
fieldType().names().indexName(),
Double.toString(point.lat()) + ',' + Double.toString(point.lon()),
fieldType());
context.doc().add(field);
}
if (fieldType().isGeohashEnabled()) {
if (geohash == null) {
geohash = GeoHashUtils.encode(point.lat(), point.lon());
}
addGeohashField(context, geohash);
}
if (fieldType().isLatLonEnabled()) {
latMapper.parse(context.createExternalValueContext(point.lat()));
lonMapper.parse(context.createExternalValueContext(point.lon()));
}
if (fieldType().hasDocValues()) {
CustomGeoPointDocValuesField field =
(CustomGeoPointDocValuesField) context.doc().getByKey(fieldType().names().indexName());
if (field == null) {
field =
new CustomGeoPointDocValuesField(
fieldType().names().indexName(), point.lat(), point.lon());
context.doc().addWithKey(fieldType().names().indexName(), field);
} else {
field.add(point.lat(), point.lon());
}
}
multiFields.parse(this, context);
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
Field uid =
new Field(
NAME,
Uid.createUid(context.stringBuilder(), context.type(), context.id()),
Defaults.FIELD_TYPE);
context.uid(uid);
fields.add(uid);
if (fieldType().hasDocValues()) {
fields.add(new BinaryDocValuesField(NAME, new BytesRef(uid.stringValue())));
}
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
if (context.sourceToParse().routing() != null) {
String routing = context.sourceToParse().routing();
if (routing != null) {
if (fieldType.indexOptions() == IndexOptions.NONE && !fieldType.stored()) {
context.ignoredValue(fieldType.names().indexName(), routing);
return;
}
fields.add(new Field(fieldType.names().indexName(), routing, fieldType));
}
}
}
private void serializeObject(final ParseContext context, String currentFieldName)
throws IOException {
if (currentFieldName == null) {
throw new MapperParsingException(
"object mapping ["
+ name
+ "] trying to serialize an object with no field associated with it, current value ["
+ context.parser().textOrNull()
+ "]");
}
context.path().add(currentFieldName);
Mapper objectMapper = mappers.get(currentFieldName);
if (objectMapper != null) {
objectMapper.parse(context);
} else {
Dynamic dynamic = this.dynamic;
if (dynamic == null) {
dynamic = context.root().dynamic();
}
if (dynamic == Dynamic.STRICT) {
throw new StrictDynamicMappingException(fullPath, currentFieldName);
} else if (dynamic == Dynamic.TRUE) {
// we sync here just so we won't add it twice. Its not the end of the world
// to sync here since next operations will get it before
synchronized (mutex) {
objectMapper = mappers.get(currentFieldName);
if (objectMapper == null) {
// remove the current field name from path, since template search and the object builder
// add it as well...
context.path().remove();
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "object");
if (builder == null) {
builder = MapperBuilders.object(currentFieldName).enabled(true).pathType(pathType);
// if this is a non root object, then explicitly set the dynamic behavior if set
if (!(this instanceof RootObjectMapper) && this.dynamic != Defaults.DYNAMIC) {
((Builder) builder).dynamic(this.dynamic);
}
}
BuilderContext builderContext =
new BuilderContext(context.indexSettings(), context.path());
objectMapper = builder.build(builderContext);
putDynamicMapper(context, currentFieldName, objectMapper);
} else {
objectMapper.parse(context);
}
}
} else {
// not dynamic, read everything up to end object
context.parser().skipChildren();
}
}
context.path().remove();
}
private void serializeArray(ParseContext context, String lastFieldName) throws IOException {
String arrayFieldName = lastFieldName;
Mapper mapper = mappers.get(lastFieldName);
if (mapper != null) {
// There is a concrete mapper for this field already. Need to check if the mapper
// expects an array, if so we pass the context straight to the mapper and if not
// we serialize the array components
if (mapper instanceof ArrayValueMapperParser) {
mapper.parse(context);
} else {
serializeNonDynamicArray(context, lastFieldName, arrayFieldName);
}
} else {
Dynamic dynamic = this.dynamic;
if (dynamic == null) {
dynamic = context.root().dynamic();
}
if (dynamic == Dynamic.STRICT) {
throw new StrictDynamicMappingException(fullPath, arrayFieldName);
} else if (dynamic == Dynamic.TRUE) {
// we sync here just so we won't add it twice. Its not the end of the world
// to sync here since next operations will get it before
synchronized (mutex) {
mapper = mappers.get(arrayFieldName);
if (mapper == null) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, arrayFieldName, "object");
if (builder == null) {
serializeNonDynamicArray(context, lastFieldName, arrayFieldName);
return;
}
BuilderContext builderContext =
new BuilderContext(context.indexSettings(), context.path());
mapper = builder.build(builderContext);
if (mapper != null && mapper instanceof ArrayValueMapperParser) {
putDynamicMapper(context, arrayFieldName, mapper);
} else {
serializeNonDynamicArray(context, lastFieldName, arrayFieldName);
}
} else {
serializeNonDynamicArray(context, lastFieldName, arrayFieldName);
}
}
} else {
serializeNonDynamicArray(context, lastFieldName, arrayFieldName);
}
}
}
private Analyzer findAnalyzer(ParseContext context) {
Analyzer analyzer = indexAnalyzer;
if (analyzer == null) {
analyzer = context.analyzer();
if (analyzer == null) {
analyzer = context.docMapper().indexAnalyzer();
if (analyzer == null) {
// This should not happen, should we log warn it?
analyzer = Lucene.STANDARD_ANALYZER;
}
}
}
return analyzer;
}
@Override
protected Field parseCreateField(ParseContext context) throws IOException {
if (context.sourceToParse().routing() != null) {
String routing = context.sourceToParse().routing();
if (routing != null) {
if (!indexed() && !stored()) {
context.ignoredValue(names.indexName(), routing);
return null;
}
return new Field(names.indexName(), routing, store, index);
}
}
return null;
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
Object addressAsObject;
if (context.externalValueSet()) {
addressAsObject = context.externalValue();
} else {
addressAsObject = context.parser().textOrNull();
}
if (addressAsObject == null) {
addressAsObject = fieldType().nullValue();
}
if (addressAsObject == null) {
return;
}
String addressAsString = addressAsObject.toString();
InetAddress address;
if (addressAsObject instanceof InetAddress) {
address = (InetAddress) addressAsObject;
} else {
try {
address = InetAddresses.forString(addressAsString);
} catch (IllegalArgumentException e) {
if (ignoreMalformed.value()) {
return;
} else {
throw e;
}
}
}
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(fieldType().name(), addressAsString, fieldType().boost());
}
if (fieldType().indexOptions() != IndexOptions.NONE) {
fields.add(new InetAddressPoint(fieldType().name(), address));
}
if (fieldType().hasDocValues()) {
fields.add(
new SortedSetDocValuesField(
fieldType().name(), new BytesRef(InetAddressPoint.encode(address))));
}
if (fieldType().stored()) {
fields.add(
new StoredField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address))));
}
}
public Mapper.Builder findTemplateBuilder(
ParseContext context, String name, String dynamicType, String matchType) {
DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType);
if (dynamicTemplate == null) {
return null;
}
Mapper.TypeParser.ParserContext parserContext = context.docMapperParser().parserContext();
String mappingType = dynamicTemplate.mappingType(dynamicType);
Mapper.TypeParser typeParser = parserContext.typeParser(mappingType);
if (typeParser == null) {
throw new MapperParsingException(
"failed to find type parsed [" + mappingType + "] for [" + name + "]");
}
return typeParser.parse(name, dynamicTemplate.mappingForName(name, dynamicType), parserContext);
}
@Override
protected Field parseCreateField(ParseContext context) throws IOException {
if (!enabled) {
return null;
}
return new Field(names.indexName(), context.index(), fieldType);
}
@Override
protected Fieldable innerParseCreateField(ParseContext context) throws IOException {
if (!enabled) {
return null;
}
return new CustomIntegerNumericField(this, context.source().length());
}
void processQuery(Query query, ParseContext context) {
ParseContext.Document doc = context.doc();
FieldType pft = (FieldType) this.fieldType();
QueryAnalyzer.Result result;
try {
result = QueryAnalyzer.analyze(query);
} catch (QueryAnalyzer.UnsupportedQueryException e) {
doc.add(
new Field(
pft.extractionResultField.name(),
EXTRACTION_FAILED,
extractionResultField.fieldType()));
return;
}
for (Term term : result.terms) {
BytesRefBuilder builder = new BytesRefBuilder();
builder.append(new BytesRef(term.field()));
builder.append(FIELD_VALUE_SEPARATOR);
builder.append(term.bytes());
doc.add(new Field(queryTermsField.name(), builder.toBytesRef(), queryTermsField.fieldType()));
}
if (result.verified) {
doc.add(
new Field(
extractionResultField.name(),
EXTRACTION_COMPLETE,
extractionResultField.fieldType()));
} else {
doc.add(
new Field(
extractionResultField.name(), EXTRACTION_PARTIAL, extractionResultField.fieldType()));
}
}
private void serializeArray(ParseContext context, String lastFieldName) throws IOException {
String arrayFieldName = lastFieldName;
Mapper mapper = mappers.get(lastFieldName);
if (mapper != null && mapper instanceof ArrayValueMapperParser) {
mapper.parse(context);
} else {
XContentParser parser = context.parser();
XContentParser.Token token;
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
if (token == XContentParser.Token.START_OBJECT) {
serializeObject(context, lastFieldName);
} else if (token == XContentParser.Token.START_ARRAY) {
serializeArray(context, lastFieldName);
} else if (token == XContentParser.Token.FIELD_NAME) {
lastFieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_NULL) {
serializeNullValue(context, lastFieldName);
} else if (token == null) {
throw new MapperParsingException(
"object mapping for ["
+ name
+ "] with array for ["
+ arrayFieldName
+ "] tried to parse as array, but got EOF, is there a mismatch in types for the same field?");
} else {
serializeValue(context, lastFieldName, token);
}
}
}
}
@Override
public Mapper parse(ParseContext context) throws IOException, MapperParsingException {
if (context.sourceToParse().ttl() < 0) { // no ttl has been provided externally
long ttl;
if (context.parser().currentToken() == XContentParser.Token.VALUE_STRING) {
ttl = TimeValue.parseTimeValue(context.parser().text(), null, "ttl").millis();
} else {
ttl = context.parser().longValue(coerce.value());
}
if (ttl <= 0) {
throw new MapperParsingException(
"TTL value must be > 0. Illegal value provided [" + ttl + "]");
}
context.sourceToParse().ttl(ttl);
}
return null;
}
