@Override
public Mapper parse(ParseContext context) throws IOException {
QueryShardContext queryShardContext = new QueryShardContext(this.queryShardContext);
if (context.doc().getField(queryBuilderField.name()) != null) {
// If a percolator query has been defined in an array object then multiple percolator queries
// could be provided. In order to prevent this we fail if we try to parse more than one query
// for the current document.
throw new IllegalArgumentException("a document can only contain one percolator query");
}
XContentParser parser = context.parser();
QueryBuilder queryBuilder =
parseQueryBuilder(queryShardContext.newParseContext(parser), parser.getTokenLocation());
verifyQuery(queryBuilder);
// Fetching of terms, shapes and indexed scripts happen during this rewrite:
queryBuilder = queryBuilder.rewrite(queryShardContext);
try (XContentBuilder builder = XContentFactory.contentBuilder(QUERY_BUILDER_CONTENT_TYPE)) {
queryBuilder.toXContent(builder, new MapParams(Collections.emptyMap()));
builder.flush();
byte[] queryBuilderAsBytes = BytesReference.toBytes(builder.bytes());
context
.doc()
.add(
new Field(
queryBuilderField.name(), queryBuilderAsBytes, queryBuilderField.fieldType()));
}
Query query = toQuery(queryShardContext, mapUnmappedFieldAsString, queryBuilder);
processQuery(query, context);
return null;
}
private void parse(ParseContext context, GeoPoint point, String geohash) throws IOException {
if (fieldType().ignoreMalformed == false) {
if (point.lat() > 90.0 || point.lat() < -90.0) {
throw new IllegalArgumentException(
"illegal latitude value [" + point.lat() + "] for " + name());
}
if (point.lon() > 180.0 || point.lon() < -180) {
throw new IllegalArgumentException(
"illegal longitude value [" + point.lon() + "] for " + name());
}
}
if (fieldType().coerce) {
// by setting coerce to false we are assuming all geopoints are already in a valid coordinate
// system
// thus this extra step can be skipped
// LUCENE WATCH: This will be folded back into Lucene's GeoPointField
GeoUtils.normalizePoint(point, true, true);
}
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
Field field =
new Field(
fieldType().names().indexName(),
Double.toString(point.lat()) + ',' + Double.toString(point.lon()),
fieldType());
context.doc().add(field);
}
if (fieldType().isGeohashEnabled()) {
if (geohash == null) {
geohash = GeoHashUtils.encode(point.lat(), point.lon());
}
addGeohashField(context, geohash);
}
if (fieldType().isLatLonEnabled()) {
latMapper.parse(context.createExternalValueContext(point.lat()));
lonMapper.parse(context.createExternalValueContext(point.lon()));
}
if (fieldType().hasDocValues()) {
CustomGeoPointDocValuesField field =
(CustomGeoPointDocValuesField) context.doc().getByKey(fieldType().names().indexName());
if (field == null) {
field =
new CustomGeoPointDocValuesField(
fieldType().names().indexName(), point.lat(), point.lon());
context.doc().addWithKey(fieldType().names().indexName(), field);
} else {
field.add(point.lat(), point.lon());
}
}
multiFields.parse(this, context);
}
@Override
public void validate(ParseContext context) throws MapperParsingException {
String routing = context.sourceToParse().routing();
if (path != null && routing != null) {
// we have a path, check if we can validate we have the same routing value as the one in the
// doc...
String value = null;
Fieldable field = context.doc().getFieldable(path);
if (field != null) {
value = field.stringValue();
if (value == null) {
// maybe its a numeric field...
if (field instanceof NumberFieldMapper.CustomNumericField) {
value = ((NumberFieldMapper.CustomNumericField) field).numericAsString();
}
}
}
if (value == null) {
value = context.ignoredValue(path);
}
if (value == null) {
// maybe its a numeric field
}
if (!routing.equals(value)) {
throw new MapperParsingException(
"External routing ["
+ routing
+ "] and document path routing ["
+ value
+ "] mismatch");
}
}
}
void processQuery(Query query, ParseContext context) {
ParseContext.Document doc = context.doc();
FieldType pft = (FieldType) this.fieldType();
QueryAnalyzer.Result result;
try {
result = QueryAnalyzer.analyze(query);
} catch (QueryAnalyzer.UnsupportedQueryException e) {
doc.add(
new Field(
pft.extractionResultField.name(),
EXTRACTION_FAILED,
extractionResultField.fieldType()));
return;
}
for (Term term : result.terms) {
BytesRefBuilder builder = new BytesRefBuilder();
builder.append(new BytesRef(term.field()));
builder.append(FIELD_VALUE_SEPARATOR);
builder.append(term.bytes());
doc.add(new Field(queryTermsField.name(), builder.toBytesRef(), queryTermsField.fieldType()));
}
if (result.verified) {
doc.add(
new Field(
extractionResultField.name(),
EXTRACTION_COMPLETE,
extractionResultField.fieldType()));
} else {
doc.add(
new Field(
extractionResultField.name(), EXTRACTION_PARTIAL, extractionResultField.fieldType()));
}
}
@Override
protected Field parseCreateField(ParseContext context) throws IOException {
if (context.parser().currentName() != null
&& context.parser().currentName().equals(Defaults.NAME)) {
// we are in the parsing of _parent phase
String parentId = context.parser().text();
context.sourceToParse().parent(parentId);
return new Field(
names.indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType);
}
// otherwise, we are running it post processing of the xcontent
String parsedParentId = context.doc().get(Defaults.NAME);
if (context.sourceToParse().parent() != null) {
String parentId = context.sourceToParse().parent();
if (parsedParentId == null) {
if (parentId == null) {
throw new MapperParsingException(
"No parent id provided, not within the document, and not externally");
}
// we did not add it in the parsing phase, add it now
return new Field(
names.indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType);
} else if (parentId != null
&& !parsedParentId.equals(Uid.createUid(context.stringBuilder(), type, parentId))) {
throw new MapperParsingException(
"Parent id mismatch, document value is ["
+ Uid.createUid(parsedParentId).id()
+ "], while external value is ["
+ parentId
+ "]");
}
}
// we have parent mapping, yet no value was set, ignore it...
return null;
}
@Override
public void createField(ParseContext context, Object object) throws IOException {
Long token = (Long) object;
if (token != null) {
context.doc().add(new LongFieldMapper.CustomLongNumericField(token, fieldType()));
context.doc().add(new SortedNumericDocValuesField(fieldType().names().indexName(), token));
}
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
boolean parent = context.docMapper().isParent(context.type());
if (parent) {
addJoinFieldIfNeeded(fields, parentJoinFieldType, context.id());
}
if (!active()) {
return;
}
if (context.parser().currentName() != null
&& context.parser().currentName().equals(Defaults.NAME)) {
// we are in the parsing of _parent phase
String parentId = context.parser().text();
context.sourceToParse().parent(parentId);
fields.add(
new Field(
fieldType().name(),
Uid.createUid(context.stringBuilder(), parentType, parentId),
fieldType()));
addJoinFieldIfNeeded(fields, childJoinFieldType, parentId);
} else {
// otherwise, we are running it post processing of the xcontent
String parsedParentId = context.doc().get(Defaults.NAME);
if (context.sourceToParse().parent() != null) {
String parentId = context.sourceToParse().parent();
if (parsedParentId == null) {
if (parentId == null) {
throw new MapperParsingException(
"No parent id provided, not within the document, and not externally");
}
// we did not add it in the parsing phase, add it now
fields.add(
new Field(
fieldType().name(),
Uid.createUid(context.stringBuilder(), parentType, parentId),
fieldType()));
addJoinFieldIfNeeded(fields, childJoinFieldType, parentId);
} else if (parentId != null
&& !parsedParentId.equals(
Uid.createUid(context.stringBuilder(), parentType, parentId))) {
throw new MapperParsingException(
"Parent id mismatch, document value is ["
+ Uid.createUid(parsedParentId).id()
+ "], while external value is ["
+ parentId
+ "]");
}
}
}
// we have parent mapping, yet no value was set, ignore it...
}
private void parseData(
ParseContext context, FieldMapper<String> mapper, Collection<String> values)
throws IOException {
for (String value : values) {
Field field =
new Field(
mapper.names().indexName(),
value,
isUriField(mapper.name()) ? Defaults.URI_FIELD_TYPE : Defaults.LABEL_FIELD_TYPE);
context.doc().add(field);
}
}
@Override
public Mapper parse(ParseContext context) throws IOException {
try {
Shape shape = context.parseExternalValue(Shape.class);
if (shape == null) {
ShapeBuilder shapeBuilder = ShapeBuilder.parse(context.parser(), this);
if (shapeBuilder == null) {
return null;
}
shape = shapeBuilder.build();
}
if (fieldType().pointsOnly() && !(shape instanceof Point)) {
throw new MapperParsingException(
"[{"
+ fieldType().names().fullName()
+ "}] is configured for points only but a "
+ ((shape instanceof JtsGeometry)
? ((JtsGeometry) shape).getGeom().getGeometryType()
: shape.getClass())
+ " was found");
}
Field[] fields = fieldType().defaultStrategy().createIndexableFields(shape);
if (fields == null || fields.length == 0) {
return null;
}
for (Field field : fields) {
if (!customBoost()) {
field.setBoost(fieldType().boost());
}
context.doc().add(field);
}
} catch (Exception e) {
throw new MapperParsingException(
"failed to parse [" + fieldType().names().fullName() + "]", e);
}
return null;
}
@Override
public Mapper parse(ParseContext context) throws IOException {
XContentParser parser = context.parser();
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.VALUE_NULL) {
throw new MapperParsingException(
"completion field [" + fieldType().names().fullName() + "] does not support null values");
}
String surfaceForm = null;
BytesRef payload = null;
long weight = -1;
List<String> inputs = Lists.newArrayListWithExpectedSize(4);
SortedMap<String, ContextConfig> contextConfig = null;
if (token == XContentParser.Token.VALUE_STRING) {
inputs.add(parser.text());
multiFields.parse(this, context);
} else {
String currentFieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
if (!ALLOWED_CONTENT_FIELD_NAMES.contains(currentFieldName)) {
throw new IllegalArgumentException(
"Unknown field name["
+ currentFieldName
+ "], must be one of "
+ ALLOWED_CONTENT_FIELD_NAMES);
}
} else if (Fields.CONTEXT.equals(currentFieldName)) {
SortedMap<String, ContextConfig> configs = Maps.newTreeMap();
if (token == Token.START_OBJECT) {
while ((token = parser.nextToken()) != Token.END_OBJECT) {
String name = parser.text();
ContextMapping mapping = fieldType().getContextMapping().get(name);
if (mapping == null) {
throw new ElasticsearchParseException("context [{}] is not defined", name);
} else {
token = parser.nextToken();
configs.put(name, mapping.parseContext(context, parser));
}
}
contextConfig = Maps.newTreeMap();
for (ContextMapping mapping : fieldType().getContextMapping().values()) {
ContextConfig config = configs.get(mapping.name());
contextConfig.put(mapping.name(), config == null ? mapping.defaultConfig() : config);
}
} else {
throw new ElasticsearchParseException("context must be an object");
}
} else if (Fields.CONTENT_FIELD_NAME_PAYLOAD.equals(currentFieldName)) {
if (!isStoringPayloads()) {
throw new MapperException("Payloads disabled in mapping");
}
if (token == XContentParser.Token.START_OBJECT) {
XContentBuilder payloadBuilder =
XContentFactory.contentBuilder(parser.contentType()).copyCurrentStructure(parser);
payload = payloadBuilder.bytes().toBytesRef();
payloadBuilder.close();
} else if (token.isValue()) {
payload = parser.utf8BytesOrNull();
} else {
throw new MapperException("payload doesn't support type " + token);
}
} else if (token == XContentParser.Token.VALUE_STRING) {
if (Fields.CONTENT_FIELD_NAME_OUTPUT.equals(currentFieldName)) {
surfaceForm = parser.text();
}
if (Fields.CONTENT_FIELD_NAME_INPUT.equals(currentFieldName)) {
inputs.add(parser.text());
}
if (Fields.CONTENT_FIELD_NAME_WEIGHT.equals(currentFieldName)) {
Number weightValue;
try {
weightValue = Long.parseLong(parser.text());
} catch (NumberFormatException e) {
throw new IllegalArgumentException(
"Weight must be a string representing a numeric value, but was ["
+ parser.text()
+ "]");
}
weight =
weightValue.longValue(); // always parse a long to make sure we don't get overflow
checkWeight(weight);
}
} else if (token == XContentParser.Token.VALUE_NUMBER) {
if (Fields.CONTENT_FIELD_NAME_WEIGHT.equals(currentFieldName)) {
NumberType numberType = parser.numberType();
if (NumberType.LONG != numberType && NumberType.INT != numberType) {
throw new IllegalArgumentException(
"Weight must be an integer, but was [" + parser.numberValue() + "]");
}
weight = parser.longValue(); // always parse a long to make sure we don't get overflow
checkWeight(weight);
}
} else if (token == XContentParser.Token.START_ARRAY) {
if (Fields.CONTENT_FIELD_NAME_INPUT.equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
inputs.add(parser.text());
}
}
}
}
}
if (contextConfig == null) {
contextConfig = Maps.newTreeMap();
for (ContextMapping mapping : fieldType().getContextMapping().values()) {
contextConfig.put(mapping.name(), mapping.defaultConfig());
}
}
final ContextMapping.Context ctx = new ContextMapping.Context(contextConfig, context.doc());
payload = payload == null ? EMPTY : payload;
if (surfaceForm == null) { // no surface form use the input
for (String input : inputs) {
if (input.length() == 0) {
continue;
}
BytesRef suggestPayload =
fieldType()
.analyzingSuggestLookupProvider
.buildPayload(new BytesRef(input), weight, payload);
context.doc().add(getCompletionField(ctx, input, suggestPayload));
}
} else {
BytesRef suggestPayload =
fieldType()
.analyzingSuggestLookupProvider
.buildPayload(new BytesRef(surfaceForm), weight, payload);
for (String input : inputs) {
if (input.length() == 0) {
continue;
}
context.doc().add(getCompletionField(ctx, input, suggestPayload));
}
}
return null;
}
@Override
public void parse(ParseContext context) throws IOException {
XContentParser parser = context.parser();
XContentParser.Token token = parser.currentToken();
String surfaceForm = null;
BytesRef payload = null;
long weight = -1;
List<String> inputs = Lists.newArrayListWithExpectedSize(4);
if (token == XContentParser.Token.VALUE_STRING) {
inputs.add(parser.text());
} else {
String currentFieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if ("payload".equals(currentFieldName)) {
if (!isStoringPayloads()) {
throw new MapperException("Payloads disabled in mapping");
}
if (token == XContentParser.Token.START_OBJECT) {
XContentBuilder payloadBuilder =
XContentFactory.contentBuilder(parser.contentType()).copyCurrentStructure(parser);
payload = payloadBuilder.bytes().toBytesRef();
payloadBuilder.close();
}
} else if (token == XContentParser.Token.VALUE_STRING) {
if ("output".equals(currentFieldName)) {
surfaceForm = parser.text();
}
if ("input".equals(currentFieldName)) {
inputs.add(parser.text());
}
} else if (token == XContentParser.Token.VALUE_NUMBER) {
if ("weight".equals(currentFieldName)) {
weight =
parser.longValue(); // always parse a long to make sure we don't get the overflow
// value
if (weight < 0 || weight > Integer.MAX_VALUE) {
throw new ElasticSearchIllegalArgumentException(
"Weight must be in the interval [0..2147483647] but was " + weight);
}
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("input".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
inputs.add(parser.text());
}
}
}
}
}
payload = payload == null ? EMPTY : payload;
if (surfaceForm == null) { // no surface form use the input
for (String input : inputs) {
BytesRef suggestPayload =
analyzingSuggestLookupProvider.buildPayload(new BytesRef(input), weight, payload);
context.doc().add(getCompletionField(input, suggestPayload));
}
} else {
BytesRef suggestPayload =
analyzingSuggestLookupProvider.buildPayload(new BytesRef(surfaceForm), weight, payload);
for (String input : inputs) {
context.doc().add(getCompletionField(input, suggestPayload));
}
}
}
public void parse(ParseContext context) throws IOException {
if (!enabled) {
context.parser().skipChildren();
return;
}
XContentParser parser = context.parser();
String currentFieldName = parser.currentName();
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.VALUE_NULL) {
// the object is null ("obj1" : null), simply bail
return;
}
if (token.isValue() && !allowValue()) {
// if we are parsing an object but it is just a value, its only allowed on root level parsers
// with there
// is a field name with the same name as the type
throw new MapperParsingException(
"object mapping for [" + name + "] tried to parse as object, but found a concrete value");
}
Document restoreDoc = null;
if (nested.isNested()) {
Document nestedDoc = new Document();
// pre add the uid field if possible (id was already provided)
IndexableField uidField = context.doc().getField(UidFieldMapper.NAME);
if (uidField != null) {
// we don't need to add it as a full uid field in nested docs, since we don't need
// versioning
// we also rely on this for UidField#loadVersion
// this is a deeply nested field
if (uidField.stringValue() != null) {
nestedDoc.add(
new Field(
UidFieldMapper.NAME,
uidField.stringValue(),
UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
} else {
nestedDoc.add(
new Field(
UidFieldMapper.NAME,
((UidField) uidField).uid(),
UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
}
}
// the type of the nested doc starts with __, so we can identify that its a nested one in
// filters
// note, we don't prefix it with the type of the doc since it allows us to execute a nested
// query
// across types (for example, with similar nested objects)
nestedDoc.add(
new Field(
TypeFieldMapper.NAME, nestedTypePathAsString, TypeFieldMapper.Defaults.FIELD_TYPE));
restoreDoc = context.switchDoc(nestedDoc);
context.addDoc(nestedDoc);
}
ContentPath.Type origPathType = context.path().pathType();
context.path().pathType(pathType);
// if we are at the end of the previous object, advance
if (token == XContentParser.Token.END_OBJECT) {
token = parser.nextToken();
}
if (token == XContentParser.Token.START_OBJECT) {
// if we are just starting an OBJECT, advance, this is the object we are parsing, we need the
// name first
token = parser.nextToken();
}
while (token != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.START_OBJECT) {
serializeObject(context, currentFieldName);
} else if (token == XContentParser.Token.START_ARRAY) {
serializeArray(context, currentFieldName);
} else if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_NULL) {
serializeNullValue(context, currentFieldName);
} else if (token == null) {
throw new MapperParsingException(
"object mapping for ["
+ name
+ "] tried to parse as object, but got EOF, has a concrete value been provided to it?");
} else if (token.isValue()) {
serializeValue(context, currentFieldName, token);
}
token = parser.nextToken();
}
// restore the enable path flag
context.path().pathType(origPathType);
if (nested.isNested()) {
Document nestedDoc = context.switchDoc(restoreDoc);
if (nested.isIncludeInParent()) {
for (IndexableField field : nestedDoc.getFields()) {
if (field.name().equals(UidFieldMapper.NAME)
|| field.name().equals(TypeFieldMapper.NAME)) {
continue;
} else {
context.doc().add(field);
}
}
}
if (nested.isIncludeInRoot()) {
// don't add it twice, if its included in parent, and we are handling the master doc...
if (!(nested.isIncludeInParent() && context.doc() == context.rootDoc())) {
for (IndexableField field : nestedDoc.getFields()) {
if (field.name().equals(UidFieldMapper.NAME)
|| field.name().equals(TypeFieldMapper.NAME)) {
continue;
} else {
context.rootDoc().add(field);
}
}
}
}
}
}
@Override
protected void innerParseCreateField(ParseContext context, List<Field> fields)
throws IOException {
double value;
float boost = this.boost;
if (context.externalValueSet()) {
Object externalValue = context.externalValue();
if (externalValue == null) {
if (nullValue == null) {
return;
}
value = nullValue;
} else if (externalValue instanceof String) {
String sExternalValue = (String) externalValue;
if (sExternalValue.length() == 0) {
if (nullValue == null) {
return;
}
value = nullValue;
} else {
value = Double.parseDouble(sExternalValue);
}
} else {
value = ((Number) externalValue).doubleValue();
}
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), Double.toString(value), boost);
}
} else {
XContentParser parser = context.parser();
if (parser.currentToken() == XContentParser.Token.VALUE_NULL
|| (parser.currentToken() == XContentParser.Token.VALUE_STRING
&& parser.textLength() == 0)) {
if (nullValue == null) {
return;
}
value = nullValue;
if (nullValueAsString != null && (context.includeInAll(includeInAll, this))) {
context.allEntries().addText(names.fullName(), nullValueAsString, boost);
}
} else if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
XContentParser.Token token;
String currentFieldName = null;
Double objValue = nullValue;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
objValue = parser.doubleValue(coerce.value());
}
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
} else {
throw new ElasticsearchIllegalArgumentException(
"unknown property [" + currentFieldName + "]");
}
}
}
if (objValue == null) {
// no value
return;
}
value = objValue;
} else {
value = parser.doubleValue(coerce.value());
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), parser.text(), boost);
}
}
}
if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
CustomDoubleNumericField field = new CustomDoubleNumericField(this, value, fieldType);
field.setBoost(boost);
fields.add(field);
}
if (hasDocValues()) {
if (useSortedNumericDocValues) {
addDocValue(context, fields, NumericUtils.doubleToSortableLong(value));
} else {
CustomDoubleNumericDocValuesField field =
(CustomDoubleNumericDocValuesField) context.doc().getByKey(names().indexName());
if (field != null) {
field.add(value);
} else {
field = new CustomDoubleNumericDocValuesField(names().indexName(), value);
context.doc().addWithKey(names().indexName(), field);
}
}
}
}