/** * Parse a field as though it were a string. * * @param context parse context used during parsing * @param nullValue value to use for null * @param defaultBoost default boost value returned unless overwritten in the field * @return the parsed field and the boost either parsed or defaulted * @throws IOException if thrown while parsing */ public static ValueAndBoost parseCreateFieldForString( ParseContext context, String nullValue, float defaultBoost) throws IOException { if (context.externalValueSet()) { return new ValueAndBoost(context.externalValue().toString(), defaultBoost); } XContentParser parser = context.parser(); if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { return new ValueAndBoost(nullValue, defaultBoost); } if (parser.currentToken() == XContentParser.Token.START_OBJECT && Version.indexCreated(context.indexSettings()).before(Version.V_3_0_0)) { XContentParser.Token token; String currentFieldName = null; String value = nullValue; float boost = defaultBoost; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { value = parser.textOrNull(); } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); } else { throw new IllegalArgumentException("unknown property [" + currentFieldName + "]"); } } } return new ValueAndBoost(value, boost); } return new ValueAndBoost(parser.textOrNull(), defaultBoost); }
private void serializeObject(final ParseContext context, String currentFieldName) throws IOException { if (currentFieldName == null) { throw new MapperParsingException( "object mapping [" + name + "] trying to serialize an object with no field associated with it, current value [" + context.parser().textOrNull() + "]"); } context.path().add(currentFieldName); Mapper objectMapper = mappers.get(currentFieldName); if (objectMapper != null) { objectMapper.parse(context); } else { Dynamic dynamic = this.dynamic; if (dynamic == null) { dynamic = context.root().dynamic(); } if (dynamic == Dynamic.STRICT) { throw new StrictDynamicMappingException(fullPath, currentFieldName); } else if (dynamic == Dynamic.TRUE) { // we sync here just so we won't add it twice. Its not the end of the world // to sync here since next operations will get it before synchronized (mutex) { objectMapper = mappers.get(currentFieldName); if (objectMapper == null) { // remove the current field name from path, since template search and the object builder // add it as well... context.path().remove(); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "object"); if (builder == null) { builder = MapperBuilders.object(currentFieldName).enabled(true).pathType(pathType); // if this is a non root object, then explicitly set the dynamic behavior if set if (!(this instanceof RootObjectMapper) && this.dynamic != Defaults.DYNAMIC) { ((Builder) builder).dynamic(this.dynamic); } } BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); objectMapper = builder.build(builderContext); putDynamicMapper(context, currentFieldName, objectMapper); } else { objectMapper.parse(context); } } } else { // not dynamic, read everything up to end object context.parser().skipChildren(); } } context.path().remove(); }
private void serializeArray(ParseContext context, String lastFieldName) throws IOException { String arrayFieldName = lastFieldName; Mapper mapper = mappers.get(lastFieldName); if (mapper != null) { // There is a concrete mapper for this field already. Need to check if the mapper // expects an array, if so we pass the context straight to the mapper and if not // we serialize the array components if (mapper instanceof ArrayValueMapperParser) { mapper.parse(context); } else { serializeNonDynamicArray(context, lastFieldName, arrayFieldName); } } else { Dynamic dynamic = this.dynamic; if (dynamic == null) { dynamic = context.root().dynamic(); } if (dynamic == Dynamic.STRICT) { throw new StrictDynamicMappingException(fullPath, arrayFieldName); } else if (dynamic == Dynamic.TRUE) { // we sync here just so we won't add it twice. Its not the end of the world // to sync here since next operations will get it before synchronized (mutex) { mapper = mappers.get(arrayFieldName); if (mapper == null) { Mapper.Builder builder = context.root().findTemplateBuilder(context, arrayFieldName, "object"); if (builder == null) { serializeNonDynamicArray(context, lastFieldName, arrayFieldName); return; } BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); mapper = builder.build(builderContext); if (mapper != null && mapper instanceof ArrayValueMapperParser) { putDynamicMapper(context, arrayFieldName, mapper); } else { serializeNonDynamicArray(context, lastFieldName, arrayFieldName); } } else { serializeNonDynamicArray(context, lastFieldName, arrayFieldName); } } } else { serializeNonDynamicArray(context, lastFieldName, arrayFieldName); } } }
@Override protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException { byte value; float boost = fieldType().boost(); if (context.externalValueSet()) { Object externalValue = context.externalValue(); if (externalValue == null) { if (fieldType().nullValue() == null) { return; } value = fieldType().nullValue(); } else if (externalValue instanceof String) { String sExternalValue = (String) externalValue; if (sExternalValue.length() == 0) { if (fieldType().nullValue() == null) { return; } value = fieldType().nullValue(); } else { value = Byte.parseByte(sExternalValue); } } else { value = ((Number) externalValue).byteValue(); } if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(fieldType().name(), Byte.toString(value), boost); } } else { XContentParser parser = context.parser(); if (parser.currentToken() == XContentParser.Token.VALUE_NULL || (parser.currentToken() == XContentParser.Token.VALUE_STRING && parser.textLength() == 0)) { if (fieldType().nullValue() == null) { return; } value = fieldType().nullValue(); if (fieldType().nullValueAsString() != null && (context.includeInAll(includeInAll, this))) { context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT && Version.indexCreated(context.indexSettings()).before(Version.V_3_0_0)) { XContentParser.Token token; String currentFieldName = null; Byte objValue = fieldType().nullValue(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { objValue = (byte) parser.shortValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); } else { throw new IllegalArgumentException("unknown property [" + currentFieldName + "]"); } } } if (objValue == null) { // no value return; } value = objValue; } else { value = (byte) parser.shortValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(fieldType().name(), parser.text(), boost); } } } if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomByteNumericField field = new CustomByteNumericField(value, fieldType()); field.setBoost(boost); fields.add(field); } if (fieldType().hasDocValues()) { addDocValue(context, fields, value); } }
private void serializeValue( final ParseContext context, String currentFieldName, XContentParser.Token token) throws IOException { if (currentFieldName == null) { throw new MapperParsingException( "object mapping [" + name + "] trying to serialize a value with no field associated with it, current value [" + context.parser().textOrNull() + "]"); } Mapper mapper = mappers.get(currentFieldName); if (mapper != null) { mapper.parse(context); return; } Dynamic dynamic = this.dynamic; if (dynamic == null) { dynamic = context.root().dynamic(); } if (dynamic == Dynamic.STRICT) { throw new StrictDynamicMappingException(fullPath, currentFieldName); } if (dynamic == Dynamic.FALSE) { return; } // we sync here since we don't want to add this field twice to the document mapper // its not the end of the world, since we add it to the mappers once we create it // so next time we won't even get here for this field boolean newMapper = false; synchronized (mutex) { mapper = mappers.get(currentFieldName); if (mapper == null) { newMapper = true; BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); if (token == XContentParser.Token.VALUE_STRING) { boolean resolved = false; // do a quick test to see if its fits a dynamic template, if so, use it. // we need to do it here so we can handle things like attachment templates, where calling // text (to see if its a date) causes the binary value to be cleared if (!resolved) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string", null); if (builder != null) { mapper = builder.build(builderContext); resolved = true; } } if (!resolved && context.parser().textLength() == 0) { // empty string with no mapping, treat it like null value return; } if (!resolved && context.root().dateDetection()) { String text = context.parser().text(); // a safe check since "1" gets parsed as well if (Strings.countOccurrencesOf(text, ":") > 1 || Strings.countOccurrencesOf(text, "-") > 1 || Strings.countOccurrencesOf(text, "/") > 1) { for (FormatDateTimeFormatter dateTimeFormatter : context.root().dynamicDateTimeFormatters()) { try { dateTimeFormatter.parser().parseMillis(text); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "date"); if (builder == null) { builder = dateField(currentFieldName).dateTimeFormatter(dateTimeFormatter); } mapper = builder.build(builderContext); resolved = true; break; } catch (Exception e) { // failure to parse this, continue } } } } if (!resolved && context.root().numericDetection()) { String text = context.parser().text(); try { Long.parseLong(text); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); if (builder == null) { builder = longField(currentFieldName); } mapper = builder.build(builderContext); resolved = true; } catch (Exception e) { // not a long number } if (!resolved) { try { Double.parseDouble(text); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); if (builder == null) { builder = doubleField(currentFieldName); } mapper = builder.build(builderContext); resolved = true; } catch (Exception e) { // not a long number } } } // DON'T do automatic ip detection logic, since it messes up with docs that have hosts and // ips // check if its an ip // if (!resolved && text.indexOf('.') != -1) { // try { // IpFieldMapper.ipToLong(text); // XContentMapper.Builder builder = // context.root().findTemplateBuilder(context, currentFieldName, "ip"); // if (builder == null) { // builder = ipField(currentFieldName); // } // mapper = builder.build(builderContext); // resolved = true; // } catch (Exception e) { // // failure to parse, not ip... // } // } if (!resolved) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string"); if (builder == null) { builder = stringField(currentFieldName); } mapper = builder.build(builderContext); } } else if (token == XContentParser.Token.VALUE_NUMBER) { XContentParser.NumberType numberType = context.parser().numberType(); if (numberType == XContentParser.NumberType.INT) { if (context.parser().estimatedNumberType()) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); if (builder == null) { builder = longField(currentFieldName); } mapper = builder.build(builderContext); } else { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "integer"); if (builder == null) { builder = integerField(currentFieldName); } mapper = builder.build(builderContext); } } else if (numberType == XContentParser.NumberType.LONG) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); if (builder == null) { builder = longField(currentFieldName); } mapper = builder.build(builderContext); } else if (numberType == XContentParser.NumberType.FLOAT) { if (context.parser().estimatedNumberType()) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); if (builder == null) { builder = doubleField(currentFieldName); } mapper = builder.build(builderContext); } else { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "float"); if (builder == null) { builder = floatField(currentFieldName); } mapper = builder.build(builderContext); } } else if (numberType == XContentParser.NumberType.DOUBLE) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); if (builder == null) { builder = doubleField(currentFieldName); } mapper = builder.build(builderContext); } } else if (token == XContentParser.Token.VALUE_BOOLEAN) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "boolean"); if (builder == null) { builder = booleanField(currentFieldName); } mapper = builder.build(builderContext); } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "binary"); if (builder == null) { builder = binaryField(currentFieldName); } mapper = builder.build(builderContext); } else { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, null); if (builder != null) { mapper = builder.build(builderContext); } else { // TODO how do we identify dynamically that its a binary value? throw new ElasticSearchIllegalStateException( "Can't handle serializing a dynamic type with content token [" + token + "] and field name [" + currentFieldName + "]"); } } putMapper(mapper); context.setMappingsModified(); } } if (newMapper) { mapper.traverse(context.newFieldMappers()); } mapper.parse(context); }
@Override public void parse(ParseContext context) throws IOException { String iri; XContentParser parser = context.parser(); XContentParser.Token token = parser.currentToken(); if (token == XContentParser.Token.VALUE_STRING) { iri = parser.text(); } else { throw new MapperParsingException(name() + " does not contain String value"); } ContentPath.Type origPathType = context.path().pathType(); context.path().pathType(ContentPath.Type.FULL); context.path().add(names.name()); try { OntologyHelper helper = getHelper(ontologySettings, threadPool); OntologyData data = findOntologyData(helper, iri); if (data == null) { logger.debug("Cannot find OWL class for IRI {}", iri); } else { addFieldData( context, getPredefinedMapper(FieldMappings.URI, context), Collections.singletonList(iri)); // Look up the label(s) addFieldData(context, getPredefinedMapper(FieldMappings.LABEL, context), data.getLabels()); // Look up the synonyms addFieldData( context, getPredefinedMapper(FieldMappings.SYNONYMS, context), data.getLabels()); // Add the child details addRelatedNodesWithLabels( context, data.getChildIris(), getPredefinedMapper(FieldMappings.CHILD_URI, context), data.getChildLabels(), getPredefinedMapper(FieldMappings.CHILD_LABEL, context)); // Add the parent details addRelatedNodesWithLabels( context, data.getParentIris(), getPredefinedMapper(FieldMappings.PARENT_URI, context), data.getParentLabels(), getPredefinedMapper(FieldMappings.PARENT_LABEL, context)); if (ontologySettings.isIncludeIndirect()) { // Add the descendant details addRelatedNodesWithLabels( context, data.getDescendantIris(), getPredefinedMapper(FieldMappings.DESCENDANT_URI, context), data.getDescendantLabels(), getPredefinedMapper(FieldMappings.DESCENDANT_LABEL, context)); // Add the ancestor details addRelatedNodesWithLabels( context, data.getAncestorIris(), getPredefinedMapper(FieldMappings.ANCESTOR_URI, context), data.getAncestorLabels(), getPredefinedMapper(FieldMappings.ANCESTOR_LABEL, context)); } if (ontologySettings.isIncludeRelations()) { // Add the related nodes Map<String, Collection<String>> relations = data.getRelationIris(); for (String relation : relations.keySet()) { // Sanitise the relation name String sanRelation = relation.replaceAll("\\W+", "_"); String uriMapperName = sanRelation + DYNAMIC_URI_FIELD_SUFFIX; String labelMapperName = sanRelation + DYNAMIC_LABEL_FIELD_SUFFIX; // Get the mapper for the relation FieldMapper<String> uriMapper = mappers.get(context.path().fullPathAsText(uriMapperName)); FieldMapper<String> labelMapper = mappers.get(context.path().fullPathAsText(labelMapperName)); if (uriMapper == null) { // No mappers created yet - build new ones for URI and label BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); uriMapper = MapperBuilders.stringField(uriMapperName) .store(true) .index(true) .tokenized(false) .build(builderContext); labelMapper = MapperBuilders.stringField(labelMapperName) .store(true) .index(true) .tokenized(true) .build(builderContext); } addRelatedNodesWithLabels( context, relations.get(relation), uriMapper, helper.findLabelsForIRIs(relations.get(relation)), labelMapper); } } } helper.updateLastCallTime(); } catch (OntologyHelperException e) { throw new ElasticsearchException("Could not initialise ontology helper", e); } finally { context.path().remove(); context.path().pathType(origPathType); } }
private void serializeObject(final ParseContext context, String currentFieldName) throws IOException { if (currentFieldName == null) { throw new MapperParsingException( "object mapping [" + name + "] trying to serialize an object with no field associated with it, current value [" + context.parser().textOrNull() + "]"); } context.path().add(currentFieldName); Mapper objectMapper = mappers.get(currentFieldName); if (objectMapper != null) { objectMapper.parse(context); } else { Dynamic dynamic = this.dynamic; if (dynamic == null) { dynamic = context.root().dynamic(); } if (dynamic == Dynamic.STRICT) { throw new StrictDynamicMappingException(currentFieldName); } else if (dynamic == Dynamic.TRUE) { // we sync here just so we won't add it twice. Its not the end of the world // to sync here since next operations will get it before boolean newMapper = false; synchronized (mutex) { objectMapper = mappers.get(currentFieldName); if (objectMapper == null) { newMapper = true; Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "object"); if (builder == null) { builder = MapperBuilders.object(currentFieldName) .enabled(true) .dynamic(dynamic) .pathType(pathType); } // remove the current field name from path, since the object builder adds it as well... context.path().remove(); BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); objectMapper = builder.build(builderContext); putMapper(objectMapper); // now re add it context.path().add(currentFieldName); context.addedMapper(); } } // traverse and parse outside of the mutex if (newMapper) { // we need to traverse in case we have a dynamic template and need to add field mappers // introduced by it objectMapper.traverse( new FieldMapperListener() { @Override public void fieldMapper(FieldMapper fieldMapper) { context.docMapper().addFieldMapper(fieldMapper); } }); objectMapper.traverse( new ObjectMapperListener() { @Override public void objectMapper(ObjectMapper objectMapper) { context.docMapper().addObjectMapper(objectMapper); } }); } // now, parse it objectMapper.parse(context); } else { // not dynamic, read everything up to end object context.parser().skipChildren(); } } context.path().remove(); }
public void parseDynamicValue( final ParseContext context, String currentFieldName, XContentParser.Token token) throws IOException { Dynamic dynamic = this.dynamic; if (dynamic == null) { dynamic = context.root().dynamic(); } if (dynamic == Dynamic.STRICT) { throw new StrictDynamicMappingException(fullPath, currentFieldName); } if (dynamic == Dynamic.FALSE) { return; } // we sync here since we don't want to add this field twice to the document mapper // its not the end of the world, since we add it to the mappers once we create it // so next time we won't even get here for this field synchronized (mutex) { Mapper mapper = mappers.get(currentFieldName); if (mapper == null) { BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path()); if (token == XContentParser.Token.VALUE_STRING) { boolean resolved = false; // do a quick test to see if its fits a dynamic template, if so, use it. // we need to do it here so we can handle things like attachment templates, where calling // text (to see if its a date) causes the binary value to be cleared if (!resolved) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string", null); if (builder != null) { mapper = builder.build(builderContext); resolved = true; } } if (!resolved && context.parser().textLength() == 0) { // empty string with no mapping, treat it like null value return; } if (!resolved && context.root().dateDetection()) { String text = context.parser().text(); // a safe check since "1" gets parsed as well if (Strings.countOccurrencesOf(text, ":") > 1 || Strings.countOccurrencesOf(text, "-") > 1 || Strings.countOccurrencesOf(text, "/") > 1) { for (FormatDateTimeFormatter dateTimeFormatter : context.root().dynamicDateTimeFormatters()) { try { dateTimeFormatter.parser().parseMillis(text); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "date"); if (builder == null) { builder = dateField(currentFieldName).dateTimeFormatter(dateTimeFormatter); } mapper = builder.build(builderContext); resolved = true; break; } catch (Exception e) { // failure to parse this, continue } } } } if (!resolved && context.root().numericDetection()) { String text = context.parser().text(); try { Long.parseLong(text); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); if (builder == null) { builder = longField(currentFieldName); } mapper = builder.build(builderContext); resolved = true; } catch (Exception e) { // not a long number } if (!resolved) { try { Double.parseDouble(text); Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); if (builder == null) { builder = doubleField(currentFieldName); } mapper = builder.build(builderContext); resolved = true; } catch (Exception e) { // not a long number } } } if (!resolved) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "string"); if (builder == null) { builder = stringField(currentFieldName); } mapper = builder.build(builderContext); } } else if (token == XContentParser.Token.VALUE_NUMBER) { XContentParser.NumberType numberType = context.parser().numberType(); if (numberType == XContentParser.NumberType.INT) { if (context.parser().estimatedNumberType()) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); if (builder == null) { builder = longField(currentFieldName); } mapper = builder.build(builderContext); } else { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "integer"); if (builder == null) { builder = integerField(currentFieldName); } mapper = builder.build(builderContext); } } else if (numberType == XContentParser.NumberType.LONG) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "long"); if (builder == null) { builder = longField(currentFieldName); } mapper = builder.build(builderContext); } else if (numberType == XContentParser.NumberType.FLOAT) { if (context.parser().estimatedNumberType()) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); if (builder == null) { builder = doubleField(currentFieldName); } mapper = builder.build(builderContext); } else { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "float"); if (builder == null) { builder = floatField(currentFieldName); } mapper = builder.build(builderContext); } } else if (numberType == XContentParser.NumberType.DOUBLE) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "double"); if (builder == null) { builder = doubleField(currentFieldName); } mapper = builder.build(builderContext); } } else if (token == XContentParser.Token.VALUE_BOOLEAN) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "boolean"); if (builder == null) { builder = booleanField(currentFieldName); } mapper = builder.build(builderContext); } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "binary"); if (builder == null) { builder = binaryField(currentFieldName); } mapper = builder.build(builderContext); } else { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, null); if (builder != null) { mapper = builder.build(builderContext); } else { // TODO how do we identify dynamically that its a binary value? throw new ElasticsearchIllegalStateException( "Can't handle serializing a dynamic type with content token [" + token + "] and field name [" + currentFieldName + "]"); } } if (context.isWithinNewMapper()) { mapper.parse(context); } else { context.setWithinNewMapper(); try { mapper.parse(context); FieldMapperListener.Aggregator newFields = new FieldMapperListener.Aggregator(); mapper.traverse(newFields); context.docMapper().addFieldMappers(newFields.mappers); } finally { context.clearWithinNewMapper(); } } // only put after we traversed and did the callbacks, so other parsing won't see it only // after we // properly traversed it and adding the mappers putMapper(mapper); context.setMappingsModified(); } else { mapper.parse(context); } } }