@Override protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException { String ipAsString; if (context.externalValueSet()) { ipAsString = (String) context.externalValue(); if (ipAsString == null) { ipAsString = nullValue; } } else { if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) { ipAsString = nullValue; } else { ipAsString = context.parser().text(); } } if (ipAsString == null) { return; } if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), ipAsString, boost); } final long value = ipToLong(ipAsString); if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { CustomLongNumericField field = new CustomLongNumericField(this, value, fieldType); field.setBoost(boost); fields.add(field); } if (hasDocValues()) { addDocValue(context, fields, value); } }
/** * Parse a field as though it were a string. * * @param context parse context used during parsing * @param nullValue value to use for null * @param defaultBoost default boost value returned unless overwritten in the field * @return the parsed field and the boost either parsed or defaulted * @throws IOException if thrown while parsing */ public static ValueAndBoost parseCreateFieldForString( ParseContext context, String nullValue, float defaultBoost) throws IOException { if (context.externalValueSet()) { return new ValueAndBoost(context.externalValue().toString(), defaultBoost); } XContentParser parser = context.parser(); if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { return new ValueAndBoost(nullValue, defaultBoost); } if (parser.currentToken() == XContentParser.Token.START_OBJECT && Version.indexCreated(context.indexSettings()).before(Version.V_3_0_0)) { XContentParser.Token token; String currentFieldName = null; String value = nullValue; float boost = defaultBoost; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { value = parser.textOrNull(); } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); } else { throw new IllegalArgumentException("unknown property [" + currentFieldName + "]"); } } } return new ValueAndBoost(value, boost); } return new ValueAndBoost(parser.textOrNull(), defaultBoost); }
@Override protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException { final Object value; if (context.externalValueSet()) { value = context.externalValue(); } else { value = context.parser().textOrNull(); } if (value != null) { final BytesRef bytes = new BytesRef(value.toString()); final long hash = MurmurHash3.hash128(bytes.bytes, bytes.offset, bytes.length, 0, new MurmurHash3.Hash128()) .h1; context.externalValue(hash); super.innerParseCreateField(context, fields); } }
@Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { Object addressAsObject; if (context.externalValueSet()) { addressAsObject = context.externalValue(); } else { addressAsObject = context.parser().textOrNull(); } if (addressAsObject == null) { addressAsObject = fieldType().nullValue(); } if (addressAsObject == null) { return; } String addressAsString = addressAsObject.toString(); InetAddress address; if (addressAsObject instanceof InetAddress) { address = (InetAddress) addressAsObject; } else { try { address = InetAddresses.forString(addressAsString); } catch (IllegalArgumentException e) { if (ignoreMalformed.value()) { return; } else { throw e; } } } if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(fieldType().name(), addressAsString, fieldType().boost()); } if (fieldType().indexOptions() != IndexOptions.NONE) { fields.add(new InetAddressPoint(fieldType().name(), address)); } if (fieldType().hasDocValues()) { fields.add( new SortedSetDocValuesField( fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); } if (fieldType().stored()) { fields.add( new StoredField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); } }
@Override protected Field parseCreateField(ParseContext context) throws IOException { String value = nullValue; float boost = this.boost; if (context.externalValueSet()) { value = (String) context.externalValue(); } else { XContentParser parser = context.parser(); if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { value = nullValue; } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) { XContentParser.Token token; String currentFieldName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { value = parser.textOrNull(); } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); } else { throw new ElasticSearchIllegalArgumentException( "unknown property [" + currentFieldName + "]"); } } } } else { value = parser.textOrNull(); } } if (value == null) { return null; } if (ignoreAbove > 0 && value.length() > ignoreAbove) { return null; } if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), value, boost); } if (!fieldType().indexed() && !fieldType().stored()) { context.ignoredValue(names.indexName(), value); return null; } Field field = new StringField(names.indexName(), value, fieldType); field.setBoost(boost); return field; }
@Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { final String value; if (context.externalValueSet()) { value = context.externalValue().toString(); } else { value = context.parser().textOrNull(); } if (value == null) { return; } if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(fieldType().name(), value, fieldType().boost()); } if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { Field field = new Field(fieldType().name(), value, fieldType()); fields.add(field); } }
@Override protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException { int value; float boost = this.boost; if (context.externalValueSet()) { Object externalValue = context.externalValue(); if (externalValue == null) { if (nullValue == null) { return; } value = nullValue; } else if (externalValue instanceof String) { String sExternalValue = (String) externalValue; if (sExternalValue.length() == 0) { if (nullValue == null) { return; } value = nullValue; } else { value = Integer.parseInt(sExternalValue); } } else { value = ((Number) externalValue).intValue(); } if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), Integer.toString(value), boost); } } else { XContentParser parser = context.parser(); if (parser.currentToken() == XContentParser.Token.VALUE_NULL || (parser.currentToken() == XContentParser.Token.VALUE_STRING && parser.textLength() == 0)) { if (nullValue == null) { return; } value = nullValue; if (nullValueAsString != null && (context.includeInAll(includeInAll, this))) { context.allEntries().addText(names.fullName(), nullValueAsString, boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) { XContentParser.Token token; String currentFieldName = null; Integer objValue = nullValue; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { objValue = parser.intValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); } else { throw new ElasticsearchIllegalArgumentException( "unknown property [" + currentFieldName + "]"); } } } if (objValue == null) { // no value return; } value = objValue; } else { value = parser.intValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } } } addIntegerFields(context, fields, value, boost); }
@Override protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException { byte value; float boost = fieldType().boost(); if (context.externalValueSet()) { Object externalValue = context.externalValue(); if (externalValue == null) { if (fieldType().nullValue() == null) { return; } value = fieldType().nullValue(); } else if (externalValue instanceof String) { String sExternalValue = (String) externalValue; if (sExternalValue.length() == 0) { if (fieldType().nullValue() == null) { return; } value = fieldType().nullValue(); } else { value = Byte.parseByte(sExternalValue); } } else { value = ((Number) externalValue).byteValue(); } if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(fieldType().name(), Byte.toString(value), boost); } } else { XContentParser parser = context.parser(); if (parser.currentToken() == XContentParser.Token.VALUE_NULL || (parser.currentToken() == XContentParser.Token.VALUE_STRING && parser.textLength() == 0)) { if (fieldType().nullValue() == null) { return; } value = fieldType().nullValue(); if (fieldType().nullValueAsString() != null && (context.includeInAll(includeInAll, this))) { context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT && Version.indexCreated(context.indexSettings()).before(Version.V_3_0_0)) { XContentParser.Token token; String currentFieldName = null; Byte objValue = fieldType().nullValue(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { objValue = (byte) parser.shortValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); } else { throw new IllegalArgumentException("unknown property [" + currentFieldName + "]"); } } } if (objValue == null) { // no value return; } value = objValue; } else { value = (byte) parser.shortValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(fieldType().name(), parser.text(), boost); } } } if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { CustomByteNumericField field = new CustomByteNumericField(value, fieldType()); field.setBoost(boost); fields.add(field); } if (fieldType().hasDocValues()) { addDocValue(context, fields, value); } }
@Override protected Field innerParseCreateField(ParseContext context) throws IOException { long value; float boost = this.boost; if (context.externalValueSet()) { Object externalValue = context.externalValue(); if (externalValue == null) { if (nullValue == null) { return null; } value = nullValue; } else if (externalValue instanceof String) { String sExternalValue = (String) externalValue; if (sExternalValue.length() == 0) { if (nullValue == null) { return null; } value = nullValue; } else { value = Long.parseLong(sExternalValue); } } else { value = ((Number) externalValue).longValue(); } if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), Long.toString(value), boost); } } else { XContentParser parser = context.parser(); if (parser.currentToken() == XContentParser.Token.VALUE_NULL || (parser.currentToken() == XContentParser.Token.VALUE_STRING && parser.textLength() == 0)) { if (nullValue == null) { return null; } value = nullValue; if (nullValueAsString != null && (context.includeInAll(includeInAll, this))) { context.allEntries().addText(names.fullName(), nullValueAsString, boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) { XContentParser.Token token; String currentFieldName = null; Long objValue = nullValue; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { objValue = parser.longValue(); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); } else { throw new ElasticSearchIllegalArgumentException( "unknown property [" + currentFieldName + "]"); } } } if (objValue == null) { // no value return null; } value = objValue; } else { value = parser.longValue(); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } } } CustomLongNumericField field = new CustomLongNumericField(this, value, fieldType); field.setBoost(boost); return field; }
@Override protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException { double value; float boost = this.boost; if (context.externalValueSet()) { Object externalValue = context.externalValue(); if (externalValue == null) { if (nullValue == null) { return; } value = nullValue; } else if (externalValue instanceof String) { String sExternalValue = (String) externalValue; if (sExternalValue.length() == 0) { if (nullValue == null) { return; } value = nullValue; } else { value = Double.parseDouble(sExternalValue); } } else { value = ((Number) externalValue).doubleValue(); } if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), Double.toString(value), boost); } } else { XContentParser parser = context.parser(); if (parser.currentToken() == XContentParser.Token.VALUE_NULL || (parser.currentToken() == XContentParser.Token.VALUE_STRING && parser.textLength() == 0)) { if (nullValue == null) { return; } value = nullValue; if (nullValueAsString != null && (context.includeInAll(includeInAll, this))) { context.allEntries().addText(names.fullName(), nullValueAsString, boost); } } else if (parser.currentToken() == XContentParser.Token.START_OBJECT) { XContentParser.Token token; String currentFieldName = null; Double objValue = nullValue; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { objValue = parser.doubleValue(coerce.value()); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); } else { throw new ElasticsearchIllegalArgumentException( "unknown property [" + currentFieldName + "]"); } } } if (objValue == null) { // no value return; } value = objValue; } else { value = parser.doubleValue(coerce.value()); if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), parser.text(), boost); } } } if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { CustomDoubleNumericField field = new CustomDoubleNumericField(this, value, fieldType); field.setBoost(boost); fields.add(field); } if (hasDocValues()) { if (useSortedNumericDocValues) { addDocValue(context, fields, NumericUtils.doubleToSortableLong(value)); } else { CustomDoubleNumericDocValuesField field = (CustomDoubleNumericDocValuesField) context.doc().getByKey(names().indexName()); if (field != null) { field.add(value); } else { field = new CustomDoubleNumericDocValuesField(names().indexName(), value); context.doc().addWithKey(names().indexName(), field); } } } }
@Override protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException { String dateAsString = null; Long value = null; float boost = this.boost; if (context.externalValueSet()) { Object externalValue = context.externalValue(); if (externalValue instanceof Number) { value = ((Number) externalValue).longValue(); } else { dateAsString = (String) externalValue; if (dateAsString == null) { dateAsString = nullValue; } } } else { XContentParser parser = context.parser(); XContentParser.Token token = parser.currentToken(); if (token == XContentParser.Token.VALUE_NULL) { dateAsString = nullValue; } else if (token == XContentParser.Token.VALUE_NUMBER) { value = parser.longValue(coerce.value()); } else if (token == XContentParser.Token.START_OBJECT) { String currentFieldName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { if (token == XContentParser.Token.VALUE_NULL) { dateAsString = nullValue; } else if (token == XContentParser.Token.VALUE_NUMBER) { value = parser.longValue(coerce.value()); } else { dateAsString = parser.text(); } } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { boost = parser.floatValue(); } else { throw new ElasticsearchIllegalArgumentException( "unknown property [" + currentFieldName + "]"); } } } } else { dateAsString = parser.text(); } } if (dateAsString != null) { assert value == null; if (context.includeInAll(includeInAll, this)) { context.allEntries().addText(names.fullName(), dateAsString, boost); } value = parseStringValue(dateAsString); } if (value != null) { if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { CustomLongNumericField field = new CustomLongNumericField(this, value, fieldType); field.setBoost(boost); fields.add(field); } if (hasDocValues()) { addDocValue(context, fields, value); } } }
@Override public Mapper parse(ParseContext context) throws IOException { byte[] content = null; String contentType = null; int indexedChars = defaultIndexedChars; boolean langDetect = defaultLangDetect; String name = null; String language = null; XContentParser parser = context.parser(); XContentParser.Token token = parser.currentToken(); if (token == XContentParser.Token.VALUE_STRING) { content = parser.binaryValue(); } else { String currentFieldName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.VALUE_STRING) { if ("_content".equals(currentFieldName)) { content = parser.binaryValue(); } else if ("_content_type".equals(currentFieldName)) { contentType = parser.text(); } else if ("_name".equals(currentFieldName)) { name = parser.text(); } else if ("_language".equals(currentFieldName)) { language = parser.text(); } } else if (token == XContentParser.Token.VALUE_NUMBER) { if ("_indexed_chars".equals(currentFieldName) || "_indexedChars".equals(currentFieldName)) { indexedChars = parser.intValue(); } } else if (token == XContentParser.Token.VALUE_BOOLEAN) { if ("_detect_language".equals(currentFieldName) || "_detectLanguage".equals(currentFieldName)) { langDetect = parser.booleanValue(); } } } } // Throw clean exception when no content is provided Fix #23 if (content == null) { throw new MapperParsingException("No content is provided."); } Metadata metadata = new Metadata(); if (contentType != null) { metadata.add(Metadata.CONTENT_TYPE, contentType); } if (name != null) { metadata.add(Metadata.RESOURCE_NAME_KEY, name); } String parsedContent; try { parsedContent = TikaImpl.parse(content, metadata, indexedChars); } catch (Throwable e) { // #18: we could ignore errors when Tika does not parse data if (!ignoreErrors) { logger.trace("exception caught", e); throw new MapperParsingException( "Failed to extract [" + indexedChars + "] characters of text for [" + name + "] : " + e.getMessage(), e); } else { logger.debug( "Failed to extract [{}] characters of text for [{}]: [{}]", indexedChars, name, e.getMessage()); logger.trace("exception caught", e); } return null; } context = context.createExternalValueContext(parsedContent); contentMapper.parse(context); if (langDetect) { try { if (language != null) { metadata.add(Metadata.CONTENT_LANGUAGE, language); } else { LanguageIdentifier identifier = new LanguageIdentifier(parsedContent); language = identifier.getLanguage(); } context = context.createExternalValueContext(language); languageMapper.parse(context); } catch (Throwable t) { logger.debug("Cannot detect language: [{}]", t.getMessage()); } } if (name != null) { try { context = context.createExternalValueContext(name); nameMapper.parse(context); } catch (MapperParsingException e) { if (!ignoreErrors) throw e; if (logger.isDebugEnabled()) logger.debug( "Ignoring MapperParsingException catch while parsing name: [{}]", e.getMessage()); } } if (metadata.get(Metadata.DATE) != null) { try { context = context.createExternalValueContext(metadata.get(Metadata.DATE)); dateMapper.parse(context); } catch (MapperParsingException e) { if (!ignoreErrors) throw e; if (logger.isDebugEnabled()) logger.debug( "Ignoring MapperParsingException catch while parsing date: [{}]: [{}]", e.getMessage(), context.externalValue()); } } if (metadata.get(Metadata.TITLE) != null) { try { context = context.createExternalValueContext(metadata.get(Metadata.TITLE)); titleMapper.parse(context); } catch (MapperParsingException e) { if (!ignoreErrors) throw e; if (logger.isDebugEnabled()) logger.debug( "Ignoring MapperParsingException catch while parsing title: [{}]: [{}]", e.getMessage(), context.externalValue()); } } if (metadata.get(Metadata.AUTHOR) != null) { try { context = context.createExternalValueContext(metadata.get(Metadata.AUTHOR)); authorMapper.parse(context); } catch (MapperParsingException e) { if (!ignoreErrors) throw e; if (logger.isDebugEnabled()) logger.debug( "Ignoring MapperParsingException catch while parsing author: [{}]: [{}]", e.getMessage(), context.externalValue()); } } if (metadata.get(Metadata.KEYWORDS) != null) { try { context = context.createExternalValueContext(metadata.get(Metadata.KEYWORDS)); keywordsMapper.parse(context); } catch (MapperParsingException e) { if (!ignoreErrors) throw e; if (logger.isDebugEnabled()) logger.debug( "Ignoring MapperParsingException catch while parsing keywords: [{}]: [{}]", e.getMessage(), context.externalValue()); } } if (contentType == null) { contentType = metadata.get(Metadata.CONTENT_TYPE); } if (contentType != null) { try { context = context.createExternalValueContext(contentType); contentTypeMapper.parse(context); } catch (MapperParsingException e) { if (!ignoreErrors) throw e; if (logger.isDebugEnabled()) logger.debug( "Ignoring MapperParsingException catch while parsing content_type: [{}]: [{}]", e.getMessage(), context.externalValue()); } } int length = content.length; // If we have CONTENT_LENGTH from Tika we use it if (metadata.get(Metadata.CONTENT_LENGTH) != null) { length = Integer.parseInt(metadata.get(Metadata.CONTENT_LENGTH)); } try { context = context.createExternalValueContext(length); contentLengthMapper.parse(context); } catch (MapperParsingException e) { if (!ignoreErrors) throw e; if (logger.isDebugEnabled()) logger.debug( "Ignoring MapperParsingException catch while parsing content_length: [{}]: [{}]", e.getMessage(), context.externalValue()); } // multiFields.parse(this, context); return null; }