@Override
protected void innerParseCreateField(ParseContext context, List<Field> fields)
throws IOException {
String ipAsString;
if (context.externalValueSet()) {
ipAsString = (String) context.externalValue();
if (ipAsString == null) {
ipAsString = nullValue;
}
} else {
if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) {
ipAsString = nullValue;
} else {
ipAsString = context.parser().text();
}
}
if (ipAsString == null) {
return;
}
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), ipAsString, boost);
}
final long value = ipToLong(ipAsString);
if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
CustomLongNumericField field = new CustomLongNumericField(this, value, fieldType);
field.setBoost(boost);
fields.add(field);
}
if (hasDocValues()) {
addDocValue(context, fields, value);
}
}
/**
* Parse a field as though it were a string.
*
* @param context parse context used during parsing
* @param nullValue value to use for null
* @param defaultBoost default boost value returned unless overwritten in the field
* @return the parsed field and the boost either parsed or defaulted
* @throws IOException if thrown while parsing
*/
public static ValueAndBoost parseCreateFieldForString(
ParseContext context, String nullValue, float defaultBoost) throws IOException {
if (context.externalValueSet()) {
return new ValueAndBoost(context.externalValue().toString(), defaultBoost);
}
XContentParser parser = context.parser();
if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
return new ValueAndBoost(nullValue, defaultBoost);
}
if (parser.currentToken() == XContentParser.Token.START_OBJECT
&& Version.indexCreated(context.indexSettings()).before(Version.V_3_0_0)) {
XContentParser.Token token;
String currentFieldName = null;
String value = nullValue;
float boost = defaultBoost;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
value = parser.textOrNull();
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
} else {
throw new IllegalArgumentException("unknown property [" + currentFieldName + "]");
}
}
}
return new ValueAndBoost(value, boost);
}
return new ValueAndBoost(parser.textOrNull(), defaultBoost);
}
@Override
protected void innerParseCreateField(ParseContext context, List<Field> fields)
throws IOException {
final Object value;
if (context.externalValueSet()) {
value = context.externalValue();
} else {
value = context.parser().textOrNull();
}
if (value != null) {
final BytesRef bytes = new BytesRef(value.toString());
final long hash =
MurmurHash3.hash128(bytes.bytes, bytes.offset, bytes.length, 0, new MurmurHash3.Hash128())
.h1;
context.externalValue(hash);
super.innerParseCreateField(context, fields);
}
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
Object addressAsObject;
if (context.externalValueSet()) {
addressAsObject = context.externalValue();
} else {
addressAsObject = context.parser().textOrNull();
}
if (addressAsObject == null) {
addressAsObject = fieldType().nullValue();
}
if (addressAsObject == null) {
return;
}
String addressAsString = addressAsObject.toString();
InetAddress address;
if (addressAsObject instanceof InetAddress) {
address = (InetAddress) addressAsObject;
} else {
try {
address = InetAddresses.forString(addressAsString);
} catch (IllegalArgumentException e) {
if (ignoreMalformed.value()) {
return;
} else {
throw e;
}
}
}
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(fieldType().name(), addressAsString, fieldType().boost());
}
if (fieldType().indexOptions() != IndexOptions.NONE) {
fields.add(new InetAddressPoint(fieldType().name(), address));
}
if (fieldType().hasDocValues()) {
fields.add(
new SortedSetDocValuesField(
fieldType().name(), new BytesRef(InetAddressPoint.encode(address))));
}
if (fieldType().stored()) {
fields.add(
new StoredField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address))));
}
}
@Override
protected Field parseCreateField(ParseContext context) throws IOException {
String value = nullValue;
float boost = this.boost;
if (context.externalValueSet()) {
value = (String) context.externalValue();
} else {
XContentParser parser = context.parser();
if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
value = nullValue;
} else if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
XContentParser.Token token;
String currentFieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
value = parser.textOrNull();
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
} else {
throw new ElasticSearchIllegalArgumentException(
"unknown property [" + currentFieldName + "]");
}
}
}
} else {
value = parser.textOrNull();
}
}
if (value == null) {
return null;
}
if (ignoreAbove > 0 && value.length() > ignoreAbove) {
return null;
}
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), value, boost);
}
if (!fieldType().indexed() && !fieldType().stored()) {
context.ignoredValue(names.indexName(), value);
return null;
}
Field field = new StringField(names.indexName(), value, fieldType);
field.setBoost(boost);
return field;
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
final String value;
if (context.externalValueSet()) {
value = context.externalValue().toString();
} else {
value = context.parser().textOrNull();
}
if (value == null) {
return;
}
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(fieldType().name(), value, fieldType().boost());
}
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
Field field = new Field(fieldType().name(), value, fieldType());
fields.add(field);
}
}
@Override
protected void innerParseCreateField(ParseContext context, List<Field> fields)
throws IOException {
int value;
float boost = this.boost;
if (context.externalValueSet()) {
Object externalValue = context.externalValue();
if (externalValue == null) {
if (nullValue == null) {
return;
}
value = nullValue;
} else if (externalValue instanceof String) {
String sExternalValue = (String) externalValue;
if (sExternalValue.length() == 0) {
if (nullValue == null) {
return;
}
value = nullValue;
} else {
value = Integer.parseInt(sExternalValue);
}
} else {
value = ((Number) externalValue).intValue();
}
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), Integer.toString(value), boost);
}
} else {
XContentParser parser = context.parser();
if (parser.currentToken() == XContentParser.Token.VALUE_NULL
|| (parser.currentToken() == XContentParser.Token.VALUE_STRING
&& parser.textLength() == 0)) {
if (nullValue == null) {
return;
}
value = nullValue;
if (nullValueAsString != null && (context.includeInAll(includeInAll, this))) {
context.allEntries().addText(names.fullName(), nullValueAsString, boost);
}
} else if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
XContentParser.Token token;
String currentFieldName = null;
Integer objValue = nullValue;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
objValue = parser.intValue(coerce.value());
}
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
} else {
throw new ElasticsearchIllegalArgumentException(
"unknown property [" + currentFieldName + "]");
}
}
}
if (objValue == null) {
// no value
return;
}
value = objValue;
} else {
value = parser.intValue(coerce.value());
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), parser.text(), boost);
}
}
}
addIntegerFields(context, fields, value, boost);
}
@Override
protected void innerParseCreateField(ParseContext context, List<Field> fields)
throws IOException {
byte value;
float boost = fieldType().boost();
if (context.externalValueSet()) {
Object externalValue = context.externalValue();
if (externalValue == null) {
if (fieldType().nullValue() == null) {
return;
}
value = fieldType().nullValue();
} else if (externalValue instanceof String) {
String sExternalValue = (String) externalValue;
if (sExternalValue.length() == 0) {
if (fieldType().nullValue() == null) {
return;
}
value = fieldType().nullValue();
} else {
value = Byte.parseByte(sExternalValue);
}
} else {
value = ((Number) externalValue).byteValue();
}
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(fieldType().name(), Byte.toString(value), boost);
}
} else {
XContentParser parser = context.parser();
if (parser.currentToken() == XContentParser.Token.VALUE_NULL
|| (parser.currentToken() == XContentParser.Token.VALUE_STRING
&& parser.textLength() == 0)) {
if (fieldType().nullValue() == null) {
return;
}
value = fieldType().nullValue();
if (fieldType().nullValueAsString() != null && (context.includeInAll(includeInAll, this))) {
context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost);
}
} else if (parser.currentToken() == XContentParser.Token.START_OBJECT
&& Version.indexCreated(context.indexSettings()).before(Version.V_3_0_0)) {
XContentParser.Token token;
String currentFieldName = null;
Byte objValue = fieldType().nullValue();
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
objValue = (byte) parser.shortValue(coerce.value());
}
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
} else {
throw new IllegalArgumentException("unknown property [" + currentFieldName + "]");
}
}
}
if (objValue == null) {
// no value
return;
}
value = objValue;
} else {
value = (byte) parser.shortValue(coerce.value());
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(fieldType().name(), parser.text(), boost);
}
}
}
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
CustomByteNumericField field = new CustomByteNumericField(value, fieldType());
field.setBoost(boost);
fields.add(field);
}
if (fieldType().hasDocValues()) {
addDocValue(context, fields, value);
}
}
@Override
protected Field innerParseCreateField(ParseContext context) throws IOException {
long value;
float boost = this.boost;
if (context.externalValueSet()) {
Object externalValue = context.externalValue();
if (externalValue == null) {
if (nullValue == null) {
return null;
}
value = nullValue;
} else if (externalValue instanceof String) {
String sExternalValue = (String) externalValue;
if (sExternalValue.length() == 0) {
if (nullValue == null) {
return null;
}
value = nullValue;
} else {
value = Long.parseLong(sExternalValue);
}
} else {
value = ((Number) externalValue).longValue();
}
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), Long.toString(value), boost);
}
} else {
XContentParser parser = context.parser();
if (parser.currentToken() == XContentParser.Token.VALUE_NULL
|| (parser.currentToken() == XContentParser.Token.VALUE_STRING
&& parser.textLength() == 0)) {
if (nullValue == null) {
return null;
}
value = nullValue;
if (nullValueAsString != null && (context.includeInAll(includeInAll, this))) {
context.allEntries().addText(names.fullName(), nullValueAsString, boost);
}
} else if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
XContentParser.Token token;
String currentFieldName = null;
Long objValue = nullValue;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
objValue = parser.longValue();
}
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
} else {
throw new ElasticSearchIllegalArgumentException(
"unknown property [" + currentFieldName + "]");
}
}
}
if (objValue == null) {
// no value
return null;
}
value = objValue;
} else {
value = parser.longValue();
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), parser.text(), boost);
}
}
}
CustomLongNumericField field = new CustomLongNumericField(this, value, fieldType);
field.setBoost(boost);
return field;
}
@Override
protected void innerParseCreateField(ParseContext context, List<Field> fields)
throws IOException {
double value;
float boost = this.boost;
if (context.externalValueSet()) {
Object externalValue = context.externalValue();
if (externalValue == null) {
if (nullValue == null) {
return;
}
value = nullValue;
} else if (externalValue instanceof String) {
String sExternalValue = (String) externalValue;
if (sExternalValue.length() == 0) {
if (nullValue == null) {
return;
}
value = nullValue;
} else {
value = Double.parseDouble(sExternalValue);
}
} else {
value = ((Number) externalValue).doubleValue();
}
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), Double.toString(value), boost);
}
} else {
XContentParser parser = context.parser();
if (parser.currentToken() == XContentParser.Token.VALUE_NULL
|| (parser.currentToken() == XContentParser.Token.VALUE_STRING
&& parser.textLength() == 0)) {
if (nullValue == null) {
return;
}
value = nullValue;
if (nullValueAsString != null && (context.includeInAll(includeInAll, this))) {
context.allEntries().addText(names.fullName(), nullValueAsString, boost);
}
} else if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
XContentParser.Token token;
String currentFieldName = null;
Double objValue = nullValue;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
objValue = parser.doubleValue(coerce.value());
}
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
} else {
throw new ElasticsearchIllegalArgumentException(
"unknown property [" + currentFieldName + "]");
}
}
}
if (objValue == null) {
// no value
return;
}
value = objValue;
} else {
value = parser.doubleValue(coerce.value());
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), parser.text(), boost);
}
}
}
if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
CustomDoubleNumericField field = new CustomDoubleNumericField(this, value, fieldType);
field.setBoost(boost);
fields.add(field);
}
if (hasDocValues()) {
if (useSortedNumericDocValues) {
addDocValue(context, fields, NumericUtils.doubleToSortableLong(value));
} else {
CustomDoubleNumericDocValuesField field =
(CustomDoubleNumericDocValuesField) context.doc().getByKey(names().indexName());
if (field != null) {
field.add(value);
} else {
field = new CustomDoubleNumericDocValuesField(names().indexName(), value);
context.doc().addWithKey(names().indexName(), field);
}
}
}
}
@Override
protected void innerParseCreateField(ParseContext context, List<Field> fields)
throws IOException {
String dateAsString = null;
Long value = null;
float boost = this.boost;
if (context.externalValueSet()) {
Object externalValue = context.externalValue();
if (externalValue instanceof Number) {
value = ((Number) externalValue).longValue();
} else {
dateAsString = (String) externalValue;
if (dateAsString == null) {
dateAsString = nullValue;
}
}
} else {
XContentParser parser = context.parser();
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.VALUE_NULL) {
dateAsString = nullValue;
} else if (token == XContentParser.Token.VALUE_NUMBER) {
value = parser.longValue(coerce.value());
} else if (token == XContentParser.Token.START_OBJECT) {
String currentFieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else {
if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
if (token == XContentParser.Token.VALUE_NULL) {
dateAsString = nullValue;
} else if (token == XContentParser.Token.VALUE_NUMBER) {
value = parser.longValue(coerce.value());
} else {
dateAsString = parser.text();
}
} else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) {
boost = parser.floatValue();
} else {
throw new ElasticsearchIllegalArgumentException(
"unknown property [" + currentFieldName + "]");
}
}
}
} else {
dateAsString = parser.text();
}
}
if (dateAsString != null) {
assert value == null;
if (context.includeInAll(includeInAll, this)) {
context.allEntries().addText(names.fullName(), dateAsString, boost);
}
value = parseStringValue(dateAsString);
}
if (value != null) {
if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
CustomLongNumericField field = new CustomLongNumericField(this, value, fieldType);
field.setBoost(boost);
fields.add(field);
}
if (hasDocValues()) {
addDocValue(context, fields, value);
}
}
}
@Override
public Mapper parse(ParseContext context) throws IOException {
byte[] content = null;
String contentType = null;
int indexedChars = defaultIndexedChars;
boolean langDetect = defaultLangDetect;
String name = null;
String language = null;
XContentParser parser = context.parser();
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.VALUE_STRING) {
content = parser.binaryValue();
} else {
String currentFieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_STRING) {
if ("_content".equals(currentFieldName)) {
content = parser.binaryValue();
} else if ("_content_type".equals(currentFieldName)) {
contentType = parser.text();
} else if ("_name".equals(currentFieldName)) {
name = parser.text();
} else if ("_language".equals(currentFieldName)) {
language = parser.text();
}
} else if (token == XContentParser.Token.VALUE_NUMBER) {
if ("_indexed_chars".equals(currentFieldName)
|| "_indexedChars".equals(currentFieldName)) {
indexedChars = parser.intValue();
}
} else if (token == XContentParser.Token.VALUE_BOOLEAN) {
if ("_detect_language".equals(currentFieldName)
|| "_detectLanguage".equals(currentFieldName)) {
langDetect = parser.booleanValue();
}
}
}
}
// Throw clean exception when no content is provided Fix #23
if (content == null) {
throw new MapperParsingException("No content is provided.");
}
Metadata metadata = new Metadata();
if (contentType != null) {
metadata.add(Metadata.CONTENT_TYPE, contentType);
}
if (name != null) {
metadata.add(Metadata.RESOURCE_NAME_KEY, name);
}
String parsedContent;
try {
parsedContent = TikaImpl.parse(content, metadata, indexedChars);
} catch (Throwable e) {
// #18: we could ignore errors when Tika does not parse data
if (!ignoreErrors) {
logger.trace("exception caught", e);
throw new MapperParsingException(
"Failed to extract ["
+ indexedChars
+ "] characters of text for ["
+ name
+ "] : "
+ e.getMessage(),
e);
} else {
logger.debug(
"Failed to extract [{}] characters of text for [{}]: [{}]",
indexedChars,
name,
e.getMessage());
logger.trace("exception caught", e);
}
return null;
}
context = context.createExternalValueContext(parsedContent);
contentMapper.parse(context);
if (langDetect) {
try {
if (language != null) {
metadata.add(Metadata.CONTENT_LANGUAGE, language);
} else {
LanguageIdentifier identifier = new LanguageIdentifier(parsedContent);
language = identifier.getLanguage();
}
context = context.createExternalValueContext(language);
languageMapper.parse(context);
} catch (Throwable t) {
logger.debug("Cannot detect language: [{}]", t.getMessage());
}
}
if (name != null) {
try {
context = context.createExternalValueContext(name);
nameMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing name: [{}]", e.getMessage());
}
}
if (metadata.get(Metadata.DATE) != null) {
try {
context = context.createExternalValueContext(metadata.get(Metadata.DATE));
dateMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing date: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
if (metadata.get(Metadata.TITLE) != null) {
try {
context = context.createExternalValueContext(metadata.get(Metadata.TITLE));
titleMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing title: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
if (metadata.get(Metadata.AUTHOR) != null) {
try {
context = context.createExternalValueContext(metadata.get(Metadata.AUTHOR));
authorMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing author: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
if (metadata.get(Metadata.KEYWORDS) != null) {
try {
context = context.createExternalValueContext(metadata.get(Metadata.KEYWORDS));
keywordsMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing keywords: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
if (contentType == null) {
contentType = metadata.get(Metadata.CONTENT_TYPE);
}
if (contentType != null) {
try {
context = context.createExternalValueContext(contentType);
contentTypeMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing content_type: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
}
int length = content.length;
// If we have CONTENT_LENGTH from Tika we use it
if (metadata.get(Metadata.CONTENT_LENGTH) != null) {
length = Integer.parseInt(metadata.get(Metadata.CONTENT_LENGTH));
}
try {
context = context.createExternalValueContext(length);
contentLengthMapper.parse(context);
} catch (MapperParsingException e) {
if (!ignoreErrors) throw e;
if (logger.isDebugEnabled())
logger.debug(
"Ignoring MapperParsingException catch while parsing content_length: [{}]: [{}]",
e.getMessage(),
context.externalValue());
}
// multiFields.parse(this, context);
return null;
}