private long parseValue(Object value) {
if (value instanceof Number) {
return ((Number) value).longValue();
}
if (value instanceof BytesRef) {
return dateTimeFormatter.parser().parseMillis(((BytesRef) value).utf8ToString());
}
return dateTimeFormatter.parser().parseMillis(value.toString());
}
@Override
public DateFieldMapper build(BuilderContext context) {
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
if (!locale.equals(dateTimeFormatter.locale())) {
dateTimeFormatter =
new FormatDateTimeFormatter(
dateTimeFormatter.format(),
dateTimeFormatter.parser(),
dateTimeFormatter.printer(),
locale);
}
DateFieldMapper fieldMapper =
new DateFieldMapper(
buildNames(context),
dateTimeFormatter,
fieldType.numericPrecisionStep(),
boost,
fieldType,
docValues,
nullValue,
timeUnit,
ignoreMalformed(context),
coerce(context),
postingsProvider,
docValuesProvider,
similarity,
normsLoading,
fieldDataSettings,
context.indexSettings(),
multiFieldsBuilder.build(this, context),
copyTo);
fieldMapper.includeInAll(includeInAll);
return fieldMapper;
}
@Test
public void testSlashInFormat() {
FormatDateTimeFormatter formatter = Joda.forPattern("MM/yyyy");
formatter.parser().parseMillis("01/2001");
formatter = Joda.forPattern("yyyy/MM/dd HH:mm:ss");
long millis = formatter.parser().parseMillis("1970/01/01 00:00:00");
formatter.printer().print(millis);
try {
millis = formatter.parser().parseMillis("1970/01/01");
assert false;
} catch (IllegalArgumentException e) {
// it really can't parse this one
}
}
@Test
public void testMultipleDifferentFormats() {
FormatDateTimeFormatter formatter = Joda.forPattern("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd");
String input = "1970/01/01 00:00:00";
long millis = formatter.parser().parseMillis(input);
assertThat(input, is(formatter.printer().print(millis)));
Joda.forPattern("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd||dateOptionalTime");
Joda.forPattern("dateOptionalTime||yyyy/MM/dd HH:mm:ss||yyyy/MM/dd");
Joda.forPattern("yyyy/MM/dd HH:mm:ss||dateOptionalTime||yyyy/MM/dd");
Joda.forPattern("date_time||date_time_no_millis");
Joda.forPattern(" date_time || date_time_no_millis");
}
@Test
public void testIsoVsCustom() {
DateTimeFormatter formatter =
ISODateTimeFormat.dateOptionalTimeParser().withZone(DateTimeZone.UTC);
long millis = formatter.parseMillis("1970-01-01T00:00:00");
assertThat(millis, equalTo(0l));
formatter = DateTimeFormat.forPattern("yyyy/MM/dd HH:mm:ss").withZone(DateTimeZone.UTC);
millis = formatter.parseMillis("1970/01/01 00:00:00");
assertThat(millis, equalTo(0l));
FormatDateTimeFormatter formatter2 = Joda.forPattern("yyyy/MM/dd HH:mm:ss");
millis = formatter2.parser().parseMillis("1970/01/01 00:00:00");
assertThat(millis, equalTo(0l));
}
private long parseStringValue(String value) {
try {
return dateTimeFormatter.parser().parseMillis(value);
} catch (RuntimeException e) {
try {
long time = Long.parseLong(value);
return timeUnit.toMillis(time);
} catch (NumberFormatException e1) {
throw new MapperParsingException(
"failed to parse date field ["
+ value
+ "], tried both date format ["
+ dateTimeFormatter.format()
+ "], and timestamp number with locale ["
+ dateTimeFormatter.locale()
+ "]",
e);
}
}
}
private void serializeValue(
final ParseContext context, String currentFieldName, XContentParser.Token token)
throws IOException {
if (currentFieldName == null) {
throw new MapperParsingException(
"object mapping ["
+ name
+ "] trying to serialize a value with no field associated with it, current value ["
+ context.parser().textOrNull()
+ "]");
}
Mapper mapper = mappers.get(currentFieldName);
if (mapper != null) {
mapper.parse(context);
return;
}
Dynamic dynamic = this.dynamic;
if (dynamic == null) {
dynamic = context.root().dynamic();
}
if (dynamic == Dynamic.STRICT) {
throw new StrictDynamicMappingException(fullPath, currentFieldName);
}
if (dynamic == Dynamic.FALSE) {
return;
}
// we sync here since we don't want to add this field twice to the document mapper
// its not the end of the world, since we add it to the mappers once we create it
// so next time we won't even get here for this field
boolean newMapper = false;
synchronized (mutex) {
mapper = mappers.get(currentFieldName);
if (mapper == null) {
newMapper = true;
BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path());
if (token == XContentParser.Token.VALUE_STRING) {
boolean resolved = false;
// do a quick test to see if its fits a dynamic template, if so, use it.
// we need to do it here so we can handle things like attachment templates, where calling
// text (to see if its a date) causes the binary value to be cleared
if (!resolved) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "string", null);
if (builder != null) {
mapper = builder.build(builderContext);
resolved = true;
}
}
if (!resolved && context.parser().textLength() == 0) {
// empty string with no mapping, treat it like null value
return;
}
if (!resolved && context.root().dateDetection()) {
String text = context.parser().text();
// a safe check since "1" gets parsed as well
if (Strings.countOccurrencesOf(text, ":") > 1
|| Strings.countOccurrencesOf(text, "-") > 1
|| Strings.countOccurrencesOf(text, "/") > 1) {
for (FormatDateTimeFormatter dateTimeFormatter :
context.root().dynamicDateTimeFormatters()) {
try {
dateTimeFormatter.parser().parseMillis(text);
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "date");
if (builder == null) {
builder = dateField(currentFieldName).dateTimeFormatter(dateTimeFormatter);
}
mapper = builder.build(builderContext);
resolved = true;
break;
} catch (Exception e) {
// failure to parse this, continue
}
}
}
}
if (!resolved && context.root().numericDetection()) {
String text = context.parser().text();
try {
Long.parseLong(text);
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "long");
if (builder == null) {
builder = longField(currentFieldName);
}
mapper = builder.build(builderContext);
resolved = true;
} catch (Exception e) {
// not a long number
}
if (!resolved) {
try {
Double.parseDouble(text);
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "double");
if (builder == null) {
builder = doubleField(currentFieldName);
}
mapper = builder.build(builderContext);
resolved = true;
} catch (Exception e) {
// not a long number
}
}
}
// DON'T do automatic ip detection logic, since it messes up with docs that have hosts and
// ips
// check if its an ip
// if (!resolved && text.indexOf('.') != -1) {
// try {
// IpFieldMapper.ipToLong(text);
// XContentMapper.Builder builder =
// context.root().findTemplateBuilder(context, currentFieldName, "ip");
// if (builder == null) {
// builder = ipField(currentFieldName);
// }
// mapper = builder.build(builderContext);
// resolved = true;
// } catch (Exception e) {
// // failure to parse, not ip...
// }
// }
if (!resolved) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "string");
if (builder == null) {
builder = stringField(currentFieldName);
}
mapper = builder.build(builderContext);
}
} else if (token == XContentParser.Token.VALUE_NUMBER) {
XContentParser.NumberType numberType = context.parser().numberType();
if (numberType == XContentParser.NumberType.INT) {
if (context.parser().estimatedNumberType()) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "long");
if (builder == null) {
builder = longField(currentFieldName);
}
mapper = builder.build(builderContext);
} else {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "integer");
if (builder == null) {
builder = integerField(currentFieldName);
}
mapper = builder.build(builderContext);
}
} else if (numberType == XContentParser.NumberType.LONG) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "long");
if (builder == null) {
builder = longField(currentFieldName);
}
mapper = builder.build(builderContext);
} else if (numberType == XContentParser.NumberType.FLOAT) {
if (context.parser().estimatedNumberType()) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "double");
if (builder == null) {
builder = doubleField(currentFieldName);
}
mapper = builder.build(builderContext);
} else {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "float");
if (builder == null) {
builder = floatField(currentFieldName);
}
mapper = builder.build(builderContext);
}
} else if (numberType == XContentParser.NumberType.DOUBLE) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "double");
if (builder == null) {
builder = doubleField(currentFieldName);
}
mapper = builder.build(builderContext);
}
} else if (token == XContentParser.Token.VALUE_BOOLEAN) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "boolean");
if (builder == null) {
builder = booleanField(currentFieldName);
}
mapper = builder.build(builderContext);
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "binary");
if (builder == null) {
builder = binaryField(currentFieldName);
}
mapper = builder.build(builderContext);
} else {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, null);
if (builder != null) {
mapper = builder.build(builderContext);
} else {
// TODO how do we identify dynamically that its a binary value?
throw new ElasticSearchIllegalStateException(
"Can't handle serializing a dynamic type with content token ["
+ token
+ "] and field name ["
+ currentFieldName
+ "]");
}
}
putMapper(mapper);
context.setMappingsModified();
}
}
if (newMapper) {
mapper.traverse(context.newFieldMappers());
}
mapper.parse(context);
}
private static Mapper.Builder<?, ?> createBuilderFromDynamicValue(
final ParseContext context, XContentParser.Token token, String currentFieldName)
throws IOException {
if (token == XContentParser.Token.VALUE_STRING) {
// do a quick test to see if its fits a dynamic template, if so, use it.
// we need to do it here so we can handle things like attachment templates, where calling
// text (to see if its a date) causes the binary value to be cleared
{
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "string", null);
if (builder != null) {
return builder;
}
}
if (context.root().dateDetection()) {
String text = context.parser().text();
// a safe check since "1" gets parsed as well
if (Strings.countOccurrencesOf(text, ":") > 1
|| Strings.countOccurrencesOf(text, "-") > 1
|| Strings.countOccurrencesOf(text, "/") > 1) {
for (FormatDateTimeFormatter dateTimeFormatter :
context.root().dynamicDateTimeFormatters()) {
try {
dateTimeFormatter.parser().parseMillis(text);
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "date");
if (builder == null) {
builder =
MapperBuilders.dateField(currentFieldName).dateTimeFormatter(dateTimeFormatter);
}
return builder;
} catch (Exception e) {
// failure to parse this, continue
}
}
}
}
if (context.root().numericDetection()) {
String text = context.parser().text();
try {
Long.parseLong(text);
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "long");
if (builder == null) {
builder = MapperBuilders.longField(currentFieldName);
}
return builder;
} catch (NumberFormatException e) {
// not a long number
}
try {
Double.parseDouble(text);
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "double");
if (builder == null) {
builder = MapperBuilders.doubleField(currentFieldName);
}
return builder;
} catch (NumberFormatException e) {
// not a long number
}
}
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "string");
if (builder == null) {
builder = MapperBuilders.stringField(currentFieldName);
}
return builder;
} else if (token == XContentParser.Token.VALUE_NUMBER) {
XContentParser.NumberType numberType = context.parser().numberType();
if (numberType == XContentParser.NumberType.INT
|| numberType == XContentParser.NumberType.LONG) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "long");
if (builder == null) {
builder = MapperBuilders.longField(currentFieldName);
}
return builder;
} else if (numberType == XContentParser.NumberType.FLOAT
|| numberType == XContentParser.NumberType.DOUBLE) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "double");
if (builder == null) {
// no templates are defined, we use float by default instead of double
// since this is much more space-efficient and should be enough most of
// the time
builder = MapperBuilders.floatField(currentFieldName);
}
return builder;
}
} else if (token == XContentParser.Token.VALUE_BOOLEAN) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "boolean");
if (builder == null) {
builder = MapperBuilders.booleanField(currentFieldName);
}
return builder;
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "binary");
if (builder == null) {
builder = MapperBuilders.binaryField(currentFieldName);
}
return builder;
} else {
Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, null);
if (builder != null) {
return builder;
}
}
// TODO how do we identify dynamically that its a binary value?
throw new IllegalStateException(
"Can't handle serializing a dynamic type with content token ["
+ token
+ "] and field name ["
+ currentFieldName
+ "]");
}
public void parseDynamicValue(
final ParseContext context, String currentFieldName, XContentParser.Token token)
throws IOException {
Dynamic dynamic = this.dynamic;
if (dynamic == null) {
dynamic = context.root().dynamic();
}
if (dynamic == Dynamic.STRICT) {
throw new StrictDynamicMappingException(fullPath, currentFieldName);
}
if (dynamic == Dynamic.FALSE) {
return;
}
// we sync here since we don't want to add this field twice to the document mapper
// its not the end of the world, since we add it to the mappers once we create it
// so next time we won't even get here for this field
synchronized (mutex) {
Mapper mapper = mappers.get(currentFieldName);
if (mapper == null) {
BuilderContext builderContext = new BuilderContext(context.indexSettings(), context.path());
if (token == XContentParser.Token.VALUE_STRING) {
boolean resolved = false;
// do a quick test to see if its fits a dynamic template, if so, use it.
// we need to do it here so we can handle things like attachment templates, where calling
// text (to see if its a date) causes the binary value to be cleared
if (!resolved) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "string", null);
if (builder != null) {
mapper = builder.build(builderContext);
resolved = true;
}
}
if (!resolved && context.parser().textLength() == 0) {
// empty string with no mapping, treat it like null value
return;
}
if (!resolved && context.root().dateDetection()) {
String text = context.parser().text();
// a safe check since "1" gets parsed as well
if (Strings.countOccurrencesOf(text, ":") > 1
|| Strings.countOccurrencesOf(text, "-") > 1
|| Strings.countOccurrencesOf(text, "/") > 1) {
for (FormatDateTimeFormatter dateTimeFormatter :
context.root().dynamicDateTimeFormatters()) {
try {
dateTimeFormatter.parser().parseMillis(text);
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "date");
if (builder == null) {
builder = dateField(currentFieldName).dateTimeFormatter(dateTimeFormatter);
}
mapper = builder.build(builderContext);
resolved = true;
break;
} catch (Exception e) {
// failure to parse this, continue
}
}
}
}
if (!resolved && context.root().numericDetection()) {
String text = context.parser().text();
try {
Long.parseLong(text);
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "long");
if (builder == null) {
builder = longField(currentFieldName);
}
mapper = builder.build(builderContext);
resolved = true;
} catch (Exception e) {
// not a long number
}
if (!resolved) {
try {
Double.parseDouble(text);
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "double");
if (builder == null) {
builder = doubleField(currentFieldName);
}
mapper = builder.build(builderContext);
resolved = true;
} catch (Exception e) {
// not a long number
}
}
}
if (!resolved) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "string");
if (builder == null) {
builder = stringField(currentFieldName);
}
mapper = builder.build(builderContext);
}
} else if (token == XContentParser.Token.VALUE_NUMBER) {
XContentParser.NumberType numberType = context.parser().numberType();
if (numberType == XContentParser.NumberType.INT) {
if (context.parser().estimatedNumberType()) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "long");
if (builder == null) {
builder = longField(currentFieldName);
}
mapper = builder.build(builderContext);
} else {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "integer");
if (builder == null) {
builder = integerField(currentFieldName);
}
mapper = builder.build(builderContext);
}
} else if (numberType == XContentParser.NumberType.LONG) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "long");
if (builder == null) {
builder = longField(currentFieldName);
}
mapper = builder.build(builderContext);
} else if (numberType == XContentParser.NumberType.FLOAT) {
if (context.parser().estimatedNumberType()) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "double");
if (builder == null) {
builder = doubleField(currentFieldName);
}
mapper = builder.build(builderContext);
} else {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "float");
if (builder == null) {
builder = floatField(currentFieldName);
}
mapper = builder.build(builderContext);
}
} else if (numberType == XContentParser.NumberType.DOUBLE) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "double");
if (builder == null) {
builder = doubleField(currentFieldName);
}
mapper = builder.build(builderContext);
}
} else if (token == XContentParser.Token.VALUE_BOOLEAN) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "boolean");
if (builder == null) {
builder = booleanField(currentFieldName);
}
mapper = builder.build(builderContext);
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, "binary");
if (builder == null) {
builder = binaryField(currentFieldName);
}
mapper = builder.build(builderContext);
} else {
Mapper.Builder builder =
context.root().findTemplateBuilder(context, currentFieldName, null);
if (builder != null) {
mapper = builder.build(builderContext);
} else {
// TODO how do we identify dynamically that its a binary value?
throw new ElasticsearchIllegalStateException(
"Can't handle serializing a dynamic type with content token ["
+ token
+ "] and field name ["
+ currentFieldName
+ "]");
}
}
if (context.isWithinNewMapper()) {
mapper.parse(context);
} else {
context.setWithinNewMapper();
try {
mapper.parse(context);
FieldMapperListener.Aggregator newFields = new FieldMapperListener.Aggregator();
mapper.traverse(newFields);
context.docMapper().addFieldMappers(newFields.mappers);
} finally {
context.clearWithinNewMapper();
}
}
// only put after we traversed and did the callbacks, so other parsing won't see it only
// after we
// properly traversed it and adding the mappers
putMapper(mapper);
context.setMappingsModified();
} else {
mapper.parse(context);
}
}
}
@Test
public void testMultipleFormats() {
FormatDateTimeFormatter formatter = Joda.forPattern("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd");
long millis = formatter.parser().parseMillis("1970/01/01 00:00:00");
assertThat("1970/01/01 00:00:00", is(formatter.printer().print(millis)));
}