@Override
public boolean equals(Object other) {
if (other == this) return true;
if (!(other instanceof Nested)) return false;
Nested o = (Nested) other;
return WireInternal.equals(unknownFields(), o.unknownFields())
&& WireInternal.equals(value, o.value);
}
protected static void parseNested(
String name, Map<String, Object> node, ObjectMapper.Builder builder) {
boolean nested = false;
boolean nestedIncludeInParent = false;
boolean nestedIncludeInRoot = false;
Object fieldNode = node.get("type");
if (fieldNode != null) {
String type = fieldNode.toString();
if (type.equals(CONTENT_TYPE)) {
builder.nested = Nested.NO;
} else if (type.equals(NESTED_CONTENT_TYPE)) {
nested = true;
} else {
throw new MapperParsingException(
"Trying to parse an object but has a different type ["
+ type
+ "] for ["
+ name
+ "]");
}
}
fieldNode = node.get("include_in_parent");
if (fieldNode != null) {
nestedIncludeInParent = nodeBooleanValue(fieldNode);
node.remove("include_in_parent");
}
fieldNode = node.get("include_in_root");
if (fieldNode != null) {
nestedIncludeInRoot = nodeBooleanValue(fieldNode);
node.remove("include_in_root");
}
if (nested) {
builder.nested = Nested.newNested(nestedIncludeInParent, nestedIncludeInRoot);
}
}
@Override
protected ObjectMapper createMapper(
String name,
String fullPath,
boolean enabled,
Nested nested,
Dynamic dynamic,
ContentPath.Type pathType,
Map<String, Mapper> mappers,
@Nullable @IndexSettings Settings settings) {
assert !nested.isNested();
FormatDateTimeFormatter[] dates = null;
if (dynamicDateTimeFormatters == null) {
dates = new FormatDateTimeFormatter[0];
} else if (dynamicDateTimeFormatters.isEmpty()) {
// add the default one
dates = Defaults.DYNAMIC_DATE_TIME_FORMATTERS;
} else {
dates =
dynamicDateTimeFormatters.toArray(
new FormatDateTimeFormatter[dynamicDateTimeFormatters.size()]);
}
return new RootObjectMapper(
name,
enabled,
dynamic,
pathType,
mappers,
dates,
dynamicTemplates.toArray(new DynamicTemplate[dynamicTemplates.size()]),
dateDetection,
numericDetection,
settings);
}
@Test
public void deepCopyMutateUtf8() {
Nested n = new Nested();
Utf8 utf8 = new Utf8("Sample");
n.name = utf8;
n.value = 12;
Nested copy = new Nested();
RecordUtils.copy(n, copy);
modify(n.name);
n.value = null;
// but our copies should NOT be affected
assertEquals("Sample", copy.name.toString());
assertEquals(12, (int) copy.value);
}
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
Map<String, Object> objectNode = node;
ObjectMapper.Builder builder = createBuilder(name);
boolean nested = false;
boolean nestedIncludeInParent = false;
boolean nestedIncludeInRoot = false;
for (Map.Entry<String, Object> entry : objectNode.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("dynamic")) {
String value = fieldNode.toString();
if (value.equalsIgnoreCase("strict")) {
builder.dynamic(Dynamic.STRICT);
} else {
builder.dynamic(nodeBooleanValue(fieldNode) ? Dynamic.TRUE : Dynamic.FALSE);
}
} else if (fieldName.equals("type")) {
String type = fieldNode.toString();
if (type.equals(CONTENT_TYPE)) {
builder.nested = Nested.NO;
} else if (type.equals(NESTED_CONTENT_TYPE)) {
nested = true;
} else {
throw new MapperParsingException(
"Trying to parse an object but has a different type ["
+ type
+ "] for ["
+ name
+ "]");
}
} else if (fieldName.equals("include_in_parent")) {
nestedIncludeInParent = nodeBooleanValue(fieldNode);
} else if (fieldName.equals("include_in_root")) {
nestedIncludeInRoot = nodeBooleanValue(fieldNode);
} else if (fieldName.equals("enabled")) {
builder.enabled(nodeBooleanValue(fieldNode));
} else if (fieldName.equals("path")) {
builder.pathType(parsePathType(name, fieldNode.toString()));
} else if (fieldName.equals("properties")) {
parseProperties(builder, (Map<String, Object>) fieldNode, parserContext);
} else if (fieldName.equals("include_in_all")) {
builder.includeInAll(nodeBooleanValue(fieldNode));
} else {
processField(builder, fieldName, fieldNode);
}
}
if (nested) {
builder.nested = Nested.newNested(nestedIncludeInParent, nestedIncludeInRoot);
}
return builder;
}
@Override
public int hashCode() {
int result = super.hashCode;
if (result == 0) {
result = unknownFields().hashCode();
result = result * 37 + (foo != null ? foo.hashCode() : 0);
result = result * 37 + (bar != null ? bar.hashCode() : 0);
result = result * 37 + (baz != null ? baz.hashCode() : 0);
result = result * 37 + (qux != null ? qux.hashCode() : 0);
result = result * 37 + (fred != null ? fred.hashCode() : 1);
result = result * 37 + (daisy != null ? daisy.hashCode() : 0);
result = result * 37 + (nested != null ? nested.hashCode() : 1);
result = result * 37 + (ext != null ? ext.hashCode() : 0);
result = result * 37 + (rep != null ? rep.hashCode() : 1);
super.hashCode = result;
}
return result;
}
@Test
public void deepCopyMutateList() {
Gend in = new Gend();
in.collection = new ArrayList<Nested>();
Nested n1 = new Nested();
n1.name = new Utf8("one");
n1.value = 11;
in.collection.add(n1);
Nested n2 = new Nested();
n2.name = new Utf8("two");
in.collection.add(n2);
Gend out = new Gend();
RecordUtils.copy(in, out);
n1.name = null;
modify(n2.name);
in.collection.clear();
assertEquals(2, out.collection.size());
assertEquals("one", out.collection.get(0).name.toString());
assertEquals(11, (int) out.collection.get(0).value);
assertNull(out.collection.get(1).value);
}
public void testResetRootDocId() throws Exception {
Directory directory = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(null);
iwc.setMergePolicy(NoMergePolicy.INSTANCE);
RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory, iwc);
List<Document> documents = new ArrayList<>();
// 1 segment with, 1 root document, with 3 nested sub docs
Document document = new Document();
document.add(
new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
document.add(
new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE));
documents.add(document);
document = new Document();
document.add(
new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
document.add(
new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE));
documents.add(document);
document = new Document();
document.add(
new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
document.add(
new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE));
documents.add(document);
document = new Document();
document.add(new Field(UidFieldMapper.NAME, "type#1", UidFieldMapper.Defaults.FIELD_TYPE));
document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE));
documents.add(document);
indexWriter.addDocuments(documents);
indexWriter.commit();
documents.clear();
// 1 segment with:
// 1 document, with 1 nested subdoc
document = new Document();
document.add(
new Field(UidFieldMapper.NAME, "type#2", UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
document.add(
new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE));
documents.add(document);
document = new Document();
document.add(new Field(UidFieldMapper.NAME, "type#2", UidFieldMapper.Defaults.FIELD_TYPE));
document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE));
documents.add(document);
indexWriter.addDocuments(documents);
documents.clear();
// and 1 document, with 1 nested subdoc
document = new Document();
document.add(
new Field(UidFieldMapper.NAME, "type#3", UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
document.add(
new Field(TypeFieldMapper.NAME, "__nested_field", TypeFieldMapper.Defaults.FIELD_TYPE));
documents.add(document);
document = new Document();
document.add(new Field(UidFieldMapper.NAME, "type#3", UidFieldMapper.Defaults.FIELD_TYPE));
document.add(new Field(TypeFieldMapper.NAME, "test", TypeFieldMapper.Defaults.FIELD_TYPE));
documents.add(document);
indexWriter.addDocuments(documents);
indexWriter.commit();
indexWriter.close();
IndexService indexService = createIndex("test");
DirectoryReader directoryReader = DirectoryReader.open(directory);
directoryReader =
ElasticsearchDirectoryReader.wrap(directoryReader, new ShardId(indexService.index(), 0));
IndexSearcher searcher = new IndexSearcher(directoryReader);
indexService
.mapperService()
.merge(
"test",
new CompressedXContent(
PutMappingRequest.buildFromSimplifiedDef("test", "nested_field", "type=nested")
.string()),
MapperService.MergeReason.MAPPING_UPDATE,
false);
SearchContext searchContext = createSearchContext(indexService);
AggregationContext context = new AggregationContext(searchContext);
AggregatorFactories.Builder builder = AggregatorFactories.builder();
NestedAggregatorBuilder factory = new NestedAggregatorBuilder("test", "nested_field");
builder.addAggregator(factory);
AggregatorFactories factories = builder.build(context, null);
searchContext.aggregations(new SearchContextAggregations(factories));
Aggregator[] aggs = factories.createTopLevelAggregators();
BucketCollector collector = BucketCollector.wrap(Arrays.asList(aggs));
collector.preCollection();
// A regular search always exclude nested docs, so we use NonNestedDocsFilter.INSTANCE here
// (otherwise MatchAllDocsQuery would be sufficient)
// We exclude root doc with uid type#2, this will trigger the bug if we don't reset the root doc
// when we process a new segment, because
// root doc type#3 and root doc type#1 have the same segment docid
BooleanQuery.Builder bq = new BooleanQuery.Builder();
bq.add(Queries.newNonNestedFilter(), Occur.MUST);
bq.add(new TermQuery(new Term(UidFieldMapper.NAME, "type#2")), Occur.MUST_NOT);
searcher.search(new ConstantScoreQuery(bq.build()), collector);
collector.postCollection();
Nested nested = (Nested) aggs[0].buildAggregation(0);
// The bug manifests if 6 docs are returned, because currentRootDoc isn't reset the previous
// child docs from the first segment are emitted as hits.
assertThat(nested.getDocCount(), equalTo(4L));
directoryReader.close();
directory.close();
}
public void toXContent(XContentBuilder builder, Params params, ToXContent custom)
throws IOException {
builder.startObject(simpleName());
if (nested.isNested()) {
builder.field("type", NESTED_CONTENT_TYPE);
if (nested.isIncludeInParent()) {
builder.field("include_in_parent", true);
}
if (nested.isIncludeInRoot()) {
builder.field("include_in_root", true);
}
} else if (mappers.isEmpty()
&& custom
== null) { // only write the object content type if there are no properties, otherwise,
// it is automatically detected
builder.field("type", CONTENT_TYPE);
}
if (dynamic != null) {
builder.field("dynamic", dynamic.name().toLowerCase(Locale.ROOT));
}
if (enabled != Defaults.ENABLED) {
builder.field("enabled", enabled);
}
if (pathType != Defaults.PATH_TYPE) {
builder.field("path", pathType.name().toLowerCase(Locale.ROOT));
}
if (includeInAll != null) {
builder.field("include_in_all", includeInAll);
}
if (custom != null) {
custom.toXContent(builder, params);
}
doXContent(builder, params);
// sort the mappers so we get consistent serialization format
Mapper[] sortedMappers = Iterables.toArray(mappers.values(), Mapper.class);
Arrays.sort(
sortedMappers,
new Comparator<Mapper>() {
@Override
public int compare(Mapper o1, Mapper o2) {
return o1.name().compareTo(o2.name());
}
});
int count = 0;
for (Mapper mapper : sortedMappers) {
if (!(mapper instanceof MetadataFieldMapper)) {
if (count++ == 0) {
builder.startObject("properties");
}
mapper.toXContent(builder, params);
}
}
if (count > 0) {
builder.endObject();
}
builder.endObject();
}
public void toXContent(
XContentBuilder builder, Params params, ToXContent custom, Mapper... additionalMappers)
throws IOException {
builder.startObject(name);
if (nested.isNested()) {
builder.field("type", NESTED_CONTENT_TYPE);
if (nested.isIncludeInParent()) {
builder.field("include_in_parent", true);
}
if (nested.isIncludeInRoot()) {
builder.field("include_in_root", true);
}
} else if (mappers
.isEmpty()) { // only write the object content type if there are no properties, otherwise,
// it is automatically detected
builder.field("type", CONTENT_TYPE);
}
// grr, ugly! on root, dynamic defaults to TRUE, on children, it defaults to null to
// inherit the root behavior
if (this instanceof RootObjectMapper) {
if (dynamic != Dynamic.TRUE) {
builder.field("dynamic", dynamic.name().toLowerCase());
}
} else {
if (dynamic != Defaults.DYNAMIC) {
builder.field("dynamic", dynamic.name().toLowerCase());
}
}
if (enabled != Defaults.ENABLED) {
builder.field("enabled", enabled);
}
if (pathType != Defaults.PATH_TYPE) {
builder.field("path", pathType.name().toLowerCase());
}
if (includeInAll != null) {
builder.field("include_in_all", includeInAll);
}
if (custom != null) {
custom.toXContent(builder, params);
}
doXContent(builder, params);
// sort the mappers so we get consistent serialization format
TreeMap<String, Mapper> sortedMappers = new TreeMap<String, Mapper>(mappers);
// check internal mappers first (this is only relevant for root object)
for (Mapper mapper : sortedMappers.values()) {
if (mapper instanceof InternalMapper) {
mapper.toXContent(builder, params);
}
}
if (additionalMappers != null && additionalMappers.length > 0) {
TreeMap<String, Mapper> additionalSortedMappers = new TreeMap<String, Mapper>();
for (Mapper mapper : additionalMappers) {
additionalSortedMappers.put(mapper.name(), mapper);
}
for (Mapper mapper : additionalSortedMappers.values()) {
mapper.toXContent(builder, params);
}
}
if (!mappers.isEmpty()) {
builder.startObject("properties");
for (Mapper mapper : sortedMappers.values()) {
if (!(mapper instanceof InternalMapper)) {
mapper.toXContent(builder, params);
}
}
builder.endObject();
}
builder.endObject();
}
public void parse(ParseContext context) throws IOException {
if (!enabled) {
context.parser().skipChildren();
return;
}
XContentParser parser = context.parser();
String currentFieldName = parser.currentName();
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.VALUE_NULL) {
// the object is null ("obj1" : null), simply bail
return;
}
if (token.isValue() && !allowValue()) {
// if we are parsing an object but it is just a value, its only allowed on root level parsers
// with there
// is a field name with the same name as the type
throw new MapperParsingException(
"object mapping for [" + name + "] tried to parse as object, but found a concrete value");
}
Document restoreDoc = null;
if (nested.isNested()) {
Document nestedDoc = new Document();
// pre add the uid field if possible (id was already provided)
IndexableField uidField = context.doc().getField(UidFieldMapper.NAME);
if (uidField != null) {
// we don't need to add it as a full uid field in nested docs, since we don't need
// versioning
// we also rely on this for UidField#loadVersion
// this is a deeply nested field
if (uidField.stringValue() != null) {
nestedDoc.add(
new Field(
UidFieldMapper.NAME,
uidField.stringValue(),
UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
} else {
nestedDoc.add(
new Field(
UidFieldMapper.NAME,
((UidField) uidField).uid(),
UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
}
}
// the type of the nested doc starts with __, so we can identify that its a nested one in
// filters
// note, we don't prefix it with the type of the doc since it allows us to execute a nested
// query
// across types (for example, with similar nested objects)
nestedDoc.add(
new Field(
TypeFieldMapper.NAME, nestedTypePathAsString, TypeFieldMapper.Defaults.FIELD_TYPE));
restoreDoc = context.switchDoc(nestedDoc);
context.addDoc(nestedDoc);
}
ContentPath.Type origPathType = context.path().pathType();
context.path().pathType(pathType);
// if we are at the end of the previous object, advance
if (token == XContentParser.Token.END_OBJECT) {
token = parser.nextToken();
}
if (token == XContentParser.Token.START_OBJECT) {
// if we are just starting an OBJECT, advance, this is the object we are parsing, we need the
// name first
token = parser.nextToken();
}
while (token != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.START_OBJECT) {
serializeObject(context, currentFieldName);
} else if (token == XContentParser.Token.START_ARRAY) {
serializeArray(context, currentFieldName);
} else if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_NULL) {
serializeNullValue(context, currentFieldName);
} else if (token == null) {
throw new MapperParsingException(
"object mapping for ["
+ name
+ "] tried to parse as object, but got EOF, has a concrete value been provided to it?");
} else if (token.isValue()) {
serializeValue(context, currentFieldName, token);
}
token = parser.nextToken();
}
// restore the enable path flag
context.path().pathType(origPathType);
if (nested.isNested()) {
Document nestedDoc = context.switchDoc(restoreDoc);
if (nested.isIncludeInParent()) {
for (IndexableField field : nestedDoc.getFields()) {
if (field.name().equals(UidFieldMapper.NAME)
|| field.name().equals(TypeFieldMapper.NAME)) {
continue;
} else {
context.doc().add(field);
}
}
}
if (nested.isIncludeInRoot()) {
// don't add it twice, if its included in parent, and we are handling the master doc...
if (!(nested.isIncludeInParent() && context.doc() == context.rootDoc())) {
for (IndexableField field : nestedDoc.getFields()) {
if (field.name().equals(UidFieldMapper.NAME)
|| field.name().equals(TypeFieldMapper.NAME)) {
continue;
} else {
context.rootDoc().add(field);
}
}
}
}
}
}
@Override
public Nested redact(Nested value) {
Builder builder = value.newBuilder();
builder.clearUnknownFields();
return builder.build();
}
@Override
public void encode(ProtoWriter writer, Nested value) throws IOException {
if (value.value != null) FooBarBazEnum.ADAPTER.encodeWithTag(writer, 1, value.value);
writer.writeBytes(value.unknownFields());
}
@Override
public int encodedSize(Nested value) {
return (value.value != null ? FooBarBazEnum.ADAPTER.encodedSizeWithTag(1, value.value) : 0)
+ value.unknownFields().size();
}
public NestedDemo() {
Nested nested = new Nested();
System.out.println("Outside of Nested; nested.n = " + nested.n);
nested.f();
}
public void toXContent(
XContentBuilder builder, Params params, ToXContent custom, Mapper... additionalMappers)
throws IOException {
builder.startObject(name);
if (nested.isNested()) {
builder.field("type", NESTED_CONTENT_TYPE);
if (nested.isIncludeInParent()) {
builder.field("include_in_parent", true);
}
if (nested.isIncludeInRoot()) {
builder.field("include_in_root", true);
}
} else if (mappers
.isEmpty()) { // only write the object content type if there are no properties, otherwise,
// it is automatically detected
builder.field("type", CONTENT_TYPE);
}
if (dynamic != null) {
builder.field("dynamic", dynamic.name().toLowerCase(Locale.ROOT));
}
if (enabled != Defaults.ENABLED) {
builder.field("enabled", enabled);
}
if (pathType != Defaults.PATH_TYPE) {
builder.field("path", pathType.name().toLowerCase(Locale.ROOT));
}
if (includeInAll != null) {
builder.field("include_in_all", includeInAll);
}
if (custom != null) {
custom.toXContent(builder, params);
}
doXContent(builder, params);
// sort the mappers so we get consistent serialization format
Mapper[] sortedMappers = Iterables.toArray(mappers.values(), Mapper.class);
Arrays.sort(
sortedMappers,
new Comparator<Mapper>() {
@Override
public int compare(Mapper o1, Mapper o2) {
return o1.name().compareTo(o2.name());
}
});
// check internal mappers first (this is only relevant for root object)
for (Mapper mapper : sortedMappers) {
if (mapper instanceof InternalMapper) {
mapper.toXContent(builder, params);
}
}
if (additionalMappers != null && additionalMappers.length > 0) {
TreeMap<String, Mapper> additionalSortedMappers = new TreeMap<>();
for (Mapper mapper : additionalMappers) {
additionalSortedMappers.put(mapper.name(), mapper);
}
for (Mapper mapper : additionalSortedMappers.values()) {
mapper.toXContent(builder, params);
}
}
if (!mappers.isEmpty()) {
builder.startObject("properties");
for (Mapper mapper : sortedMappers) {
if (!(mapper instanceof InternalMapper)) {
mapper.toXContent(builder, params);
}
}
builder.endObject();
}
builder.endObject();
}