@Override
public Query termsQuery(List values, @Nullable QueryShardContext context) {
if (context == null) {
return super.termsQuery(values, context);
}
List<String> types = new ArrayList<>(context.getMapperService().types().size());
for (DocumentMapper documentMapper : context.getMapperService().docMappers(false)) {
if (!documentMapper.parentFieldMapper().active()) {
types.add(documentMapper.type());
}
}
List<BytesRef> bValues = new ArrayList<>(values.size());
for (Object value : values) {
BytesRef bValue = BytesRefs.toBytesRef(value);
if (Uid.hasDelimiter(bValue)) {
bValues.add(bValue);
} else {
// we use all non child types, cause we don't know if its exact or not...
for (String type : types) {
bValues.add(Uid.createUidAsBytes(type, bValue));
}
}
}
return new TermsQuery(name(), bValues);
}
@Override
public GetResult get(Get get, Function<String, Searcher> searcherFactory) throws EngineException {
try (ReleasableLock lock = readLock.acquire()) {
ensureOpen();
if (get.realtime()) {
VersionValue versionValue = versionMap.getUnderLock(get.uid().bytes());
if (versionValue != null) {
if (versionValue.delete()) {
return GetResult.NOT_EXISTS;
}
if (get.versionType().isVersionConflictForReads(versionValue.version(), get.version())) {
Uid uid = Uid.createUid(get.uid().text());
throw new VersionConflictEngineException(
shardId,
uid.type(),
uid.id(),
get.versionType().explainConflictForReads(versionValue.version(), get.version()));
}
Translog.Operation op = translog.read(versionValue.translogLocation());
if (op != null) {
return new GetResult(true, versionValue.version(), op.getSource());
}
}
}
// no version, get the version from the index, we know that we refresh on flush
return getFromSearcher(get, searcherFactory);
}
}
@Override
public Filter regexpFilter(
Object value, int flags, int maxDeterminizedStates, @Nullable QueryParseContext context) {
if (fieldType.indexOptions() != IndexOptions.NONE || context == null) {
return super.regexpFilter(value, flags, maxDeterminizedStates, context);
}
Collection<String> queryTypes = context.queryTypes();
if (queryTypes.size() == 1) {
return new RegexpFilter(
new Term(
UidFieldMapper.NAME,
Uid.createUidAsBytes(
Iterables.getFirst(queryTypes, null), BytesRefs.toBytesRef(value))),
flags,
maxDeterminizedStates);
}
XBooleanFilter filter = new XBooleanFilter();
for (String queryType : queryTypes) {
filter.add(
new RegexpFilter(
new Term(
UidFieldMapper.NAME,
Uid.createUidAsBytes(queryType, BytesRefs.toBytesRef(value))),
flags,
maxDeterminizedStates),
BooleanClause.Occur.SHOULD);
}
return filter;
}
@Override
public Query prefixQuery(
Object value,
@Nullable MultiTermQuery.RewriteMethod method,
@Nullable QueryParseContext context) {
if (fieldType.indexOptions() != IndexOptions.NONE || context == null) {
return super.prefixQuery(value, method, context);
}
Collection<String> queryTypes = context.queryTypes();
if (queryTypes.size() == 1) {
PrefixQuery prefixQuery =
new PrefixQuery(
new Term(
UidFieldMapper.NAME,
Uid.createUidAsBytes(
Iterables.getFirst(queryTypes, null), BytesRefs.toBytesRef(value))));
if (method != null) {
prefixQuery.setRewriteMethod(method);
}
return prefixQuery;
}
BooleanQuery query = new BooleanQuery();
for (String queryType : queryTypes) {
PrefixQuery prefixQuery =
new PrefixQuery(
new Term(
UidFieldMapper.NAME,
Uid.createUidAsBytes(queryType, BytesRefs.toBytesRef(value))));
if (method != null) {
prefixQuery.setRewriteMethod(method);
}
query.add(prefixQuery, BooleanClause.Occur.SHOULD);
}
return query;
}
@Override
protected Field parseCreateField(ParseContext context) throws IOException {
if (context.parser().currentName() != null
&& context.parser().currentName().equals(Defaults.NAME)) {
// we are in the parsing of _parent phase
String parentId = context.parser().text();
context.sourceToParse().parent(parentId);
return new Field(
names.indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType);
}
// otherwise, we are running it post processing of the xcontent
String parsedParentId = context.doc().get(Defaults.NAME);
if (context.sourceToParse().parent() != null) {
String parentId = context.sourceToParse().parent();
if (parsedParentId == null) {
if (parentId == null) {
throw new MapperParsingException(
"No parent id provided, not within the document, and not externally");
}
// we did not add it in the parsing phase, add it now
return new Field(
names.indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType);
} else if (parentId != null
&& !parsedParentId.equals(Uid.createUid(context.stringBuilder(), type, parentId))) {
throw new MapperParsingException(
"Parent id mismatch, document value is ["
+ Uid.createUid(parsedParentId).id()
+ "], while external value is ["
+ parentId
+ "]");
}
}
// we have parent mapping, yet no value was set, ignore it...
return null;
}
@Test
public void testSimple() throws Exception {
Directory directory = newDirectory();
RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
for (int parent = 1; parent <= 5; parent++) {
Document document = new Document();
document.add(
new StringField(
UidFieldMapper.NAME,
Uid.createUid("parent", Integer.toString(parent)),
Field.Store.NO));
document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO));
indexWriter.addDocument(document);
for (int child = 1; child <= 3; child++) {
document = new Document();
document.add(
new StringField(
UidFieldMapper.NAME,
Uid.createUid("child", Integer.toString(parent * 3 + child)),
Field.Store.NO));
document.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO));
document.add(
new StringField(
ParentFieldMapper.NAME,
Uid.createUid("parent", Integer.toString(parent)),
Field.Store.NO));
document.add(new StringField("field1", "value" + child, Field.Store.NO));
indexWriter.addDocument(document);
}
}
IndexReader indexReader = DirectoryReader.open(indexWriter.w, false);
IndexSearcher searcher = new IndexSearcher(indexReader);
TermQuery childQuery = new TermQuery(new Term("field1", "value" + (1 + random().nextInt(3))));
TermFilter parentFilter = new TermFilter(new Term(TypeFieldMapper.NAME, "parent"));
int shortCircuitParentDocSet = random().nextInt(5);
ChildrenConstantScoreQuery query =
new ChildrenConstantScoreQuery(
childQuery, "parent", "child", parentFilter, shortCircuitParentDocSet, null);
BitSetCollector collector = new BitSetCollector(indexReader.maxDoc());
searcher.search(query, collector);
FixedBitSet actualResult = collector.getResult();
assertThat(actualResult.cardinality(), equalTo(5));
indexWriter.close();
indexReader.close();
directory.close();
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
boolean parent = context.docMapper().isParent(context.type());
if (parent) {
addJoinFieldIfNeeded(fields, parentJoinFieldType, context.id());
}
if (!active()) {
return;
}
if (context.parser().currentName() != null
&& context.parser().currentName().equals(Defaults.NAME)) {
// we are in the parsing of _parent phase
String parentId = context.parser().text();
context.sourceToParse().parent(parentId);
fields.add(
new Field(
fieldType().name(),
Uid.createUid(context.stringBuilder(), parentType, parentId),
fieldType()));
addJoinFieldIfNeeded(fields, childJoinFieldType, parentId);
} else {
// otherwise, we are running it post processing of the xcontent
String parsedParentId = context.doc().get(Defaults.NAME);
if (context.sourceToParse().parent() != null) {
String parentId = context.sourceToParse().parent();
if (parsedParentId == null) {
if (parentId == null) {
throw new MapperParsingException(
"No parent id provided, not within the document, and not externally");
}
// we did not add it in the parsing phase, add it now
fields.add(
new Field(
fieldType().name(),
Uid.createUid(context.stringBuilder(), parentType, parentId),
fieldType()));
addJoinFieldIfNeeded(fields, childJoinFieldType, parentId);
} else if (parentId != null
&& !parsedParentId.equals(
Uid.createUid(context.stringBuilder(), parentType, parentId))) {
throw new MapperParsingException(
"Parent id mismatch, document value is ["
+ Uid.createUid(parsedParentId).id()
+ "], while external value is ["
+ parentId
+ "]");
}
}
}
// we have parent mapping, yet no value was set, ignore it...
}
@Override
public void collect(int doc) {
try {
FieldsVisitor fieldsVisitor = new FieldsVisitor(false);
context.reader().document(doc, fieldsVisitor);
Uid uid = fieldsVisitor.uid();
final long version =
Versions.loadVersion(context.reader(), new Term(UidFieldMapper.NAME, uid.toBytesRef()));
docsToPurge.add(new DocToPurge(uid.type(), uid.id(), version, fieldsVisitor.routing()));
} catch (Exception e) {
logger.trace("failed to collect doc", e);
}
}
@Override
public BytesRef indexedValueForSearch(Object value) {
if (value instanceof BytesRef) {
BytesRef bytesRef = (BytesRef) value;
if (Uid.hasDelimiter(bytesRef)) {
return bytesRef;
}
return Uid.createUidAsBytes(typeAsBytes, bytesRef);
}
String sValue = value.toString();
if (sValue.indexOf(Uid.DELIMITER) == -1) {
return Uid.createUidAsBytes(type, sValue);
}
return super.indexedValueForSearch(value);
}
@Nullable
private Function rewriteAndValidateFields(Function function, Context context) {
if (function.arguments().size() == 2) {
Symbol left = function.arguments().get(0);
Symbol right = function.arguments().get(1);
if (left.symbolType() == SymbolType.REFERENCE && right.symbolType().isValueSymbol()) {
Reference ref = (Reference) left;
if (ref.info().ident().columnIdent().equals(DocSysColumns.ID)) {
function.setArgument(
0,
new Reference(DocSysColumns.forTable(ref.ident().tableIdent(), DocSysColumns.UID)));
function.setArgument(
1,
Literal.newLiteral(
Uid.createUid(
Constants.DEFAULT_MAPPING_TYPE, ValueSymbolVisitor.STRING.process(right))));
} else {
String unsupportedMessage =
context.unsupportedMessage(ref.info().ident().columnIdent().name());
if (unsupportedMessage != null) {
throw new UnsupportedFeatureException(unsupportedMessage);
}
}
}
}
return function;
}
@Override
public Filter termsFilter(List values, @Nullable QueryParseContext context) {
if (fieldType.indexOptions() != IndexOptions.NONE || context == null) {
return super.termsFilter(values, context);
}
return new TermsFilter(UidFieldMapper.NAME, Uid.createTypeUids(context.queryTypes(), values));
}
@Override
public Uid value(Object value) {
if (value == null) {
return null;
}
return Uid.createUid(value.toString());
}
@Override
public Query termQuery(Object value, @Nullable QueryParseContext context) {
if (indexOptions() != IndexOptions.NONE || context == null) {
return super.termQuery(value, context);
}
final BytesRef[] uids = Uid.createUidsForTypesAndId(context.queryTypes(), value);
return new TermsQuery(UidFieldMapper.NAME, uids);
}
@Override
public Query termsQuery(List values, @Nullable QueryParseContext context) {
if (indexOptions() != IndexOptions.NONE || context == null) {
return super.termsQuery(values, context);
}
return new TermsQuery(
UidFieldMapper.NAME, Uid.createUidsForTypesAndIds(context.queryTypes(), values));
}
@Override
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
uid = Uid.createUid(value);
} else {
addValue(fieldInfo.name, value);
}
}
@Override
public Filter termFilter(Object value, @Nullable QueryParseContext context) {
if (!fieldType.indexed()) {
return new PrefixFilter(
new Term(UidFieldMapper.NAME, Uid.typePrefixAsBytes(BytesRefs.toBytesRef(value))));
}
return new TermFilter(names().createIndexNameTerm(BytesRefs.toBytesRef(value)));
}
protected ExplainResponse shardOperation(ExplainRequest request, int shardId)
throws ElasticSearchException {
IndexService indexService = indicesService.indexService(request.index());
IndexShard indexShard = indexService.shardSafe(shardId);
Term uidTerm =
new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(request.getType(), request.getId()));
Engine.GetResult result = indexShard.get(new Engine.Get(false, uidTerm));
if (!result.exists()) {
return new ExplainResponse(false);
}
SearchContext context =
new SearchContext(
0,
new ShardSearchRequest()
.types(new String[] {request.getType()})
.filteringAliases(request.getFilteringAlias()),
null,
result.searcher(),
indexService,
indexShard,
scriptService);
SearchContext.setCurrent(context);
try {
context.parsedQuery(parseQuery(request, indexService));
context.preProcess();
int topLevelDocId = result.docIdAndVersion().docId + result.docIdAndVersion().reader.docBase;
Explanation explanation;
if (context.rescore() != null) {
RescoreSearchContext ctx = context.rescore();
Rescorer rescorer = ctx.rescorer();
explanation = rescorer.explain(topLevelDocId, context, ctx);
} else {
explanation = context.searcher().explain(context.query(), topLevelDocId);
}
if (request.getFields() != null) {
if (request.getFields().length == 1 && "_source".equals(request.getFields()[0])) {
request.setFields(null); // Load the _source field
}
// Advantage is that we're not opening a second searcher to retrieve the _source. Also
// because we are working in the same searcher in engineGetResult we can be sure that a
// doc isn't deleted between the initial get and this call.
GetResult getResult =
indexShard
.getService()
.get(result, request.getId(), request.getType(), request.getFields());
return new ExplainResponse(true, explanation, getResult);
} else {
return new ExplainResponse(true, explanation);
}
} catch (IOException e) {
throw new ElasticSearchException("Could not explain", e);
} finally {
context.release();
SearchContext.removeCurrent();
}
}
@Override
public void stringField(FieldInfo fieldInfo, byte[] bytes) throws IOException {
final String value = new String(bytes, StandardCharsets.UTF_8);
if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
uid = Uid.createUid(value);
} else {
addValue(fieldInfo.name, value);
}
}
@Override
protected Field parseCreateField(ParseContext context) throws IOException {
// so, caching uid stream and field is fine
// since we don't do any mapping parsing without immediate indexing
// and, when percolating, we don't index the uid
UidField field = fieldCache.get();
field.setUid(Uid.createUid(context.stringBuilder(), context.type(), context.id()));
context.uid(field);
return field; // version get updated by the engine
}
@Override
public TopDocs topDocs(SearchContext context, FetchSubPhase.HitContext hitContext)
throws IOException {
final String field;
final String term;
if (isParentHit(hitContext.hit())) {
field = ParentFieldMapper.NAME;
term = Uid.createUid(hitContext.hit().type(), hitContext.hit().id());
} else if (isChildHit(hitContext.hit())) {
field = UidFieldMapper.NAME;
SearchHitField parentField = hitContext.hit().field(ParentFieldMapper.NAME);
if (parentField != null) {
term = parentField.getValue();
} else {
SingleFieldsVisitor fieldsVisitor = new SingleFieldsVisitor(ParentFieldMapper.NAME);
hitContext.reader().document(hitContext.docId(), fieldsVisitor);
if (fieldsVisitor.fields().isEmpty()) {
return Lucene.EMPTY_TOP_DOCS;
}
term = (String) fieldsVisitor.fields().get(ParentFieldMapper.NAME).get(0);
}
} else {
return Lucene.EMPTY_TOP_DOCS;
}
// Only include docs that have the current hit as parent
Filter filter = new TermFilter(new Term(field, term));
// Only include docs that have this inner hits type
Filter typeFilter = documentMapper.typeFilter();
if (size() == 0) {
TotalHitCountCollector collector = new TotalHitCountCollector();
context
.searcher()
.search(
new XFilteredQuery(query, new AndFilter(Arrays.asList(filter, typeFilter))),
collector);
return new TopDocs(collector.getTotalHits(), Lucene.EMPTY_SCORE_DOCS, 0);
} else {
int topN = from() + size();
TopDocsCollector topDocsCollector;
if (sort() != null) {
topDocsCollector =
TopFieldCollector.create(sort(), topN, true, trackScores(), trackScores(), false);
} else {
topDocsCollector = TopScoreDocCollector.create(topN, false);
}
context
.searcher()
.search(
new XFilteredQuery(query, new AndFilter(Arrays.asList(filter, typeFilter))),
topDocsCollector);
return topDocsCollector.topDocs(from(), size());
}
}
@Override
public Query regexpQuery(
String value,
int flags,
int maxDeterminizedStates,
@Nullable MultiTermQuery.RewriteMethod method,
@Nullable QueryParseContext context) {
if (indexOptions() != IndexOptions.NONE || context == null) {
return super.regexpQuery(value, flags, maxDeterminizedStates, method, context);
}
Collection<String> queryTypes = context.queryTypes();
if (queryTypes.size() == 1) {
RegexpQuery regexpQuery =
new RegexpQuery(
new Term(
UidFieldMapper.NAME,
Uid.createUidAsBytes(
Iterables.getFirst(queryTypes, null), BytesRefs.toBytesRef(value))),
flags,
maxDeterminizedStates);
if (method != null) {
regexpQuery.setRewriteMethod(method);
}
return regexpQuery;
}
BooleanQuery.Builder query = new BooleanQuery.Builder();
for (String queryType : queryTypes) {
RegexpQuery regexpQuery =
new RegexpQuery(
new Term(
UidFieldMapper.NAME,
Uid.createUidAsBytes(queryType, BytesRefs.toBytesRef(value))),
flags,
maxDeterminizedStates);
if (method != null) {
regexpQuery.setRewriteMethod(method);
}
query.add(regexpQuery, BooleanClause.Occur.SHOULD);
}
return query.build();
}
@Override
public void collect(int doc) throws IOException {
// the _source is the query
Document document = reader.document(doc, new UidAndSourceFieldSelector());
String id = Uid.createUid(document.get(UidFieldMapper.NAME)).id();
byte[] source = document.getBinaryValue(SourceFieldMapper.NAME);
try {
queries.put(id, percolator.parseQuery(id, source, 0, source.length));
} catch (Exception e) {
logger.warn("failed to add query [{}]", e, id);
}
}
@Override
protected Fieldable parseCreateField(ParseContext context) throws IOException {
if (context.id() == null) {
throw new MapperParsingException("No id found while parsing the content source");
}
context.uid(Uid.createUid(context.stringBuilder(), context.type(), context.id()));
// so, caching uid stream and field is fine
// since we don't do any mapping parsing without immediate indexing
// and, when percolating, we don't index the uid
UidField field = fieldCache.get();
field.setUid(context.uid());
return field; // version get updated by the engine
}
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
Field uid =
new Field(
NAME,
Uid.createUid(context.stringBuilder(), context.type(), context.id()),
Defaults.FIELD_TYPE);
context.uid(uid);
fields.add(uid);
if (fieldType().hasDocValues()) {
fields.add(new BinaryDocValuesField(NAME, new BytesRef(uid.stringValue())));
}
}
@Override
public Filter termFilter(Object value, @Nullable QueryParseContext context) {
if (context == null) {
return super.termFilter(value, context);
}
BytesRef bValue = BytesRefs.toBytesRef(value);
// we use all types, cause we don't know if its exact or not...
BytesRef[] typesValues = new BytesRef[context.mapperService().types().size()];
int i = 0;
for (String type : context.mapperService().types()) {
typesValues[i++] = Uid.createUidAsBytes(type, bValue);
}
return new TermsFilter(names.indexName(), typesValues);
}
@Override
public Filter termsFilter(List values, @Nullable QueryParseContext context) {
if (context == null) {
return super.termsFilter(values, context);
}
List<BytesRef> bValues = new ArrayList<BytesRef>(values.size());
for (Object value : values) {
BytesRef bValue = BytesRefs.toBytesRef(value);
// we use all types, cause we don't know if its exact or not...
for (String type : context.mapperService().types()) {
bValues.add(Uid.createUidAsBytes(type, bValue));
}
}
return new TermsFilter(names.indexName(), bValues);
}
public void testSimpleParserNoTypeNoId() throws Exception {
String mapping =
copyToStringFromClasspath("/org/elasticsearch/index/mapper/simple/test-mapping.json");
DocumentMapper docMapper =
createIndex("test").mapperService().documentMapperParser().parse(mapping);
BytesReference json =
new BytesArray(
copyToBytesFromClasspath(
"/org/elasticsearch/index/mapper/simple/test1-notype-noid.json"));
Document doc = docMapper.parse("test", "person", "1", json).rootDoc();
assertThat(
doc.get(docMapper.uidMapper().fieldType().names().indexName()),
equalTo(Uid.createUid("person", "1")));
assertThat(
doc.get(docMapper.mappers().getMapper("name.first").fieldType().names().indexName()),
equalTo("shay"));
}
private Set<Uid> getShardDocUIDs(final IndexShard shard) throws IOException {
shard.refresh("get_uids");
try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
Set<Uid> ids = new HashSet<>();
for (LeafReaderContext leafContext : searcher.reader().leaves()) {
LeafReader reader = leafContext.reader();
Bits liveDocs = reader.getLiveDocs();
for (int i = 0; i < reader.maxDoc(); i++) {
if (liveDocs == null || liveDocs.get(i)) {
Document uuid = reader.document(i, Collections.singleton(UidFieldMapper.NAME));
ids.add(Uid.createUid(uuid.get(UidFieldMapper.NAME)));
}
}
}
return ids;
}
}
@Test
public void testSimpleParserNoTypeNoId() throws Exception {
String mapping =
copyToStringFromClasspath(
"/org/elasticsearch/test/unit/index/mapper/simple/test-mapping.json");
DocumentMapper docMapper = MapperTests.newParser().parse(mapping);
BytesReference json =
new BytesArray(
copyToBytesFromClasspath(
"/org/elasticsearch/test/unit/index/mapper/simple/test1-notype-noid.json"));
Document doc = docMapper.parse("person", "1", json).rootDoc();
assertThat(
doc.get(docMapper.uidMapper().names().indexName()), equalTo(Uid.createUid("person", "1")));
assertThat((double) doc.getBoost(), closeTo(3.7, 0.01));
assertThat(
doc.get(docMapper.mappers().name("first").mapper().names().indexName()), equalTo("shay"));
// System.out.println("Document: " + doc);
// System.out.println("Json: " + docMapper.sourceMapper().value(doc));
}
public void postProcess(MapperService mapperService) {
if (uid != null) {
DocumentMapper documentMapper = mapperService.documentMapper(uid.type());
if (documentMapper != null) {
// we can derive the exact type for the mapping
postProcess(documentMapper);
return;
}
}
// can't derive exact mapping type
for (Map.Entry<String, List<Object>> entry : fields().entrySet()) {
FieldMappers fieldMappers = mapperService.indexName(entry.getKey());
if (fieldMappers == null) {
continue;
}
List<Object> fieldValues = entry.getValue();
for (int i = 0; i < fieldValues.size(); i++) {
fieldValues.set(i, fieldMappers.mapper().valueForSearch(fieldValues.get(i)));
}
}
}