private static String resolveIndexName(String fieldName, QueryShardContext context) { MappedFieldType fieldType = context.fieldMapper(fieldName); if (fieldType != null) { return fieldType.names().indexName(); } return fieldName; }
@Override public IndexFieldData<AtomicNumericFieldData> build( Index index, @IndexSettings Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { return new PackedArrayIndexFieldData( index, indexSettings, fieldType.names(), fieldType.fieldDataType(), cache, numericType, breakerService); }
@Override protected Query doToQuery(QueryShardContext context) throws IOException { String field; MappedFieldType fieldType = context.fieldMapper(fieldName); if (fieldType != null) { field = fieldType.names().indexName(); } else { field = fieldName; } Analyzer analyzerObj; if (analyzer == null) { if (fieldType != null) { analyzerObj = context.getSearchAnalyzer(fieldType); } else { analyzerObj = context.getMapperService().searchAnalyzer(); } } else { analyzerObj = context.getMapperService().analysisService().analyzer(analyzer); if (analyzerObj == null) { throw new QueryShardException(context, "[common] analyzer [" + analyzer + "] not found"); } } Occur highFreqOccur = highFreqOperator.toBooleanClauseOccur(); Occur lowFreqOccur = lowFreqOperator.toBooleanClauseOccur(); ExtendedCommonTermsQuery commonsQuery = new ExtendedCommonTermsQuery( highFreqOccur, lowFreqOccur, cutoffFrequency, disableCoord, fieldType); return parseQueryString( commonsQuery, text, field, analyzerObj, lowFreqMinimumShouldMatch, highFreqMinimumShouldMatch); }
@Override public void execute(SearchContext context) { FieldsVisitor fieldsVisitor; Set<String> fieldNames = null; List<String> extractFieldNames = null; boolean loadAllStored = false; if (!context.hasFieldNames()) { // no fields specified, default to return source if no explicit indication if (!context.hasScriptFields() && !context.hasFetchSourceContext()) { context.fetchSourceContext(new FetchSourceContext(true)); } fieldsVisitor = new FieldsVisitor(context.sourceRequested()); } else if (context.fieldNames().isEmpty()) { fieldsVisitor = new FieldsVisitor(context.sourceRequested()); } else { for (String fieldName : context.fieldNames()) { if (fieldName.equals("*")) { loadAllStored = true; continue; } if (fieldName.equals(SourceFieldMapper.NAME)) { if (context.hasFetchSourceContext()) { context.fetchSourceContext().fetchSource(true); } else { context.fetchSourceContext(new FetchSourceContext(true)); } continue; } MappedFieldType fieldType = context.smartNameFieldType(fieldName); if (fieldType == null) { // Only fail if we know it is a object field, missing paths / fields shouldn't fail. if (context.getObjectMapper(fieldName) != null) { throw new IllegalArgumentException("field [" + fieldName + "] isn't a leaf field"); } } else if (fieldType.stored()) { if (fieldNames == null) { fieldNames = new HashSet<>(); } fieldNames.add(fieldType.names().indexName()); } else { if (extractFieldNames == null) { extractFieldNames = newArrayList(); } extractFieldNames.add(fieldName); } } if (loadAllStored) { fieldsVisitor = new AllFieldsVisitor(); // load everything, including _source } else if (fieldNames != null) { boolean loadSource = extractFieldNames != null || context.sourceRequested(); fieldsVisitor = new CustomFieldsVisitor(fieldNames, loadSource); } else { fieldsVisitor = new FieldsVisitor(extractFieldNames != null || context.sourceRequested()); } } InternalSearchHit[] hits = new InternalSearchHit[context.docIdsToLoadSize()]; FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(); for (int index = 0; index < context.docIdsToLoadSize(); index++) { int docId = context.docIdsToLoad()[context.docIdsToLoadFrom() + index]; int readerIndex = ReaderUtil.subIndex(docId, context.searcher().getIndexReader().leaves()); LeafReaderContext subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex); int subDocId = docId - subReaderContext.docBase; final InternalSearchHit searchHit; try { int rootDocId = findRootDocumentIfNested(context, subReaderContext, subDocId); if (rootDocId != -1) { searchHit = createNestedSearchHit( context, docId, subDocId, rootDocId, extractFieldNames, loadAllStored, fieldNames, subReaderContext); } else { searchHit = createSearchHit( context, fieldsVisitor, docId, subDocId, extractFieldNames, subReaderContext); } } catch (IOException e) { throw ExceptionsHelper.convertToElastic(e); } hits[index] = searchHit; hitContext.reset(searchHit, subReaderContext, subDocId, context.searcher()); for (FetchSubPhase fetchSubPhase : fetchSubPhases) { if (fetchSubPhase.hitExecutionNeeded(context)) { fetchSubPhase.hitExecute(context, hitContext); } } } for (FetchSubPhase fetchSubPhase : fetchSubPhases) { if (fetchSubPhase.hitsExecutionNeeded(context)) { fetchSubPhase.hitsExecute(context, hits); } } context .fetchResult() .hits( new InternalSearchHits( hits, context.queryResult().topDocs().totalHits, context.queryResult().topDocs().getMaxScore())); }
@Override public Query parse(QueryParseContext parseContext) throws IOException, QueryParsingException { XContentParser parser = parseContext.parser(); MoreLikeThisQuery mltQuery = new MoreLikeThisQuery(); mltQuery.setSimilarity(parseContext.searchSimilarity()); Analyzer analyzer = null; List<String> moreLikeFields = null; boolean failOnUnsupportedField = true; String queryName = null; boolean include = false; XContentParser.Token token; String currentFieldName = null; List<String> likeTexts = new ArrayList<>(); MultiTermVectorsRequest likeItems = new MultiTermVectorsRequest(); List<String> unlikeTexts = new ArrayList<>(); MultiTermVectorsRequest unlikeItems = new MultiTermVectorsRequest(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token.isValue()) { if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.LIKE_TEXT)) { likeTexts.add(parser.text()); } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.LIKE)) { parseLikeField(parser, likeTexts, likeItems); } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.UNLIKE)) { parseLikeField(parser, unlikeTexts, unlikeItems); } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.MIN_TERM_FREQ)) { mltQuery.setMinTermFrequency(parser.intValue()); } else if (parseContext .parseFieldMatcher() .match(currentFieldName, Fields.MAX_QUERY_TERMS)) { mltQuery.setMaxQueryTerms(parser.intValue()); } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.MIN_DOC_FREQ)) { mltQuery.setMinDocFreq(parser.intValue()); } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.MAX_DOC_FREQ)) { mltQuery.setMaxDocFreq(parser.intValue()); } else if (parseContext .parseFieldMatcher() .match(currentFieldName, Fields.MIN_WORD_LENGTH)) { mltQuery.setMinWordLen(parser.intValue()); } else if (parseContext .parseFieldMatcher() .match(currentFieldName, Fields.MAX_WORD_LENGTH)) { mltQuery.setMaxWordLen(parser.intValue()); } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.BOOST_TERMS)) { float boostFactor = parser.floatValue(); if (boostFactor != 0) { mltQuery.setBoostTerms(true); mltQuery.setBoostTermsFactor(boostFactor); } } else if (parseContext .parseFieldMatcher() .match(currentFieldName, Fields.MINIMUM_SHOULD_MATCH)) { mltQuery.setMinimumShouldMatch(parser.text()); } else if ("analyzer".equals(currentFieldName)) { analyzer = parseContext.analysisService().analyzer(parser.text()); } else if ("boost".equals(currentFieldName)) { mltQuery.setBoost(parser.floatValue()); } else if (parseContext .parseFieldMatcher() .match(currentFieldName, Fields.FAIL_ON_UNSUPPORTED_FIELD)) { failOnUnsupportedField = parser.booleanValue(); } else if ("_name".equals(currentFieldName)) { queryName = parser.text(); } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.INCLUDE)) { include = parser.booleanValue(); } else { throw new QueryParsingException( parseContext, "[mlt] query does not support [" + currentFieldName + "]"); } } else if (token == XContentParser.Token.START_ARRAY) { if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.STOP_WORDS)) { Set<String> stopWords = Sets.newHashSet(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { stopWords.add(parser.text()); } mltQuery.setStopWords(stopWords); } else if ("fields".equals(currentFieldName)) { moreLikeFields = new LinkedList<>(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { String field = parser.text(); MappedFieldType fieldType = parseContext.fieldMapper(field); moreLikeFields.add(fieldType == null ? field : fieldType.names().indexName()); } } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.DOCUMENT_IDS)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (!token.isValue()) { throw new IllegalArgumentException("ids array element should only contain ids"); } likeItems.add(newTermVectorsRequest().id(parser.text())); } } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.DOCUMENTS)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token != XContentParser.Token.START_OBJECT) { throw new IllegalArgumentException("docs array element should include an object"); } likeItems.add(parseDocument(parser)); } } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.LIKE)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { parseLikeField(parser, likeTexts, likeItems); } } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.UNLIKE)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { parseLikeField(parser, unlikeTexts, unlikeItems); } } else { throw new QueryParsingException( parseContext, "[mlt] query does not support [" + currentFieldName + "]"); } } else if (token == XContentParser.Token.START_OBJECT) { if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.LIKE)) { parseLikeField(parser, likeTexts, likeItems); } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fields.UNLIKE)) { parseLikeField(parser, unlikeTexts, unlikeItems); } else { throw new QueryParsingException( parseContext, "[mlt] query does not support [" + currentFieldName + "]"); } } } if (likeTexts.isEmpty() && likeItems.isEmpty()) { throw new QueryParsingException( parseContext, "more_like_this requires 'like' to be specified"); } if (moreLikeFields != null && moreLikeFields.isEmpty()) { throw new QueryParsingException( parseContext, "more_like_this requires 'fields' to be non-empty"); } // set analyzer if (analyzer == null) { analyzer = parseContext.mapperService().searchAnalyzer(); } mltQuery.setAnalyzer(analyzer); // set like text fields boolean useDefaultField = (moreLikeFields == null); if (useDefaultField) { moreLikeFields = Collections.singletonList(parseContext.defaultField()); } // possibly remove unsupported fields removeUnsupportedFields(moreLikeFields, analyzer, failOnUnsupportedField); if (moreLikeFields.isEmpty()) { return null; } mltQuery.setMoreLikeFields(moreLikeFields.toArray(Strings.EMPTY_ARRAY)); // support for named query if (queryName != null) { parseContext.addNamedQuery(queryName, mltQuery); } // handle like texts if (!likeTexts.isEmpty()) { mltQuery.setLikeText(likeTexts); } if (!unlikeTexts.isEmpty()) { mltQuery.setIgnoreText(unlikeTexts); } // handle items if (!likeItems.isEmpty()) { // set default index, type and fields if not specified MultiTermVectorsRequest items = likeItems; for (TermVectorsRequest item : unlikeItems) { items.add(item); } for (TermVectorsRequest item : items) { if (item.index() == null) { item.index(parseContext.index().name()); } if (item.type() == null) { if (parseContext.queryTypes().size() > 1) { throw new QueryParsingException( parseContext, "ambiguous type for item with id: " + item.id() + " and index: " + item.index()); } else { item.type(parseContext.queryTypes().iterator().next()); } } // default fields if not present but don't override for artificial docs if (item.selectedFields() == null && item.doc() == null) { if (useDefaultField) { item.selectedFields("*"); } else { item.selectedFields(moreLikeFields.toArray(new String[moreLikeFields.size()])); } } } // fetching the items with multi-termvectors API items.copyContextAndHeadersFrom(SearchContext.current()); MultiTermVectorsResponse responses = fetchService.fetchResponse(items); // getting the Fields for liked items mltQuery.setLikeText(MoreLikeThisFetchService.getFields(responses, likeItems)); // getting the Fields for ignored items if (!unlikeItems.isEmpty()) { org.apache.lucene.index.Fields[] ignoreFields = MoreLikeThisFetchService.getFields(responses, unlikeItems); if (ignoreFields.length > 0) { mltQuery.setUnlikeText(ignoreFields); } } BooleanQuery.Builder boolQuery = new BooleanQuery.Builder(); boolQuery.add(mltQuery, BooleanClause.Occur.SHOULD); // exclude the items from the search if (!include) { handleExclude(boolQuery, likeItems); } return boolQuery.build(); } return mltQuery; }
@SuppressWarnings("unchecked") public <IFD extends IndexFieldData<?>> IFD getForField(MappedFieldType fieldType) { final Names fieldNames = fieldType.names(); final FieldDataType type = fieldType.fieldDataType(); if (type == null) { throw new IllegalArgumentException( "found no fielddata type for field [" + fieldNames.fullName() + "]"); } final boolean docValues = fieldType.hasDocValues(); IndexFieldData.Builder builder = null; String format = type.getFormat(indexSettings); if (format != null && FieldDataType.DOC_VALUES_FORMAT_VALUE.equals(format) && !docValues) { logger.warn( "field [" + fieldNames.fullName() + "] has no doc values, will use default field data format"); format = null; } if (format != null) { builder = buildersByTypeAndFormat.get(Tuple.tuple(type.getType(), format)); if (builder == null) { logger.warn( "failed to find format [" + format + "] for field [" + fieldNames.fullName() + "], will use default"); } } if (builder == null && docValues) { builder = docValuesBuildersByType.get(type.getType()); } if (builder == null) { builder = buildersByType.get(type.getType()); } if (builder == null) { throw new IllegalArgumentException( "failed to find field data builder for field " + fieldNames.fullName() + ", and type " + type.getType()); } IndexFieldDataCache cache; synchronized (this) { cache = fieldDataCaches.get(fieldNames.indexName()); if (cache == null) { // we default to node level cache, which in turn defaults to be unbounded // this means changing the node level settings is simple, just set the bounds there String cacheType = type.getSettings() .get("cache", indexSettings.get(FIELDDATA_CACHE_KEY, FIELDDATA_CACHE_VALUE_NODE)); if (FIELDDATA_CACHE_VALUE_NODE.equals(cacheType)) { cache = indicesFieldDataCache.buildIndexFieldDataCache(listener, index, fieldNames, type); } else if ("none".equals(cacheType)) { cache = new IndexFieldDataCache.None(); } else { throw new IllegalArgumentException( "cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]"); } fieldDataCaches.put(fieldNames.indexName(), cache); } } return (IFD) builder.build(index, indexSettings, fieldType, cache, circuitBreakerService, mapperService); }