public QueryFetchSearchResult executeFetchPhase(QuerySearchRequest request) {
final SearchContext context = findContext(request.id());
contextProcessing(context);
try {
final IndexCache indexCache = context.indexShard().indexService().cache();
context
.searcher()
.dfSource(
new CachedDfSource(
context.searcher().getIndexReader(),
request.dfs(),
context.similarityService().similarity(),
indexCache.filter(),
indexCache.filterPolicy()));
} catch (Throwable e) {
freeContext(context.id());
cleanContext(context);
throw new QueryPhaseExecutionException(context, "Failed to set aggregated df", e);
}
try {
ShardSearchStats shardSearchStats = context.indexShard().searchService();
shardSearchStats.onPreQueryPhase(context);
long time = System.nanoTime();
try {
queryPhase.execute(context);
} catch (Throwable e) {
shardSearchStats.onFailedQueryPhase(context);
throw ExceptionsHelper.convertToRuntime(e);
}
long time2 = System.nanoTime();
shardSearchStats.onQueryPhase(context, time2 - time);
shardSearchStats.onPreFetchPhase(context);
try {
shortcutDocIdsToLoad(context);
fetchPhase.execute(context);
if (context.scroll() == null) {
freeContext(request.id());
} else {
contextProcessedSuccessfully(context);
}
} catch (Throwable e) {
shardSearchStats.onFailedFetchPhase(context);
throw ExceptionsHelper.convertToRuntime(e);
}
shardSearchStats.onFetchPhase(context, System.nanoTime() - time2);
return new QueryFetchSearchResult(context.queryResult(), context.fetchResult());
} catch (Throwable e) {
logger.trace("Fetch phase failed", e);
processFailure(context, e);
throw ExceptionsHelper.convertToRuntime(e);
} finally {
cleanContext(context);
}
}
public FetchSearchResult executeFetchPhase(ShardFetchRequest request) {
final SearchContext context = findContext(request.id());
contextProcessing(context);
final ShardSearchStats shardSearchStats = context.indexShard().searchService();
try {
if (request.lastEmittedDoc() != null) {
context.lastEmittedDoc(request.lastEmittedDoc());
}
context.docIdsToLoad(request.docIds(), 0, request.docIdsSize());
shardSearchStats.onPreFetchPhase(context);
long time = System.nanoTime();
fetchPhase.execute(context);
if (context.scroll() == null) {
freeContext(request.id());
} else {
contextProcessedSuccessfully(context);
}
shardSearchStats.onFetchPhase(context, System.nanoTime() - time);
return context.fetchResult();
} catch (Throwable e) {
shardSearchStats.onFailedFetchPhase(context);
logger.trace("Fetch phase failed", e);
processFailure(context, e);
throw ExceptionsHelper.convertToRuntime(e);
} finally {
cleanContext(context);
}
}
public QuerySearchResultProvider executeQueryPhase(ShardSearchRequest request) {
final SearchContext context = createAndPutContext(request);
final ShardSearchStats shardSearchStats = context.indexShard().searchService();
try {
shardSearchStats.onPreQueryPhase(context);
long time = System.nanoTime();
contextProcessing(context);
loadOrExecuteQueryPhase(request, context, queryPhase);
if (context.queryResult().topDocs().scoreDocs.length == 0 && context.scroll() == null) {
freeContext(context.id());
} else {
contextProcessedSuccessfully(context);
}
shardSearchStats.onQueryPhase(context, System.nanoTime() - time);
return context.queryResult();
} catch (Throwable e) {
// execution exception can happen while loading the cache, strip it
if (e instanceof ExecutionException) {
e = e.getCause();
}
shardSearchStats.onFailedQueryPhase(context);
logger.trace("Query phase failed", e);
processFailure(context, e);
throw ExceptionsHelper.convertToRuntime(e);
} finally {
cleanContext(context);
}
}
public ScrollQueryFetchSearchResult executeScan(InternalScrollSearchRequest request) {
final SearchContext context = findContext(request.id());
contextProcessing(context);
try {
processScroll(request, context);
if (context.searchType() == SearchType.QUERY_THEN_FETCH) {
// first scanning, reset the from to 0
context.searchType(SearchType.SCAN);
context.from(0);
}
queryPhase.execute(context);
shortcutDocIdsToLoadForScanning(context);
fetchPhase.execute(context);
if (context.scroll() == null || context.fetchResult().hits().hits().length < context.size()) {
freeContext(request.id());
} else {
contextProcessedSuccessfully(context);
}
return new ScrollQueryFetchSearchResult(
new QueryFetchSearchResult(context.queryResult(), context.fetchResult()),
context.shardTarget());
} catch (Throwable e) {
logger.trace("Scan phase failed", e);
processFailure(context, e);
throw ExceptionsHelper.convertToRuntime(e);
} finally {
cleanContext(context);
}
}
public QuerySearchResult executeScan(ShardSearchRequest request) {
final SearchContext context = createAndPutContext(request);
final int originalSize = context.size();
try {
if (context.aggregations() != null) {
throw new IllegalArgumentException("aggregations are not supported with search_type=scan");
}
if (context.scroll() == null) {
throw new ElasticsearchException("Scroll must be provided when scanning...");
}
assert context.searchType() == SearchType.SCAN;
context.searchType(
SearchType
.QUERY_THEN_FETCH); // move to QUERY_THEN_FETCH, and then, when scrolling, move to
// SCAN
context.size(0); // set size to 0 so that we only count matches
assert context.searchType() == SearchType.QUERY_THEN_FETCH;
contextProcessing(context);
queryPhase.execute(context);
contextProcessedSuccessfully(context);
return context.queryResult();
} catch (Throwable e) {
logger.trace("Scan phase failed", e);
processFailure(context, e);
throw ExceptionsHelper.convertToRuntime(e);
} finally {
context.size(originalSize);
cleanContext(context);
}
}
@Override
public Query visitFunction(Function function, Context context) {
assert function != null;
if (fieldIgnored(function, context)) {
return Queries.newMatchAllQuery();
}
function = rewriteAndValidateFields(function, context);
FunctionToQuery toQuery = functions.get(function.info().ident().name());
if (toQuery == null) {
return genericFunctionQuery(function, context);
}
Query query;
try {
query = toQuery.apply(function, context);
} catch (IOException e) {
throw ExceptionsHelper.convertToRuntime(e);
} catch (UnsupportedOperationException e) {
return genericFunctionQuery(function, context);
}
if (query == null) {
query = queryFromInnerFunction(function, context);
if (query == null) {
return genericFunctionQuery(function, context);
}
}
return query;
}
public ScrollQueryFetchSearchResult executeFetchPhase(InternalScrollSearchRequest request) {
final SearchContext context = findContext(request.id());
contextProcessing(context);
try {
ShardSearchStats shardSearchStats = context.indexShard().searchService();
processScroll(request, context);
shardSearchStats.onPreQueryPhase(context);
long time = System.nanoTime();
try {
queryPhase.execute(context);
} catch (Throwable e) {
shardSearchStats.onFailedQueryPhase(context);
throw ExceptionsHelper.convertToRuntime(e);
}
long time2 = System.nanoTime();
shardSearchStats.onQueryPhase(context, time2 - time);
shardSearchStats.onPreFetchPhase(context);
try {
shortcutDocIdsToLoad(context);
fetchPhase.execute(context);
if (context.scroll() == null) {
freeContext(request.id());
} else {
contextProcessedSuccessfully(context);
}
} catch (Throwable e) {
shardSearchStats.onFailedFetchPhase(context);
throw ExceptionsHelper.convertToRuntime(e);
}
shardSearchStats.onFetchPhase(context, System.nanoTime() - time2);
return new ScrollQueryFetchSearchResult(
new QueryFetchSearchResult(context.queryResult(), context.fetchResult()),
context.shardTarget());
} catch (Throwable e) {
logger.trace("Fetch phase failed", e);
processFailure(context, e);
throw ExceptionsHelper.convertToRuntime(e);
} finally {
cleanContext(context);
}
}
public DfsSearchResult executeDfsPhase(ShardSearchRequest request) {
final SearchContext context = createAndPutContext(request);
try {
contextProcessing(context);
dfsPhase.execute(context);
contextProcessedSuccessfully(context);
return context.dfsResult();
} catch (Throwable e) {
logger.trace("Dfs phase failed", e);
processFailure(context, e);
throw ExceptionsHelper.convertToRuntime(e);
} finally {
cleanContext(context);
}
}
private Query queryFromInnerFunction(Function function, Context context) {
for (Symbol symbol : function.arguments()) {
if (symbol.symbolType() == SymbolType.FUNCTION) {
String functionName = ((Function) symbol).info().ident().name();
InnerFunctionToQuery functionToQuery = innerFunctions.get(functionName);
if (functionToQuery != null) {
try {
Query query = functionToQuery.apply(function, (Function) symbol, context);
if (query != null) {
return query;
}
} catch (IOException e) {
throw ExceptionsHelper.convertToRuntime(e);
}
}
}
}
return null;
}
public QuerySearchResult executeQueryPhase(QuerySearchRequest request) {
final SearchContext context = findContext(request.id());
contextProcessing(context);
try {
final IndexCache indexCache = context.indexShard().indexService().cache();
context
.searcher()
.dfSource(
new CachedDfSource(
context.searcher().getIndexReader(),
request.dfs(),
context.similarityService().similarity(),
indexCache.filter(),
indexCache.filterPolicy()));
} catch (Throwable e) {
processFailure(context, e);
cleanContext(context);
throw new QueryPhaseExecutionException(context, "Failed to set aggregated df", e);
}
ShardSearchStats shardSearchStats = context.indexShard().searchService();
try {
shardSearchStats.onPreQueryPhase(context);
long time = System.nanoTime();
queryPhase.execute(context);
if (context.queryResult().topDocs().scoreDocs.length == 0 && context.scroll() == null) {
// no hits, we can release the context since there will be no fetch phase
freeContext(context.id());
} else {
contextProcessedSuccessfully(context);
}
shardSearchStats.onQueryPhase(context, System.nanoTime() - time);
return context.queryResult();
} catch (Throwable e) {
shardSearchStats.onFailedQueryPhase(context);
logger.trace("Query phase failed", e);
processFailure(context, e);
throw ExceptionsHelper.convertToRuntime(e);
} finally {
cleanContext(context);
}
}
public ScrollQuerySearchResult executeQueryPhase(InternalScrollSearchRequest request) {
final SearchContext context = findContext(request.id());
ShardSearchStats shardSearchStats = context.indexShard().searchService();
try {
shardSearchStats.onPreQueryPhase(context);
long time = System.nanoTime();
contextProcessing(context);
processScroll(request, context);
queryPhase.execute(context);
contextProcessedSuccessfully(context);
shardSearchStats.onQueryPhase(context, System.nanoTime() - time);
return new ScrollQuerySearchResult(context.queryResult(), context.shardTarget());
} catch (Throwable e) {
shardSearchStats.onFailedQueryPhase(context);
logger.trace("Query phase failed", e);
processFailure(context, e);
throw ExceptionsHelper.convertToRuntime(e);
} finally {
cleanContext(context);
}
}
private void waitForClusterState(long clusterStateVersion) {
ClusterStateObserver observer =
new ClusterStateObserver(
clusterService, TimeValue.timeValueMinutes(5), logger, threadPool.getThreadContext());
final ClusterState clusterState = observer.observedState();
if (clusterState.getVersion() >= clusterStateVersion) {
logger.trace(
"node has cluster state with version higher than {} (current: {})",
clusterStateVersion,
clusterState.getVersion());
return;
} else {
logger.trace(
"waiting for cluster state version {} (current: {})",
clusterStateVersion,
clusterState.getVersion());
final PlainActionFuture<Void> future = new PlainActionFuture<>();
observer.waitForNextChange(
new ClusterStateObserver.Listener() {
@Override
public void onNewClusterState(ClusterState state) {
future.onResponse(null);
}
@Override
public void onClusterServiceClose() {
future.onFailure(new NodeClosedException(clusterService.localNode()));
}
@Override
public void onTimeout(TimeValue timeout) {
future.onFailure(
new IllegalStateException(
"cluster state never updated to version " + clusterStateVersion));
}
},
new ClusterStateObserver.ValidationPredicate() {
@Override
protected boolean validate(ClusterState newState) {
return newState.getVersion() >= clusterStateVersion;
}
});
try {
future.get();
logger.trace(
"successfully waited for cluster state with version {} (current: {})",
clusterStateVersion,
observer.observedState().getVersion());
} catch (Exception e) {
logger.debug(
(Supplier<?>)
() ->
new ParameterizedMessage(
"failed waiting for cluster state with version {} (current: {})",
clusterStateVersion,
observer.observedState()),
e);
throw ExceptionsHelper.convertToRuntime(e);
}
}
}
final SearchContext createContext(
ShardSearchRequest request, @Nullable Engine.Searcher searcher) {
IndexService indexService = indicesService.indexServiceSafe(request.index());
IndexShard indexShard = indexService.shardSafe(request.shardId());
SearchShardTarget shardTarget =
new SearchShardTarget(clusterService.localNode().id(), request.index(), request.shardId());
Engine.Searcher engineSearcher =
searcher == null ? indexShard.acquireSearcher("search") : searcher;
SearchContext context =
new DefaultSearchContext(
idGenerator.incrementAndGet(),
request,
shardTarget,
engineSearcher,
indexService,
indexShard,
scriptService,
pageCacheRecycler,
bigArrays,
threadPool.estimatedTimeInMillisCounter());
SearchContext.setCurrent(context);
try {
context.scroll(request.scroll());
parseTemplate(request);
parseSource(context, request.source());
parseSource(context, request.extraSource());
// if the from and size are still not set, default them
if (context.from() == -1) {
context.from(0);
}
if (context.searchType() == SearchType.COUNT) {
// so that the optimizations we apply to size=0 also apply to search_type=COUNT
// and that we close contexts when done with the query phase
context.searchType(SearchType.QUERY_THEN_FETCH);
context.size(0);
} else if (context.size() == -1) {
context.size(10);
}
// pre process
dfsPhase.preProcess(context);
queryPhase.preProcess(context);
fetchPhase.preProcess(context);
// compute the context keep alive
long keepAlive = defaultKeepAlive;
if (request.scroll() != null && request.scroll().keepAlive() != null) {
keepAlive = request.scroll().keepAlive().millis();
}
context.keepAlive(keepAlive);
} catch (Throwable e) {
context.close();
throw ExceptionsHelper.convertToRuntime(e);
}
return context;
}