public ContextIndexSearcher(SearchContext searchContext, Engine.Searcher searcher) {
super(searcher.reader());
in = searcher.searcher();
engineSearcher = searcher;
setSimilarity(searcher.searcher().getSimilarity(true));
setQueryCache(searchContext.indexShard().indexService().cache().query());
setQueryCachingPolicy(searchContext.indexShard().getQueryCachingPolicy());
}
private void loadQueries(IndexShard shard) {
try {
shard.refresh(new Engine.Refresh("percolator_load_queries").force(true));
// Maybe add a mode load? This isn't really a write. We need write b/c state=post_recovery
Engine.Searcher searcher =
shard.acquireSearcher("percolator_load_queries", IndexShard.Mode.WRITE);
try {
Query query =
new XConstantScoreQuery(
indexCache
.filter()
.cache(
new TermFilter(
new Term(TypeFieldMapper.NAME, PercolatorService.TYPE_NAME))));
QueriesLoaderCollector queryCollector =
new QueriesLoaderCollector(
PercolatorQueriesRegistry.this, logger, mapperService, indexFieldDataService);
searcher.searcher().search(query, queryCollector);
Map<HashedBytesRef, Query> queries = queryCollector.queries();
for (Map.Entry<HashedBytesRef, Query> entry : queries.entrySet()) {
Query previousQuery = percolateQueries.put(entry.getKey(), entry.getValue());
shardPercolateService.addedQuery(entry.getKey(), previousQuery, entry.getValue());
}
} finally {
searcher.release();
}
} catch (Exception e) {
throw new PercolatorException(
shardId.index(), "failed to load queries from percolator index", e);
}
}
@Override
public boolean matchesSafely(Engine.Searcher searcher) {
try {
long count = Lucene.count(searcher.searcher(), query);
return count == totalHits;
} catch (IOException e) {
return false;
}
}
public void run() throws Exception {
for (Thread t : searcherThreads) {
t.start();
}
for (Thread t : writerThreads) {
t.start();
}
barrier1.await();
Refresher refresher = new Refresher();
scheduledExecutorService.scheduleWithFixedDelay(
refresher, refreshSchedule.millis(), refreshSchedule.millis(), TimeUnit.MILLISECONDS);
Flusher flusher = new Flusher();
scheduledExecutorService.scheduleWithFixedDelay(
flusher, flushSchedule.millis(), flushSchedule.millis(), TimeUnit.MILLISECONDS);
StopWatch stopWatch = new StopWatch();
stopWatch.start();
barrier2.await();
latch.await();
stopWatch.stop();
System.out.println("Summary");
System.out.println(
" -- Readers ["
+ searcherThreads.length
+ "] with ["
+ searcherIterations
+ "] iterations");
System.out.println(
" -- Writers [" + writerThreads.length + "] with [" + writerIterations + "] iterations");
System.out.println(" -- Took: " + stopWatch.totalTime());
System.out.println(
" -- Refresh [" + refresher.id + "] took: " + refresher.stopWatch.totalTime());
System.out.println(" -- Flush [" + flusher.id + "] took: " + flusher.stopWatch.totalTime());
System.out.println(" -- Store size " + store.estimateSize());
scheduledExecutorService.shutdown();
engine.refresh(new Engine.Refresh(true));
stopWatch = new StopWatch();
stopWatch.start();
Engine.Searcher searcher = engine.searcher();
TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), idGenerator.get() + 1);
stopWatch.stop();
System.out.println(
" -- Indexed ["
+ idGenerator.get()
+ "] docs, found ["
+ topDocs.totalHits
+ "] hits, took "
+ stopWatch.totalTime());
searcher.release();
}
@Override
public void run() {
try {
barrier1.await();
barrier2.await();
for (int i = 0; i < searcherIterations; i++) {
Engine.Searcher searcher = engine.searcher();
TopDocs topDocs =
searcher.searcher().search(new TermQuery(new Term("content", content(i))), 10);
// read one
searcher.searcher().doc(topDocs.scoreDocs[0].doc, new LoadFirstFieldSelector());
searcher.release();
}
} catch (Exception e) {
System.out.println("Searcher thread failed");
e.printStackTrace();
} finally {
latch.countDown();
}
}
@Override
protected ShardValidateQueryResponse shardOperation(ShardValidateQueryRequest request) {
IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
IndexShard indexShard = indexService.getShard(request.shardId().id());
final QueryShardContext queryShardContext = indexService.newQueryShardContext();
queryShardContext.setTypes(request.types());
boolean valid;
String explanation = null;
String error = null;
Engine.Searcher searcher = indexShard.acquireSearcher("validate_query");
DefaultSearchContext searchContext =
new DefaultSearchContext(
0,
new ShardSearchLocalRequest(
request.types(), request.nowInMillis(), request.filteringAliases()),
null,
searcher,
indexService,
indexShard,
scriptService,
pageCacheRecycler,
bigArrays,
threadPool.estimatedTimeInMillisCounter(),
parseFieldMatcher,
SearchService.NO_TIMEOUT);
SearchContext.setCurrent(searchContext);
try {
searchContext.parsedQuery(queryShardContext.toQuery(request.query()));
searchContext.preProcess();
valid = true;
if (request.rewrite()) {
explanation = getRewrittenQuery(searcher.searcher(), searchContext.query());
} else if (request.explain()) {
explanation = searchContext.filteredQuery().query().toString();
}
} catch (QueryShardException | ParsingException e) {
valid = false;
error = e.getDetailedMessage();
} catch (AssertionError | IOException e) {
valid = false;
error = e.getMessage();
} finally {
searchContext.close();
SearchContext.removeCurrent();
}
return new ShardValidateQueryResponse(request.shardId(), valid, explanation, error);
}
private UidField.DocIdAndVersion loadCurrentVersionFromIndex(
BloomCache bloomCache, Engine.Searcher searcher, Term uid) {
UnicodeUtil.UTF8Result utf8 = Unicode.fromStringAsUtf8(uid.text());
for (IndexReader reader : searcher.searcher().subReaders()) {
BloomFilter filter = bloomCache.filter(reader, UidFieldMapper.NAME, true);
// we know that its not there...
if (!filter.isPresent(utf8.result, 0, utf8.length)) {
continue;
}
UidField.DocIdAndVersion docIdAndVersion = UidField.loadDocIdAndVersion(reader, uid);
// either -2 (its there, but no version associated), or an actual version
if (docIdAndVersion.docId != -1) {
return docIdAndVersion;
}
}
return null;
}
private void loadQueries(String indexName) {
IndexService indexService = percolatorIndexService();
IndexShard shard = indexService.shard(0);
Engine.Searcher searcher = shard.searcher();
try {
// create a query to fetch all queries that are registered under the index name (which is the
// type
// in the percolator).
Query query = new DeletionAwareConstantScoreQuery(indexQueriesFilter(indexName));
QueriesLoaderCollector queries = new QueriesLoaderCollector();
searcher.searcher().search(query, queries);
percolator.addQueries(queries.queries());
} catch (IOException e) {
throw new PercolatorException(index, "failed to load queries from percolator index");
} finally {
searcher.release();
}
}
public void initialize(Engine.Searcher docSearcher, ParsedDocument parsedDocument) {
this.docSearcher = docSearcher;
IndexReader indexReader = docSearcher.reader();
LeafReaderContext atomicReaderContext = indexReader.leaves().get(0);
LeafSearchLookup leafLookup = lookup().getLeafSearchLookup(atomicReaderContext);
leafLookup.setDocument(0);
leafLookup.source().setSource(parsedDocument.source());
Map<String, SearchHitField> fields = new HashMap<>();
for (IndexableField field : parsedDocument.rootDoc().getFields()) {
fields.put(field.name(), new InternalSearchHitField(field.name(), Collections.emptyList()));
}
hitContext()
.reset(
new InternalSearchHit(0, "unknown", new StringText(parsedDocument.type()), fields),
atomicReaderContext,
0,
docSearcher.searcher());
}
private void queryBasedPercolating(
Engine.Searcher percolatorSearcher,
PercolateContext context,
QueryCollector percolateCollector)
throws IOException {
Filter percolatorTypeFilter =
context.indexService().mapperService().documentMapper(TYPE_NAME).typeFilter();
percolatorTypeFilter = context.indexService().cache().filter().cache(percolatorTypeFilter);
FilteredQuery query = new FilteredQuery(context.percolateQuery(), percolatorTypeFilter);
percolatorSearcher.searcher().search(query, percolateCollector);
for (Collector queryCollector : percolateCollector.facetAndAggregatorCollector) {
if (queryCollector instanceof XCollector) {
((XCollector) queryCollector).postCollection();
}
}
if (context.facets() != null) {
facetPhase.execute(context);
}
if (context.aggregations() != null) {
aggregationPhase.execute(context);
}
}
private void purgeShards(List<IndexShard> shardsToPurge) {
for (IndexShard shardToPurge : shardsToPurge) {
Query query =
shardToPurge
.mapperService()
.fullName(TTLFieldMapper.NAME)
.rangeQuery(null, System.currentTimeMillis(), false, true);
Engine.Searcher searcher = shardToPurge.acquireSearcher("indices_ttl");
try {
logger.debug(
"[{}][{}] purging shard",
shardToPurge.routingEntry().index(),
shardToPurge.routingEntry().id());
ExpiredDocsCollector expiredDocsCollector = new ExpiredDocsCollector();
searcher.searcher().search(query, expiredDocsCollector);
List<DocToPurge> docsToPurge = expiredDocsCollector.getDocsToPurge();
BulkRequest bulkRequest = new BulkRequest();
for (DocToPurge docToPurge : docsToPurge) {
bulkRequest.add(
new DeleteRequest()
.index(shardToPurge.routingEntry().getIndexName())
.type(docToPurge.type)
.id(docToPurge.id)
.version(docToPurge.version)
.routing(docToPurge.routing));
bulkRequest = processBulkIfNeeded(bulkRequest, false);
}
processBulkIfNeeded(bulkRequest, true);
} catch (Exception e) {
logger.warn("failed to purge", e);
} finally {
searcher.close();
}
}
}
public IndexSearcher docSearcher() {
return docSearcher.searcher();
}
public ContextIndexSearcher(SearchContext searchContext, Engine.Searcher searcher) {
super(searcher.reader());
in = searcher.searcher();
this.searchContext = searchContext;
setSimilarity(searcher.searcher().getSimilarity());
}
