@Override
protected ShardSuggestResponse shardOperation(ShardSuggestRequest request)
throws ElasticSearchException {
IndexService indexService = indicesService.indexServiceSafe(request.index());
IndexShard indexShard = indexService.shardSafe(request.shardId());
final Engine.Searcher searcher = indexShard.searcher();
XContentParser parser = null;
try {
BytesReference suggest = request.suggest();
if (suggest != null && suggest.length() > 0) {
parser = XContentFactory.xContent(suggest).createParser(suggest);
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new ElasticSearchIllegalArgumentException("suggest content missing");
}
final SuggestionSearchContext context =
suggestPhase
.parseElement()
.parseInternal(
parser, indexService.mapperService(), request.index(), request.shardId());
final Suggest result = suggestPhase.execute(context, searcher.reader());
return new ShardSuggestResponse(request.index(), request.shardId(), result);
}
return new ShardSuggestResponse(request.index(), request.shardId(), new Suggest());
} catch (Throwable ex) {
throw new ElasticSearchException("failed to execute suggest", ex);
} finally {
searcher.release();
if (parser != null) {
parser.close();
}
}
}
public ContextIndexSearcher(SearchContext searchContext, Engine.Searcher searcher) {
super(searcher.reader());
in = searcher.searcher();
engineSearcher = searcher;
setSimilarity(searcher.searcher().getSimilarity(true));
setQueryCache(searchContext.indexShard().indexService().cache().query());
setQueryCachingPolicy(searchContext.indexShard().getQueryCachingPolicy());
}
private Set<Uid> getShardDocUIDs(final IndexShard shard) throws IOException {
shard.refresh("get_uids");
try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
Set<Uid> ids = new HashSet<>();
for (LeafReaderContext leafContext : searcher.reader().leaves()) {
LeafReader reader = leafContext.reader();
Bits liveDocs = reader.getLiveDocs();
for (int i = 0; i < reader.maxDoc(); i++) {
if (liveDocs == null || liveDocs.get(i)) {
Document uuid = reader.document(i, Collections.singleton(UidFieldMapper.NAME));
ids.add(Uid.createUid(uuid.get(UidFieldMapper.NAME)));
}
}
}
return ids;
}
}
public void initialize(Engine.Searcher docSearcher, ParsedDocument parsedDocument) {
this.docSearcher = docSearcher;
IndexReader indexReader = docSearcher.reader();
LeafReaderContext atomicReaderContext = indexReader.leaves().get(0);
LeafSearchLookup leafLookup = lookup().getLeafSearchLookup(atomicReaderContext);
leafLookup.setDocument(0);
leafLookup.source().setSource(parsedDocument.source());
Map<String, SearchHitField> fields = new HashMap<>();
for (IndexableField field : parsedDocument.rootDoc().getFields()) {
fields.put(field.name(), new InternalSearchHitField(field.name(), Collections.emptyList()));
}
hitContext()
.reset(
new InternalSearchHit(0, "unknown", new StringText(parsedDocument.type()), fields),
atomicReaderContext,
0,
docSearcher.searcher());
}
@Override
protected ShardTermlistResponse shardOperation(ShardTermlistRequest request)
throws ElasticSearchException {
synchronized (termlistMutex) {
InternalIndexShard indexShard =
(InternalIndexShard)
indicesService.indexServiceSafe(request.index()).shardSafe(request.shardId());
indexShard.store().directory();
Engine.Searcher searcher = indexShard.searcher();
try {
Set<String> set = new CompactHashSet();
Fields fields = MultiFields.getFields(searcher.reader());
if (fields != null) {
for (Iterator<String> it = fields.iterator(); it.hasNext(); ) {
String field = it.next();
if (field.charAt(0) == '_') {
continue;
}
if (request.getField() == null || field.equals(request.getField())) {
Terms terms = fields.terms(field);
if (terms != null) {
TermsEnum termsEnum = terms.iterator(null);
BytesRef text;
while ((text = termsEnum.next()) != null) {
set.add(text.utf8ToString());
System.out.println("field=" + field + "; text=" + text.utf8ToString());
}
}
}
}
}
return new ShardTermlistResponse(request.index(), request.shardId(), set);
} catch (IOException ex) {
throw new ElasticSearchException(ex.getMessage(), ex);
}
}
}
void warm(Engine.Searcher searcher, IndexShard shard, IndexSettings settings) {
if (shard.state() == IndexShardState.CLOSED) {
return;
}
if (settings.isWarmerEnabled() == false) {
return;
}
if (logger.isTraceEnabled()) {
logger.trace("{} top warming [{}]", shard.shardId(), searcher.reader());
}
shard.warmerService().onPreWarm();
long time = System.nanoTime();
final List<TerminationHandle> terminationHandles = new ArrayList<>();
// get a handle on pending tasks
for (final Listener listener : listeners) {
terminationHandles.add(listener.warmReader(shard, searcher));
}
// wait for termination
for (TerminationHandle terminationHandle : terminationHandles) {
try {
terminationHandle.awaitTermination();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
logger.warn("top warming has been interrupted", e);
break;
}
}
long took = System.nanoTime() - time;
shard.warmerService().onPostWarm(took);
if (shard.warmerService().logger().isTraceEnabled()) {
shard
.warmerService()
.logger()
.trace("top warming took [{}]", new TimeValue(took, TimeUnit.NANOSECONDS));
}
}
public ContextIndexSearcher(SearchContext searchContext, Engine.Searcher searcher) {
super(searcher.reader());
in = searcher.searcher();
this.searchContext = searchContext;
setSimilarity(searcher.searcher().getSimilarity());
}
@Override
public PercolateShardResponse doPercolate(
PercolateShardRequest request, PercolateContext context) {
Engine.Searcher percolatorSearcher = context.indexShard().acquireSearcher("percolate");
try {
MatchAndSort matchAndSort = QueryCollector.matchAndSort(logger, context);
queryBasedPercolating(percolatorSearcher, context, matchAndSort);
TopDocs topDocs = matchAndSort.topDocs();
long count = topDocs.totalHits;
List<BytesRef> matches = new ArrayList<BytesRef>(topDocs.scoreDocs.length);
float[] scores = new float[topDocs.scoreDocs.length];
List<Map<String, HighlightField>> hls = null;
if (context.highlight() != null) {
hls = new ArrayList<Map<String, HighlightField>>(topDocs.scoreDocs.length);
}
final FieldMapper<?> idMapper =
context.mapperService().smartNameFieldMapper(IdFieldMapper.NAME);
final IndexFieldData<?> idFieldData = context.fieldData().getForField(idMapper);
int i = 0;
final HashedBytesRef spare = new HashedBytesRef(new BytesRef());
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
int segmentIdx =
ReaderUtil.subIndex(scoreDoc.doc, percolatorSearcher.reader().leaves());
AtomicReaderContext atomicReaderContext =
percolatorSearcher.reader().leaves().get(segmentIdx);
BytesValues values = idFieldData.load(atomicReaderContext).getBytesValues(true);
final int localDocId = scoreDoc.doc - atomicReaderContext.docBase;
final int numValues = values.setDocument(localDocId);
assert numValues == 1;
spare.bytes = values.nextValue();
spare.hash = values.currentValueHash();
matches.add(values.copyShared());
if (hls != null) {
Query query = context.percolateQueries().get(spare);
context.parsedQuery(new ParsedQuery(query, ImmutableMap.<String, Filter>of()));
context.hitContext().cache().clear();
highlightPhase.hitExecute(context, context.hitContext());
hls.add(i, context.hitContext().hit().getHighlightFields());
}
scores[i++] = scoreDoc.score;
}
if (hls != null) {
return new PercolateShardResponse(
matches.toArray(new BytesRef[matches.size()]),
hls,
count,
scores,
context,
request.index(),
request.shardId());
} else {
return new PercolateShardResponse(
matches.toArray(new BytesRef[matches.size()]),
count,
scores,
context,
request.index(),
request.shardId());
}
} catch (Throwable e) {
logger.debug("failed to execute", e);
throw new PercolateException(context.indexShard().shardId(), "failed to execute", e);
} finally {
percolatorSearcher.release();
}
}
public TermVectorResponse getTermVector(TermVectorRequest request, String concreteIndex) {
final Engine.Searcher searcher = indexShard.acquireSearcher("term_vector");
IndexReader topLevelReader = searcher.reader();
final TermVectorResponse termVectorResponse =
new TermVectorResponse(concreteIndex, request.type(), request.id());
final Term uidTerm =
new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(request.type(), request.id()));
Engine.GetResult get = indexShard.get(new Engine.Get(request.realtime(), uidTerm));
boolean docFromTranslog = get.source() != null;
AggregatedDfs dfs = null;
/* fetched from translog is treated as an artificial document */
if (docFromTranslog) {
request.doc(get.source().source, false);
termVectorResponse.setDocVersion(get.version());
}
/* handle potential wildcards in fields */
if (request.selectedFields() != null) {
handleFieldWildcards(request);
}
try {
Fields topLevelFields = MultiFields.getFields(topLevelReader);
Versions.DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
/* from an artificial document */
if (request.doc() != null) {
Fields termVectorsByField = generateTermVectorsFromDoc(request, !docFromTranslog);
// if no document indexed in shard, take the queried document itself for stats
if (topLevelFields == null) {
topLevelFields = termVectorsByField;
}
if (termVectorsByField != null && useDfs(request)) {
dfs = getAggregatedDfs(termVectorsByField, request);
}
termVectorResponse.setFields(
termVectorsByField, request.selectedFields(), request.getFlags(), topLevelFields, dfs);
termVectorResponse.setExists(true);
termVectorResponse.setArtificial(!docFromTranslog);
}
/* or from an existing document */
else if (docIdAndVersion != null) {
// fields with stored term vectors
Fields termVectorsByField =
docIdAndVersion.context.reader().getTermVectors(docIdAndVersion.docId);
Set<String> selectedFields = request.selectedFields();
// generate tvs for fields where analyzer is overridden
if (selectedFields == null && request.perFieldAnalyzer() != null) {
selectedFields = getFieldsToGenerate(request.perFieldAnalyzer(), termVectorsByField);
}
// fields without term vectors
if (selectedFields != null) {
termVectorsByField =
addGeneratedTermVectors(get, termVectorsByField, request, selectedFields);
}
if (termVectorsByField != null && useDfs(request)) {
dfs = getAggregatedDfs(termVectorsByField, request);
}
termVectorResponse.setFields(
termVectorsByField, request.selectedFields(), request.getFlags(), topLevelFields, dfs);
termVectorResponse.setDocVersion(docIdAndVersion.version);
termVectorResponse.setExists(true);
} else {
termVectorResponse.setExists(false);
}
} catch (Throwable ex) {
throw new ElasticsearchException("failed to execute term vector request", ex);
} finally {
searcher.close();
get.release();
}
return termVectorResponse;
}
@Override
protected ShardStatus shardOperation(IndexShardStatusRequest request)
throws ElasticSearchException {
InternalIndexService indexService =
(InternalIndexService) indicesService.indexServiceSafe(request.index());
InternalIndexShard indexShard = (InternalIndexShard) indexService.shardSafe(request.shardId());
ShardStatus shardStatus = new ShardStatus(indexShard.routingEntry());
shardStatus.state = indexShard.state();
try {
shardStatus.storeSize = indexShard.store().estimateSize();
} catch (IOException e) {
// failure to get the store size...
}
if (indexShard.state() == IndexShardState.STARTED) {
// shardStatus.estimatedFlushableMemorySize =
// indexShard.estimateFlushableMemorySize();
shardStatus.translogId = indexShard.translog().currentId();
shardStatus.translogOperations = indexShard.translog().estimatedNumberOfOperations();
Engine.Searcher searcher = indexShard.searcher();
try {
shardStatus.docs = new DocsStatus();
shardStatus.docs.numDocs = searcher.reader().numDocs();
shardStatus.docs.maxDoc = searcher.reader().maxDoc();
shardStatus.docs.deletedDocs = searcher.reader().numDeletedDocs();
} finally {
searcher.release();
}
shardStatus.mergeStats = indexShard.mergeScheduler().stats();
shardStatus.refreshStats = indexShard.refreshStats();
shardStatus.flushStats = indexShard.flushStats();
}
if (request.recovery) {
// check on going recovery (from peer or gateway)
RecoveryStatus peerRecoveryStatus = indexShard.peerRecoveryStatus();
if (peerRecoveryStatus == null) {
peerRecoveryStatus = peerRecoveryTarget.peerRecoveryStatus(indexShard.shardId());
}
if (peerRecoveryStatus != null) {
PeerRecoveryStatus.Stage stage;
switch (peerRecoveryStatus.stage()) {
case INIT:
stage = PeerRecoveryStatus.Stage.INIT;
break;
case INDEX:
stage = PeerRecoveryStatus.Stage.INDEX;
break;
case TRANSLOG:
stage = PeerRecoveryStatus.Stage.TRANSLOG;
break;
case FINALIZE:
stage = PeerRecoveryStatus.Stage.FINALIZE;
break;
case DONE:
stage = PeerRecoveryStatus.Stage.DONE;
break;
default:
stage = PeerRecoveryStatus.Stage.INIT;
}
shardStatus.peerRecoveryStatus =
new PeerRecoveryStatus(
stage,
peerRecoveryStatus.startTime(),
peerRecoveryStatus.time(),
peerRecoveryStatus.phase1TotalSize(),
peerRecoveryStatus.phase1ExistingTotalSize(),
peerRecoveryStatus.currentFilesSize(),
peerRecoveryStatus.currentTranslogOperations());
}
IndexShardGatewayService gatewayService =
indexService.shardInjector(request.shardId()).getInstance(IndexShardGatewayService.class);
org.elasticsearch.index.gateway.RecoveryStatus gatewayRecoveryStatus =
gatewayService.recoveryStatus();
if (gatewayRecoveryStatus != null) {
GatewayRecoveryStatus.Stage stage;
switch (gatewayRecoveryStatus.stage()) {
case INIT:
stage = GatewayRecoveryStatus.Stage.INIT;
break;
case INDEX:
stage = GatewayRecoveryStatus.Stage.INDEX;
break;
case TRANSLOG:
stage = GatewayRecoveryStatus.Stage.TRANSLOG;
break;
case DONE:
stage = GatewayRecoveryStatus.Stage.DONE;
break;
default:
stage = GatewayRecoveryStatus.Stage.INIT;
}
shardStatus.gatewayRecoveryStatus =
new GatewayRecoveryStatus(
stage,
gatewayRecoveryStatus.startTime(),
gatewayRecoveryStatus.time(),
gatewayRecoveryStatus.index().totalSize(),
gatewayRecoveryStatus.index().reusedTotalSize(),
gatewayRecoveryStatus.index().currentFilesSize(),
gatewayRecoveryStatus.translog().currentTranslogOperations());
}
}
if (request.snapshot) {
IndexShardGatewayService gatewayService =
indexService.shardInjector(request.shardId()).getInstance(IndexShardGatewayService.class);
SnapshotStatus snapshotStatus = gatewayService.snapshotStatus();
if (snapshotStatus != null) {
GatewaySnapshotStatus.Stage stage;
switch (snapshotStatus.stage()) {
case DONE:
stage = GatewaySnapshotStatus.Stage.DONE;
break;
case FAILURE:
stage = GatewaySnapshotStatus.Stage.FAILURE;
break;
case TRANSLOG:
stage = GatewaySnapshotStatus.Stage.TRANSLOG;
break;
case FINALIZE:
stage = GatewaySnapshotStatus.Stage.FINALIZE;
break;
case INDEX:
stage = GatewaySnapshotStatus.Stage.INDEX;
break;
default:
stage = GatewaySnapshotStatus.Stage.NONE;
break;
}
shardStatus.gatewaySnapshotStatus =
new GatewaySnapshotStatus(
stage,
snapshotStatus.startTime(),
snapshotStatus.time(),
snapshotStatus.index().totalSize(),
snapshotStatus.translog().expectedNumberOfOperations());
}
}
return shardStatus;
}