@Override protected ShardSuggestResponse shardOperation(ShardSuggestRequest request) throws ElasticSearchException { IndexService indexService = indicesService.indexServiceSafe(request.index()); IndexShard indexShard = indexService.shardSafe(request.shardId()); final Engine.Searcher searcher = indexShard.searcher(); XContentParser parser = null; try { BytesReference suggest = request.suggest(); if (suggest != null && suggest.length() > 0) { parser = XContentFactory.xContent(suggest).createParser(suggest); if (parser.nextToken() != XContentParser.Token.START_OBJECT) { throw new ElasticSearchIllegalArgumentException("suggest content missing"); } final SuggestionSearchContext context = suggestPhase .parseElement() .parseInternal( parser, indexService.mapperService(), request.index(), request.shardId()); final Suggest result = suggestPhase.execute(context, searcher.reader()); return new ShardSuggestResponse(request.index(), request.shardId(), result); } return new ShardSuggestResponse(request.index(), request.shardId(), new Suggest()); } catch (Throwable ex) { throw new ElasticSearchException("failed to execute suggest", ex); } finally { searcher.release(); if (parser != null) { parser.close(); } } }
public ContextIndexSearcher(SearchContext searchContext, Engine.Searcher searcher) { super(searcher.reader()); in = searcher.searcher(); engineSearcher = searcher; setSimilarity(searcher.searcher().getSimilarity(true)); setQueryCache(searchContext.indexShard().indexService().cache().query()); setQueryCachingPolicy(searchContext.indexShard().getQueryCachingPolicy()); }
private Set<Uid> getShardDocUIDs(final IndexShard shard) throws IOException { shard.refresh("get_uids"); try (Engine.Searcher searcher = shard.acquireSearcher("test")) { Set<Uid> ids = new HashSet<>(); for (LeafReaderContext leafContext : searcher.reader().leaves()) { LeafReader reader = leafContext.reader(); Bits liveDocs = reader.getLiveDocs(); for (int i = 0; i < reader.maxDoc(); i++) { if (liveDocs == null || liveDocs.get(i)) { Document uuid = reader.document(i, Collections.singleton(UidFieldMapper.NAME)); ids.add(Uid.createUid(uuid.get(UidFieldMapper.NAME))); } } } return ids; } }
public void initialize(Engine.Searcher docSearcher, ParsedDocument parsedDocument) { this.docSearcher = docSearcher; IndexReader indexReader = docSearcher.reader(); LeafReaderContext atomicReaderContext = indexReader.leaves().get(0); LeafSearchLookup leafLookup = lookup().getLeafSearchLookup(atomicReaderContext); leafLookup.setDocument(0); leafLookup.source().setSource(parsedDocument.source()); Map<String, SearchHitField> fields = new HashMap<>(); for (IndexableField field : parsedDocument.rootDoc().getFields()) { fields.put(field.name(), new InternalSearchHitField(field.name(), Collections.emptyList())); } hitContext() .reset( new InternalSearchHit(0, "unknown", new StringText(parsedDocument.type()), fields), atomicReaderContext, 0, docSearcher.searcher()); }
@Override protected ShardTermlistResponse shardOperation(ShardTermlistRequest request) throws ElasticSearchException { synchronized (termlistMutex) { InternalIndexShard indexShard = (InternalIndexShard) indicesService.indexServiceSafe(request.index()).shardSafe(request.shardId()); indexShard.store().directory(); Engine.Searcher searcher = indexShard.searcher(); try { Set<String> set = new CompactHashSet(); Fields fields = MultiFields.getFields(searcher.reader()); if (fields != null) { for (Iterator<String> it = fields.iterator(); it.hasNext(); ) { String field = it.next(); if (field.charAt(0) == '_') { continue; } if (request.getField() == null || field.equals(request.getField())) { Terms terms = fields.terms(field); if (terms != null) { TermsEnum termsEnum = terms.iterator(null); BytesRef text; while ((text = termsEnum.next()) != null) { set.add(text.utf8ToString()); System.out.println("field=" + field + "; text=" + text.utf8ToString()); } } } } } return new ShardTermlistResponse(request.index(), request.shardId(), set); } catch (IOException ex) { throw new ElasticSearchException(ex.getMessage(), ex); } } }
void warm(Engine.Searcher searcher, IndexShard shard, IndexSettings settings) { if (shard.state() == IndexShardState.CLOSED) { return; } if (settings.isWarmerEnabled() == false) { return; } if (logger.isTraceEnabled()) { logger.trace("{} top warming [{}]", shard.shardId(), searcher.reader()); } shard.warmerService().onPreWarm(); long time = System.nanoTime(); final List<TerminationHandle> terminationHandles = new ArrayList<>(); // get a handle on pending tasks for (final Listener listener : listeners) { terminationHandles.add(listener.warmReader(shard, searcher)); } // wait for termination for (TerminationHandle terminationHandle : terminationHandles) { try { terminationHandle.awaitTermination(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); logger.warn("top warming has been interrupted", e); break; } } long took = System.nanoTime() - time; shard.warmerService().onPostWarm(took); if (shard.warmerService().logger().isTraceEnabled()) { shard .warmerService() .logger() .trace("top warming took [{}]", new TimeValue(took, TimeUnit.NANOSECONDS)); } }
public ContextIndexSearcher(SearchContext searchContext, Engine.Searcher searcher) { super(searcher.reader()); in = searcher.searcher(); this.searchContext = searchContext; setSimilarity(searcher.searcher().getSimilarity()); }
@Override public PercolateShardResponse doPercolate( PercolateShardRequest request, PercolateContext context) { Engine.Searcher percolatorSearcher = context.indexShard().acquireSearcher("percolate"); try { MatchAndSort matchAndSort = QueryCollector.matchAndSort(logger, context); queryBasedPercolating(percolatorSearcher, context, matchAndSort); TopDocs topDocs = matchAndSort.topDocs(); long count = topDocs.totalHits; List<BytesRef> matches = new ArrayList<BytesRef>(topDocs.scoreDocs.length); float[] scores = new float[topDocs.scoreDocs.length]; List<Map<String, HighlightField>> hls = null; if (context.highlight() != null) { hls = new ArrayList<Map<String, HighlightField>>(topDocs.scoreDocs.length); } final FieldMapper<?> idMapper = context.mapperService().smartNameFieldMapper(IdFieldMapper.NAME); final IndexFieldData<?> idFieldData = context.fieldData().getForField(idMapper); int i = 0; final HashedBytesRef spare = new HashedBytesRef(new BytesRef()); for (ScoreDoc scoreDoc : topDocs.scoreDocs) { int segmentIdx = ReaderUtil.subIndex(scoreDoc.doc, percolatorSearcher.reader().leaves()); AtomicReaderContext atomicReaderContext = percolatorSearcher.reader().leaves().get(segmentIdx); BytesValues values = idFieldData.load(atomicReaderContext).getBytesValues(true); final int localDocId = scoreDoc.doc - atomicReaderContext.docBase; final int numValues = values.setDocument(localDocId); assert numValues == 1; spare.bytes = values.nextValue(); spare.hash = values.currentValueHash(); matches.add(values.copyShared()); if (hls != null) { Query query = context.percolateQueries().get(spare); context.parsedQuery(new ParsedQuery(query, ImmutableMap.<String, Filter>of())); context.hitContext().cache().clear(); highlightPhase.hitExecute(context, context.hitContext()); hls.add(i, context.hitContext().hit().getHighlightFields()); } scores[i++] = scoreDoc.score; } if (hls != null) { return new PercolateShardResponse( matches.toArray(new BytesRef[matches.size()]), hls, count, scores, context, request.index(), request.shardId()); } else { return new PercolateShardResponse( matches.toArray(new BytesRef[matches.size()]), count, scores, context, request.index(), request.shardId()); } } catch (Throwable e) { logger.debug("failed to execute", e); throw new PercolateException(context.indexShard().shardId(), "failed to execute", e); } finally { percolatorSearcher.release(); } }
public TermVectorResponse getTermVector(TermVectorRequest request, String concreteIndex) { final Engine.Searcher searcher = indexShard.acquireSearcher("term_vector"); IndexReader topLevelReader = searcher.reader(); final TermVectorResponse termVectorResponse = new TermVectorResponse(concreteIndex, request.type(), request.id()); final Term uidTerm = new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(request.type(), request.id())); Engine.GetResult get = indexShard.get(new Engine.Get(request.realtime(), uidTerm)); boolean docFromTranslog = get.source() != null; AggregatedDfs dfs = null; /* fetched from translog is treated as an artificial document */ if (docFromTranslog) { request.doc(get.source().source, false); termVectorResponse.setDocVersion(get.version()); } /* handle potential wildcards in fields */ if (request.selectedFields() != null) { handleFieldWildcards(request); } try { Fields topLevelFields = MultiFields.getFields(topLevelReader); Versions.DocIdAndVersion docIdAndVersion = get.docIdAndVersion(); /* from an artificial document */ if (request.doc() != null) { Fields termVectorsByField = generateTermVectorsFromDoc(request, !docFromTranslog); // if no document indexed in shard, take the queried document itself for stats if (topLevelFields == null) { topLevelFields = termVectorsByField; } if (termVectorsByField != null && useDfs(request)) { dfs = getAggregatedDfs(termVectorsByField, request); } termVectorResponse.setFields( termVectorsByField, request.selectedFields(), request.getFlags(), topLevelFields, dfs); termVectorResponse.setExists(true); termVectorResponse.setArtificial(!docFromTranslog); } /* or from an existing document */ else if (docIdAndVersion != null) { // fields with stored term vectors Fields termVectorsByField = docIdAndVersion.context.reader().getTermVectors(docIdAndVersion.docId); Set<String> selectedFields = request.selectedFields(); // generate tvs for fields where analyzer is overridden if (selectedFields == null && request.perFieldAnalyzer() != null) { selectedFields = getFieldsToGenerate(request.perFieldAnalyzer(), termVectorsByField); } // fields without term vectors if (selectedFields != null) { termVectorsByField = addGeneratedTermVectors(get, termVectorsByField, request, selectedFields); } if (termVectorsByField != null && useDfs(request)) { dfs = getAggregatedDfs(termVectorsByField, request); } termVectorResponse.setFields( termVectorsByField, request.selectedFields(), request.getFlags(), topLevelFields, dfs); termVectorResponse.setDocVersion(docIdAndVersion.version); termVectorResponse.setExists(true); } else { termVectorResponse.setExists(false); } } catch (Throwable ex) { throw new ElasticsearchException("failed to execute term vector request", ex); } finally { searcher.close(); get.release(); } return termVectorResponse; }
@Override protected ShardStatus shardOperation(IndexShardStatusRequest request) throws ElasticSearchException { InternalIndexService indexService = (InternalIndexService) indicesService.indexServiceSafe(request.index()); InternalIndexShard indexShard = (InternalIndexShard) indexService.shardSafe(request.shardId()); ShardStatus shardStatus = new ShardStatus(indexShard.routingEntry()); shardStatus.state = indexShard.state(); try { shardStatus.storeSize = indexShard.store().estimateSize(); } catch (IOException e) { // failure to get the store size... } if (indexShard.state() == IndexShardState.STARTED) { // shardStatus.estimatedFlushableMemorySize = // indexShard.estimateFlushableMemorySize(); shardStatus.translogId = indexShard.translog().currentId(); shardStatus.translogOperations = indexShard.translog().estimatedNumberOfOperations(); Engine.Searcher searcher = indexShard.searcher(); try { shardStatus.docs = new DocsStatus(); shardStatus.docs.numDocs = searcher.reader().numDocs(); shardStatus.docs.maxDoc = searcher.reader().maxDoc(); shardStatus.docs.deletedDocs = searcher.reader().numDeletedDocs(); } finally { searcher.release(); } shardStatus.mergeStats = indexShard.mergeScheduler().stats(); shardStatus.refreshStats = indexShard.refreshStats(); shardStatus.flushStats = indexShard.flushStats(); } if (request.recovery) { // check on going recovery (from peer or gateway) RecoveryStatus peerRecoveryStatus = indexShard.peerRecoveryStatus(); if (peerRecoveryStatus == null) { peerRecoveryStatus = peerRecoveryTarget.peerRecoveryStatus(indexShard.shardId()); } if (peerRecoveryStatus != null) { PeerRecoveryStatus.Stage stage; switch (peerRecoveryStatus.stage()) { case INIT: stage = PeerRecoveryStatus.Stage.INIT; break; case INDEX: stage = PeerRecoveryStatus.Stage.INDEX; break; case TRANSLOG: stage = PeerRecoveryStatus.Stage.TRANSLOG; break; case FINALIZE: stage = PeerRecoveryStatus.Stage.FINALIZE; break; case DONE: stage = PeerRecoveryStatus.Stage.DONE; break; default: stage = PeerRecoveryStatus.Stage.INIT; } shardStatus.peerRecoveryStatus = new PeerRecoveryStatus( stage, peerRecoveryStatus.startTime(), peerRecoveryStatus.time(), peerRecoveryStatus.phase1TotalSize(), peerRecoveryStatus.phase1ExistingTotalSize(), peerRecoveryStatus.currentFilesSize(), peerRecoveryStatus.currentTranslogOperations()); } IndexShardGatewayService gatewayService = indexService.shardInjector(request.shardId()).getInstance(IndexShardGatewayService.class); org.elasticsearch.index.gateway.RecoveryStatus gatewayRecoveryStatus = gatewayService.recoveryStatus(); if (gatewayRecoveryStatus != null) { GatewayRecoveryStatus.Stage stage; switch (gatewayRecoveryStatus.stage()) { case INIT: stage = GatewayRecoveryStatus.Stage.INIT; break; case INDEX: stage = GatewayRecoveryStatus.Stage.INDEX; break; case TRANSLOG: stage = GatewayRecoveryStatus.Stage.TRANSLOG; break; case DONE: stage = GatewayRecoveryStatus.Stage.DONE; break; default: stage = GatewayRecoveryStatus.Stage.INIT; } shardStatus.gatewayRecoveryStatus = new GatewayRecoveryStatus( stage, gatewayRecoveryStatus.startTime(), gatewayRecoveryStatus.time(), gatewayRecoveryStatus.index().totalSize(), gatewayRecoveryStatus.index().reusedTotalSize(), gatewayRecoveryStatus.index().currentFilesSize(), gatewayRecoveryStatus.translog().currentTranslogOperations()); } } if (request.snapshot) { IndexShardGatewayService gatewayService = indexService.shardInjector(request.shardId()).getInstance(IndexShardGatewayService.class); SnapshotStatus snapshotStatus = gatewayService.snapshotStatus(); if (snapshotStatus != null) { GatewaySnapshotStatus.Stage stage; switch (snapshotStatus.stage()) { case DONE: stage = GatewaySnapshotStatus.Stage.DONE; break; case FAILURE: stage = GatewaySnapshotStatus.Stage.FAILURE; break; case TRANSLOG: stage = GatewaySnapshotStatus.Stage.TRANSLOG; break; case FINALIZE: stage = GatewaySnapshotStatus.Stage.FINALIZE; break; case INDEX: stage = GatewaySnapshotStatus.Stage.INDEX; break; default: stage = GatewaySnapshotStatus.Stage.NONE; break; } shardStatus.gatewaySnapshotStatus = new GatewaySnapshotStatus( stage, snapshotStatus.startTime(), snapshotStatus.time(), snapshotStatus.index().totalSize(), snapshotStatus.translog().expectedNumberOfOperations()); } } return shardStatus; }