@Override
protected ShardSuggestResponse shardOperation(ShardSuggestRequest request)
throws ElasticSearchException {
IndexService indexService = indicesService.indexServiceSafe(request.index());
IndexShard indexShard = indexService.shardSafe(request.shardId());
final Engine.Searcher searcher = indexShard.searcher();
XContentParser parser = null;
try {
BytesReference suggest = request.suggest();
if (suggest != null && suggest.length() > 0) {
parser = XContentFactory.xContent(suggest).createParser(suggest);
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new ElasticSearchIllegalArgumentException("suggest content missing");
}
final SuggestionSearchContext context =
suggestPhase
.parseElement()
.parseInternal(
parser, indexService.mapperService(), request.index(), request.shardId());
final Suggest result = suggestPhase.execute(context, searcher.reader());
return new ShardSuggestResponse(request.index(), request.shardId(), result);
}
return new ShardSuggestResponse(request.index(), request.shardId(), new Suggest());
} catch (Throwable ex) {
throw new ElasticSearchException("failed to execute suggest", ex);
} finally {
searcher.release();
if (parser != null) {
parser.close();
}
}
}
@Override
protected void shardOperationOnReplica(ReplicaOperationRequest shardRequest) {
ShardDeleteRequest request = shardRequest.request;
IndexShard indexShard =
indicesService
.indexServiceSafe(shardRequest.request.index())
.shardSafe(shardRequest.shardId);
Engine.Delete delete =
indexShard
.prepareDelete(request.type(), request.id(), request.version())
.origin(Engine.Operation.Origin.REPLICA);
// IndexDeleteAction doesn't support version type at the moment. Hard coded for the INTERNAL
// version
delete.versionType(VersionType.INTERNAL.versionTypeForReplicationAndRecovery());
assert delete.versionType().validateVersion(delete.version());
indexShard.delete(delete);
if (request.refresh()) {
try {
indexShard.refresh(new Engine.Refresh("refresh_flag_delete").force(false));
} catch (Exception e) {
// ignore
}
}
}
@Override
protected PrimaryResponse<ShardDeleteResponse, ShardDeleteRequest> shardOperationOnPrimary(
ClusterState clusterState, PrimaryOperationRequest shardRequest) {
ShardDeleteRequest request = shardRequest.request;
IndexShard indexShard =
indicesService
.indexServiceSafe(shardRequest.request.index())
.shardSafe(shardRequest.shardId);
Engine.Delete delete =
indexShard
.prepareDelete(request.type(), request.id(), request.version())
.origin(Engine.Operation.Origin.PRIMARY);
indexShard.delete(delete);
// update the version to happen on the replicas
request.version(delete.version());
if (request.refresh()) {
try {
indexShard.refresh(new Engine.Refresh("refresh_flag_delete").force(false));
} catch (Exception e) {
// ignore
}
}
ShardDeleteResponse response = new ShardDeleteResponse(delete.version(), delete.found());
return new PrimaryResponse<ShardDeleteResponse, ShardDeleteRequest>(
shardRequest.request, response, null);
}
@Inject
public PercolatorService(
Index index,
@IndexSettings Settings indexSettings,
IndicesService indicesService,
PercolatorExecutor percolator) {
super(index, indexSettings);
this.indicesService = indicesService;
this.percolator = percolator;
this.shardLifecycleListener = new ShardLifecycleListener();
this.indicesService.indicesLifecycle().addListener(shardLifecycleListener);
this.percolator.setIndicesService(indicesService);
// if percolator is already allocated, make sure to register real time percolation
if (percolatorAllocated()) {
IndexService percolatorIndexService = percolatorIndexService();
if (percolatorIndexService != null) {
for (IndexShard indexShard : percolatorIndexService) {
try {
indexShard.addListener(realTimePercolatorOperationListener);
} catch (Exception e) {
// ignore
}
}
}
}
}
private void loadQueries(IndexShard shard) {
try {
shard.refresh(new Engine.Refresh("percolator_load_queries").force(true));
// Maybe add a mode load? This isn't really a write. We need write b/c state=post_recovery
Engine.Searcher searcher =
shard.acquireSearcher("percolator_load_queries", IndexShard.Mode.WRITE);
try {
Query query =
new XConstantScoreQuery(
indexCache
.filter()
.cache(
new TermFilter(
new Term(TypeFieldMapper.NAME, PercolatorService.TYPE_NAME))));
QueriesLoaderCollector queryCollector =
new QueriesLoaderCollector(
PercolatorQueriesRegistry.this, logger, mapperService, indexFieldDataService);
searcher.searcher().search(query, queryCollector);
Map<HashedBytesRef, Query> queries = queryCollector.queries();
for (Map.Entry<HashedBytesRef, Query> entry : queries.entrySet()) {
Query previousQuery = percolateQueries.put(entry.getKey(), entry.getValue());
shardPercolateService.addedQuery(entry.getKey(), previousQuery, entry.getValue());
}
} finally {
searcher.release();
}
} catch (Exception e) {
throw new PercolatorException(
shardId.index(), "failed to load queries from percolator index", e);
}
}
private Fields generateTermVectorsFromDoc(TermVectorRequest request, boolean doAllFields)
throws IOException {
// parse the document, at the moment we do update the mapping, just like percolate
ParsedDocument parsedDocument =
parseDocument(indexShard.shardId().getIndex(), request.type(), request.doc());
// select the right fields and generate term vectors
ParseContext.Document doc = parsedDocument.rootDoc();
Collection<String> seenFields = new HashSet<>();
Collection<GetField> getFields = new HashSet<>();
for (IndexableField field : doc.getFields()) {
FieldMapper fieldMapper = indexShard.mapperService().smartNameFieldMapper(field.name());
if (seenFields.contains(field.name())) {
continue;
} else {
seenFields.add(field.name());
}
if (!isValidField(fieldMapper)) {
continue;
}
if (request.selectedFields() == null
&& !doAllFields
&& !fieldMapper.fieldType().storeTermVectors()) {
continue;
}
if (request.selectedFields() != null && !request.selectedFields().contains(field.name())) {
continue;
}
String[] values = doc.getValues(field.name());
getFields.add(new GetField(field.name(), Arrays.asList((Object[]) values)));
}
return generateTermVectors(getFields, request.offsets(), request.perFieldAnalyzer());
}
@Override
protected ShardRefreshResponse shardOperation(ShardRefreshRequest request)
throws ElasticSearchException {
IndexShard indexShard =
indicesService.indexServiceSafe(request.index()).shardSafe(request.shardId());
indexShard.refresh(new Engine.Refresh(request.waitForOperations()));
return new ShardRefreshResponse(request.index(), request.shardId());
}
@Override
public void afterIndexShardCreated(IndexShard indexShard) {
// add a listener that will update based on changes done to the _percolate index
// the relevant indices with loaded queries
if (indexShard.shardId().index().name().equals(INDEX_NAME)) {
indexShard.addListener(realTimePercolatorOperationListener);
}
}
protected ExplainResponse shardOperation(ExplainRequest request, int shardId)
throws ElasticSearchException {
IndexService indexService = indicesService.indexService(request.index());
IndexShard indexShard = indexService.shardSafe(shardId);
Term uidTerm =
new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(request.getType(), request.getId()));
Engine.GetResult result = indexShard.get(new Engine.Get(false, uidTerm));
if (!result.exists()) {
return new ExplainResponse(false);
}
SearchContext context =
new SearchContext(
0,
new ShardSearchRequest()
.types(new String[] {request.getType()})
.filteringAliases(request.getFilteringAlias()),
null,
result.searcher(),
indexService,
indexShard,
scriptService);
SearchContext.setCurrent(context);
try {
context.parsedQuery(parseQuery(request, indexService));
context.preProcess();
int topLevelDocId = result.docIdAndVersion().docId + result.docIdAndVersion().reader.docBase;
Explanation explanation;
if (context.rescore() != null) {
RescoreSearchContext ctx = context.rescore();
Rescorer rescorer = ctx.rescorer();
explanation = rescorer.explain(topLevelDocId, context, ctx);
} else {
explanation = context.searcher().explain(context.query(), topLevelDocId);
}
if (request.getFields() != null) {
if (request.getFields().length == 1 && "_source".equals(request.getFields()[0])) {
request.setFields(null); // Load the _source field
}
// Advantage is that we're not opening a second searcher to retrieve the _source. Also
// because we are working in the same searcher in engineGetResult we can be sure that a
// doc isn't deleted between the initial get and this call.
GetResult getResult =
indexShard
.getService()
.get(result, request.getId(), request.getType(), request.getFields());
return new ExplainResponse(true, explanation, getResult);
} else {
return new ExplainResponse(true, explanation);
}
} catch (IOException e) {
throw new ElasticSearchException("Could not explain", e);
} finally {
context.release();
SearchContext.removeCurrent();
}
}
@Override
protected ShardFlushResponse shardOperation(ShardFlushRequest request)
throws ElasticSearchException {
IndexShard indexShard =
indicesService.indexServiceSafe(request.index()).shardSafe(request.shardId());
indexShard.flush(
new Engine.Flush().refresh(request.refresh()).full(request.full()).force(request.force()));
return new ShardFlushResponse(request.index(), request.shardId());
}
@Override
protected MultiGetShardResponse shardOperation(MultiGetShardRequest request, int shardId)
throws ElasticSearchException {
IndexService indexService = indicesService.indexServiceSafe(request.index());
IndexShard indexShard = indexService.shardSafe(shardId);
if (request.refresh() && !request.realtime()) {
indexShard.refresh(
new Engine.Refresh("refresh_flag_mget").force(TransportGetAction.REFRESH_FORCE));
}
MultiGetShardResponse response = new MultiGetShardResponse();
for (int i = 0; i < request.locations.size(); i++) {
String type = request.types.get(i);
String id = request.ids.get(i);
String[] fields = request.fields.get(i);
long version = request.versions.get(i);
VersionType versionType = request.versionTypes.get(i);
if (versionType == null) {
versionType = VersionType.INTERNAL;
}
FetchSourceContext fetchSourceContext = request.fetchSourceContexts.get(i);
try {
GetResult getResult =
indexShard
.getService()
.get(
type, id, fields, request.realtime(), version, versionType, fetchSourceContext);
response.add(request.locations.get(i), new GetResponse(getResult));
} catch (Throwable t) {
if (TransportActions.isShardNotAvailableException(t)) {
throw (ElasticSearchException) t;
} else {
logger.debug(
"[{}][{}] failed to execute multi_get for [{}]/[{}]",
t,
request.index(),
shardId,
type,
id);
response.add(
request.locations.get(i),
new MultiGetResponse.Failure(
request.index(), type, id, ExceptionsHelper.detailedMessage(t)));
}
}
}
return response;
}
private ParsedDocument parseDocument(String index, String type, BytesReference doc) {
MapperService mapperService = indexShard.mapperService();
IndexService indexService = indexShard.indexService();
// TODO: make parsing not dynamically create fields not in the original mapping
Tuple<DocumentMapper, Boolean> docMapper = mapperService.documentMapperWithAutoCreate(type);
ParsedDocument parsedDocument =
docMapper.v1().parse(source(doc).type(type).flyweight(true)).setMappingsModified(docMapper);
if (parsedDocument.mappingsModified()) {
mappingUpdatedAction.updateMappingOnMaster(index, docMapper.v1(), indexService.indexUUID());
}
return parsedDocument;
}
@Override
public void afterIndexShardStarted(IndexShard indexShard) {
if (indexShard.shardId().index().name().equals(INDEX_NAME)) {
// percolator index has started, fetch what we can from it and initialize the indices
// we have
synchronized (mutex) {
if (initialQueriesFetchDone) {
return;
}
// we load the queries for all existing indices
for (IndexService indexService : indicesService) {
// only load queries for "this" index percolator service
if (indexService.index().equals(index())) {
logger.debug(
"loading percolator queries for index [{}]...", indexService.index().name());
loadQueries(indexService.index().name());
logger.trace(
"done loading percolator queries for index [{}]", indexService.index().name());
}
}
initialQueriesFetchDone = true;
}
}
if (!indexShard.shardId().index().equals(index())) {
// not our index, bail
return;
}
if (!percolatorAllocated()) {
return;
}
// we are only interested when the first shard on this node has been created for an index
// when it does, fetch the relevant queries if not fetched already
IndexService indexService = indicesService.indexService(indexShard.shardId().index().name());
if (indexService == null) {
return;
}
if (indexService.numberOfShards() != 1) {
return;
}
synchronized (mutex) {
if (initialQueriesFetchDone) {
return;
}
// we load queries for this index
logger.debug("loading percolator queries for index [{}]...", indexService.index().name());
loadQueries(index.name());
logger.trace("done loading percolator queries for index [{}]", indexService.index().name());
initialQueriesFetchDone = true;
}
}
@Override
protected ShardOptimizeResponse shardOperation(ShardOptimizeRequest request)
throws ElasticsearchException {
IndexShard indexShard =
indicesService.indexServiceSafe(request.index()).shardSafe(request.shardId());
indexShard.optimize(
new Engine.Optimize()
.waitForMerge(request.waitForMerge())
.maxNumSegments(request.maxNumSegments())
.onlyExpungeDeletes(request.onlyExpungeDeletes())
.flush(request.flush())
.force(request.force()));
return new ShardOptimizeResponse(request.index(), request.shardId());
}
private Fields addGeneratedTermVectors(
Engine.GetResult get,
Fields termVectorsByField,
TermVectorRequest request,
Set<String> selectedFields)
throws IOException {
/* only keep valid fields */
Set<String> validFields = new HashSet<>();
for (String field : selectedFields) {
FieldMapper fieldMapper = indexShard.mapperService().smartNameFieldMapper(field);
if (!isValidField(fieldMapper)) {
continue;
}
// already retrieved, only if the analyzer hasn't been overridden at the field
if (fieldMapper.fieldType().storeTermVectors()
&& (request.perFieldAnalyzer() == null
|| !request.perFieldAnalyzer().containsKey(field))) {
continue;
}
validFields.add(field);
}
if (validFields.isEmpty()) {
return termVectorsByField;
}
/* generate term vectors from fetched document fields */
GetResult getResult =
indexShard
.getService()
.get(
get,
request.id(),
request.type(),
validFields.toArray(Strings.EMPTY_ARRAY),
null,
false);
Fields generatedTermVectors =
generateTermVectors(
getResult.getFields().values(), request.offsets(), request.perFieldAnalyzer());
/* merge with existing Fields */
if (termVectorsByField == null) {
return generatedTermVectors;
} else {
return mergeFields(termVectorsByField, generatedTermVectors);
}
}
public void close() {
this.indicesService.indicesLifecycle().removeListener(shardLifecycleListener);
// clean up any index that has registered real time updated from the percolator shards allocated
// on this node
IndexService percolatorIndexService = percolatorIndexService();
if (percolatorIndexService != null) {
for (IndexShard indexShard : percolatorIndexService) {
try {
indexShard.removeListener(realTimePercolatorOperationListener);
} catch (Exception e) {
// ignore
}
}
}
}
private void handleFieldWildcards(TermVectorRequest request) {
Set<String> fieldNames = new HashSet<>();
for (String pattern : request.selectedFields()) {
fieldNames.addAll(indexShard.mapperService().simpleMatchToIndexNames(pattern));
}
request.selectedFields(fieldNames.toArray(Strings.EMPTY_ARRAY));
}
private void loadQueries(String indexName) {
IndexService indexService = percolatorIndexService();
IndexShard shard = indexService.shard(0);
Engine.Searcher searcher = shard.searcher();
try {
// create a query to fetch all queries that are registered under the index name (which is the
// type
// in the percolator).
Query query = new DeletionAwareConstantScoreQuery(indexQueriesFilter(indexName));
QueriesLoaderCollector queries = new QueriesLoaderCollector();
searcher.searcher().search(query, queries);
percolator.addQueries(queries.queries());
} catch (IOException e) {
throw new PercolatorException(index, "failed to load queries from percolator index");
} finally {
searcher.release();
}
}
@Override
protected void shardOperationOnReplica(ReplicaOperationRequest shardRequest) {
IndexShard indexShard =
indicesService
.indexServiceSafe(shardRequest.request.index())
.shardSafe(shardRequest.shardId);
final IngestShardRequest request = shardRequest.request;
int size = request.items().size();
for (int i = 0; i < size; i++) {
IngestItemRequest item = request.items().get(i);
if (item == null) {
continue;
}
if (item.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) item.request();
try {
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source())
.type(indexRequest.type())
.id(indexRequest.id())
.routing(indexRequest.routing())
.parent(indexRequest.parent())
.timestamp(indexRequest.timestamp())
.ttl(indexRequest.ttl());
if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
Engine.Index index =
indexShard
.prepareIndex(sourceToParse)
.version(indexRequest.version())
.origin(Engine.Operation.Origin.REPLICA);
indexShard.index(index);
} else {
Engine.Create create =
indexShard
.prepareCreate(sourceToParse)
.version(indexRequest.version())
.origin(Engine.Operation.Origin.REPLICA);
indexShard.create(create);
}
} catch (Exception e) {
// ignore, we are on backup
}
} else if (item.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) item.request();
try {
Engine.Delete delete =
indexShard
.prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version())
.origin(Engine.Operation.Origin.REPLICA);
indexShard.delete(delete);
} catch (Exception e) {
// ignore, we are on backup
}
}
}
}
@Override
protected MultiTermVectorsShardResponse shardOperation(
MultiTermVectorsShardRequest request, int shardId) throws ElasticsearchException {
MultiTermVectorsShardResponse response = new MultiTermVectorsShardResponse();
for (int i = 0; i < request.locations.size(); i++) {
TermVectorRequest termVectorRequest = request.requests.get(i);
try {
IndexService indexService = indicesService.indexServiceSafe(request.index());
IndexShard indexShard = indexService.shardSafe(shardId);
TermVectorResponse termVectorResponse =
indexShard.termVectorService().getTermVector(termVectorRequest);
response.add(request.locations.get(i), termVectorResponse);
} catch (Throwable t) {
if (TransportActions.isShardNotAvailableException(t)) {
throw (ElasticsearchException) t;
} else {
logger.debug(
"[{}][{}] failed to execute multi term vectors for [{}]/[{}]",
t,
request.index(),
shardId,
termVectorRequest.type(),
termVectorRequest.id());
response.add(
request.locations.get(i),
new MultiTermVectorsResponse.Failure(
request.index(),
termVectorRequest.type(),
termVectorRequest.id(),
ExceptionsHelper.detailedMessage(t)));
}
}
}
return response;
}
@Override
protected GetResponse shardOperation(GetRequest request, int shardId)
throws ElasticsearchException {
IndexService indexService = indicesService.indexServiceSafe(request.index());
IndexShard indexShard = indexService.shardSafe(shardId);
if (request.refresh() && !request.realtime()) {
indexShard.refresh(new Engine.Refresh("refresh_flag_get").force(REFRESH_FORCE));
}
GetResult result =
indexShard
.getService()
.get(
request.type(),
request.id(),
request.fields(),
request.realtime(),
request.version(),
request.versionType(),
request.fetchSourceContext());
return new GetResponse(result);
}
private Analyzer getAnalyzerAtField(
String field, @Nullable Map<String, String> perFieldAnalyzer) {
MapperService mapperService = indexShard.mapperService();
Analyzer analyzer;
if (perFieldAnalyzer != null && perFieldAnalyzer.containsKey(field)) {
analyzer = mapperService.analysisService().analyzer(perFieldAnalyzer.get(field).toString());
} else {
analyzer = mapperService.smartNameFieldMapper(field).indexAnalyzer();
}
if (analyzer == null) {
analyzer = mapperService.analysisService().defaultIndexAnalyzer();
}
return analyzer;
}
@Override
public void indexShardStateChanged(
IndexShard indexShard,
@Nullable IndexShardState previousState,
IndexShardState newState,
@Nullable String reason) {
List<IndexShardState> shardStates =
this.shardStates.putIfAbsent(
indexShard.shardId(),
new CopyOnWriteArrayList<IndexShardState>(new IndexShardState[] {newState}));
if (shardStates != null) {
shardStates.add(newState);
}
}
@Override
protected void shardOperationOnReplica(ReplicaOperationRequest shardRequest) {
IndexShard indexShard =
indicesService
.indexServiceSafe(shardRequest.shardId.getIndex())
.shardSafe(shardRequest.shardId.id());
IndexRequest request = shardRequest.request;
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.REPLICA, request.source())
.type(request.type())
.id(request.id())
.routing(request.routing())
.parent(request.parent())
.timestamp(request.timestamp())
.ttl(request.ttl());
if (request.opType() == IndexRequest.OpType.INDEX) {
Engine.Index index =
indexShard.prepareIndex(
sourceToParse,
request.version(),
request.versionType(),
Engine.Operation.Origin.REPLICA,
request.canHaveDuplicates());
indexShard.index(index);
} else {
Engine.Create create =
indexShard.prepareCreate(
sourceToParse,
request.version(),
request.versionType(),
Engine.Operation.Origin.REPLICA,
request.canHaveDuplicates(),
request.autoGeneratedId());
indexShard.create(create);
}
if (request.refresh()) {
try {
indexShard.refresh(new Engine.Refresh("refresh_flag_index").force(false));
} catch (Exception e) {
// ignore
}
}
}
@Override
protected ExplainResponse shardOperation(ExplainRequest request, ShardId shardId)
throws ElasticsearchException {
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
IndexShard indexShard = indexService.shardSafe(shardId.id());
Term uidTerm =
new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(request.type(), request.id()));
Engine.GetResult result = indexShard.get(new Engine.Get(false, uidTerm));
if (!result.exists()) {
return new ExplainResponse(shardId.getIndex(), request.type(), request.id(), false);
}
SearchContext context =
new DefaultSearchContext(
0,
new ShardSearchRequest(request)
.types(new String[] {request.type()})
.filteringAliases(request.filteringAlias())
.nowInMillis(request.nowInMillis),
null,
result.searcher(),
indexService,
indexShard,
scriptService,
pageCacheRecycler,
bigArrays,
threadPool.estimatedTimeInMillisCounter());
SearchContext.setCurrent(context);
try {
context.parsedQuery(indexService.queryParserService().parseQuery(request.source()));
context.preProcess();
int topLevelDocId = result.docIdAndVersion().docId + result.docIdAndVersion().context.docBase;
Explanation explanation = context.searcher().explain(context.query(), topLevelDocId);
for (RescoreSearchContext ctx : context.rescore()) {
Rescorer rescorer = ctx.rescorer();
explanation = rescorer.explain(topLevelDocId, context, ctx, explanation);
}
if (request.fields() != null
|| (request.fetchSourceContext() != null && request.fetchSourceContext().fetchSource())) {
// Advantage is that we're not opening a second searcher to retrieve the _source. Also
// because we are working in the same searcher in engineGetResult we can be sure that a
// doc isn't deleted between the initial get and this call.
GetResult getResult =
indexShard
.getService()
.get(
result,
request.id(),
request.type(),
request.fields(),
request.fetchSourceContext(),
false);
return new ExplainResponse(
shardId.getIndex(), request.type(), request.id(), true, explanation, getResult);
} else {
return new ExplainResponse(
shardId.getIndex(), request.type(), request.id(), true, explanation);
}
} catch (IOException e) {
throw new ElasticsearchException("Could not explain", e);
} finally {
context.close();
SearchContext.removeCurrent();
}
}
public TermVectorResponse getTermVector(TermVectorRequest request, String concreteIndex) {
final Engine.Searcher searcher = indexShard.acquireSearcher("term_vector");
IndexReader topLevelReader = searcher.reader();
final TermVectorResponse termVectorResponse =
new TermVectorResponse(concreteIndex, request.type(), request.id());
final Term uidTerm =
new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(request.type(), request.id()));
Engine.GetResult get = indexShard.get(new Engine.Get(request.realtime(), uidTerm));
boolean docFromTranslog = get.source() != null;
AggregatedDfs dfs = null;
/* fetched from translog is treated as an artificial document */
if (docFromTranslog) {
request.doc(get.source().source, false);
termVectorResponse.setDocVersion(get.version());
}
/* handle potential wildcards in fields */
if (request.selectedFields() != null) {
handleFieldWildcards(request);
}
try {
Fields topLevelFields = MultiFields.getFields(topLevelReader);
Versions.DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
/* from an artificial document */
if (request.doc() != null) {
Fields termVectorsByField = generateTermVectorsFromDoc(request, !docFromTranslog);
// if no document indexed in shard, take the queried document itself for stats
if (topLevelFields == null) {
topLevelFields = termVectorsByField;
}
if (termVectorsByField != null && useDfs(request)) {
dfs = getAggregatedDfs(termVectorsByField, request);
}
termVectorResponse.setFields(
termVectorsByField, request.selectedFields(), request.getFlags(), topLevelFields, dfs);
termVectorResponse.setExists(true);
termVectorResponse.setArtificial(!docFromTranslog);
}
/* or from an existing document */
else if (docIdAndVersion != null) {
// fields with stored term vectors
Fields termVectorsByField =
docIdAndVersion.context.reader().getTermVectors(docIdAndVersion.docId);
Set<String> selectedFields = request.selectedFields();
// generate tvs for fields where analyzer is overridden
if (selectedFields == null && request.perFieldAnalyzer() != null) {
selectedFields = getFieldsToGenerate(request.perFieldAnalyzer(), termVectorsByField);
}
// fields without term vectors
if (selectedFields != null) {
termVectorsByField =
addGeneratedTermVectors(get, termVectorsByField, request, selectedFields);
}
if (termVectorsByField != null && useDfs(request)) {
dfs = getAggregatedDfs(termVectorsByField, request);
}
termVectorResponse.setFields(
termVectorsByField, request.selectedFields(), request.getFlags(), topLevelFields, dfs);
termVectorResponse.setDocVersion(docIdAndVersion.version);
termVectorResponse.setExists(true);
} else {
termVectorResponse.setExists(false);
}
} catch (Throwable ex) {
throw new ElasticsearchException("failed to execute term vector request", ex);
} finally {
searcher.close();
get.release();
}
return termVectorResponse;
}
@Override
protected PrimaryResponse<IndexResponse, IndexRequest> shardOperationOnPrimary(
ClusterState clusterState, PrimaryOperationRequest shardRequest) {
final IndexRequest request = shardRequest.request;
// validate, if routing is required, that we got routing
IndexMetaData indexMetaData = clusterState.metaData().index(shardRequest.shardId.getIndex());
MappingMetaData mappingMd = indexMetaData.mappingOrDefault(request.type());
if (mappingMd != null && mappingMd.routing().required()) {
if (request.routing() == null) {
throw new RoutingMissingException(
shardRequest.shardId.getIndex(), request.type(), request.id());
}
}
IndexService indexService = indicesService.indexServiceSafe(shardRequest.shardId.getIndex());
IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id());
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.PRIMARY, request.source())
.type(request.type())
.id(request.id())
.routing(request.routing())
.parent(request.parent())
.timestamp(request.timestamp())
.ttl(request.ttl());
long version;
boolean created;
Engine.IndexingOperation op;
if (request.opType() == IndexRequest.OpType.INDEX) {
Engine.Index index =
indexShard.prepareIndex(
sourceToParse,
request.version(),
request.versionType(),
Engine.Operation.Origin.PRIMARY,
request.canHaveDuplicates());
if (index.parsedDoc().mappingsModified()) {
mappingUpdatedAction.updateMappingOnMaster(
shardRequest.shardId.getIndex(), index.docMapper(), indexService.indexUUID());
}
indexShard.index(index);
version = index.version();
op = index;
created = index.created();
} else {
Engine.Create create =
indexShard.prepareCreate(
sourceToParse,
request.version(),
request.versionType(),
Engine.Operation.Origin.PRIMARY,
request.canHaveDuplicates(),
request.autoGeneratedId());
if (create.parsedDoc().mappingsModified()) {
mappingUpdatedAction.updateMappingOnMaster(
shardRequest.shardId.getIndex(), create.docMapper(), indexService.indexUUID());
}
indexShard.create(create);
version = create.version();
op = create;
created = true;
}
if (request.refresh()) {
try {
indexShard.refresh(new Engine.Refresh("refresh_flag_index").force(false));
} catch (Throwable e) {
// ignore
}
}
// update the version on the request, so it will be used for the replicas
request.version(version);
request.versionType(request.versionType().versionTypeForReplicationAndRecovery());
assert request.versionType().validateVersionForWrites(request.version());
IndexResponse response =
new IndexResponse(
shardRequest.shardId.getIndex(), request.type(), request.id(), version, created);
return new PrimaryResponse<>(shardRequest.request, response, op);
}
private void deleteShard(
int shardId, boolean delete, boolean snapshotGateway, boolean deleteGateway)
throws ElasticSearchException {
Injector shardInjector;
IndexShard indexShard;
synchronized (this) {
Map<Integer, Injector> tmpShardInjectors = newHashMap(shardsInjectors);
shardInjector = tmpShardInjectors.remove(shardId);
if (shardInjector == null) {
if (!delete) {
return;
}
throw new IndexShardMissingException(new ShardId(index, shardId));
}
shardsInjectors = ImmutableMap.copyOf(tmpShardInjectors);
if (delete) {
logger.debug("deleting shard_id [{}]", shardId);
}
Map<Integer, IndexShard> tmpShardsMap = newHashMap(shards);
indexShard = tmpShardsMap.remove(shardId);
shards = ImmutableMap.copyOf(tmpShardsMap);
}
ShardId sId = new ShardId(index, shardId);
indicesLifecycle.beforeIndexShardClosed(sId, indexShard, delete);
for (Class<? extends CloseableIndexComponent> closeable : pluginsService.shardServices()) {
try {
shardInjector.getInstance(closeable).close(delete);
} catch (Exception e) {
logger.debug("failed to clean plugin shard service [{}]", e, closeable);
}
}
try {
// now we can close the translog service, we need to close it before the we close the shard
shardInjector.getInstance(TranslogService.class).close();
} catch (Exception e) {
// ignore
}
// close shard actions
if (indexShard != null) {
shardInjector.getInstance(IndexShardManagement.class).close();
}
// this logic is tricky, we want to close the engine so we rollback the changes done to it
// and close the shard so no operations are allowed to it
if (indexShard != null) {
indexShard.close();
}
try {
shardInjector.getInstance(Engine.class).close();
} catch (Exception e) {
// ignore
}
try {
// now, we can snapshot to the gateway, it will be only the translog
if (snapshotGateway) {
shardInjector.getInstance(IndexShardGatewayService.class).snapshotOnClose();
}
} catch (Exception e) {
// ignore
}
try {
shardInjector.getInstance(IndexShardGatewayService.class).close(deleteGateway);
} catch (Exception e) {
// ignore
}
try {
// now we can close the translog
shardInjector.getInstance(Translog.class).close(delete);
} catch (Exception e) {
// ignore
}
// call this before we close the store, so we can release resources for it
indicesLifecycle.afterIndexShardClosed(sId, delete);
// if we delete or have no gateway or the store is not persistent, clean the store...
Store store = shardInjector.getInstance(Store.class);
if (delete || indexGateway.type().equals(NoneGateway.TYPE) || !indexStore.persistent()) {
try {
store.fullDelete();
} catch (IOException e) {
logger.warn("failed to clean store on shard deletion", e);
}
}
// and close it
try {
store.close();
} catch (IOException e) {
logger.warn("failed to close store on shard deletion", e);
}
Injectors.close(injector);
// delete the shard location if needed
if (delete || indexGateway.type().equals(NoneGateway.TYPE)) {
FileSystemUtils.deleteRecursively(nodeEnv.shardLocation(sId));
}
}
private boolean hasPercolatorType(IndexShard indexShard) {
ShardId otherShardId = indexShard.shardId();
return shardId.equals(otherShardId) && mapperService.hasMapping(PercolatorService.TYPE_NAME);
}
public PercolateShardResponse percolate(PercolateShardRequest request) {
IndexService percolateIndexService = indicesService.indexServiceSafe(request.index());
IndexShard indexShard = percolateIndexService.shardSafe(request.shardId());
ShardPercolateService shardPercolateService = indexShard.shardPercolateService();
shardPercolateService.prePercolate();
long startTime = System.nanoTime();
SearchShardTarget searchShardTarget =
new SearchShardTarget(clusterService.localNode().id(), request.index(), request.shardId());
final PercolateContext context =
new PercolateContext(
request,
searchShardTarget,
indexShard,
percolateIndexService,
cacheRecycler,
pageCacheRecycler,
bigArrays,
scriptService);
try {
ParsedDocument parsedDocument = parseRequest(percolateIndexService, request, context);
if (context.percolateQueries().isEmpty()) {
return new PercolateShardResponse(context, request.index(), request.shardId());
}
if (request.docSource() != null && request.docSource().length() != 0) {
parsedDocument =
parseFetchedDoc(
context, request.docSource(), percolateIndexService, request.documentType());
} else if (parsedDocument == null) {
throw new ElasticsearchIllegalArgumentException("Nothing to percolate");
}
if (context.percolateQuery() == null
&& (context.trackScores()
|| context.doSort
|| context.facets() != null
|| context.aggregations() != null)) {
context.percolateQuery(new MatchAllDocsQuery());
}
if (context.doSort && !context.limit) {
throw new ElasticsearchIllegalArgumentException("Can't sort if size isn't specified");
}
if (context.highlight() != null && !context.limit) {
throw new ElasticsearchIllegalArgumentException("Can't highlight if size isn't specified");
}
if (context.size() < 0) {
context.size(0);
}
// parse the source either into one MemoryIndex, if it is a single document or index multiple
// docs if nested
PercolatorIndex percolatorIndex;
if (indexShard.mapperService().documentMapper(request.documentType()).hasNestedObjects()) {
percolatorIndex = multi;
} else {
percolatorIndex = single;
}
PercolatorType action;
if (request.onlyCount()) {
action = context.percolateQuery() != null ? queryCountPercolator : countPercolator;
} else {
if (context.doSort) {
action = topMatchingPercolator;
} else if (context.percolateQuery() != null) {
action = context.trackScores() ? scoringPercolator : queryPercolator;
} else {
action = matchPercolator;
}
}
context.percolatorTypeId = action.id();
percolatorIndex.prepare(context, parsedDocument);
indexShard.readAllowed();
return action.doPercolate(request, context);
} finally {
context.release();
shardPercolateService.postPercolate(System.nanoTime() - startTime);
}
}
