Example #1
0
 @Override
 protected ShardSuggestResponse shardOperation(ShardSuggestRequest request)
     throws ElasticSearchException {
   IndexService indexService = indicesService.indexServiceSafe(request.index());
   IndexShard indexShard = indexService.shardSafe(request.shardId());
   final Engine.Searcher searcher = indexShard.searcher();
   XContentParser parser = null;
   try {
     BytesReference suggest = request.suggest();
     if (suggest != null && suggest.length() > 0) {
       parser = XContentFactory.xContent(suggest).createParser(suggest);
       if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
         throw new ElasticSearchIllegalArgumentException("suggest content missing");
       }
       final SuggestionSearchContext context =
           suggestPhase
               .parseElement()
               .parseInternal(
                   parser, indexService.mapperService(), request.index(), request.shardId());
       final Suggest result = suggestPhase.execute(context, searcher.reader());
       return new ShardSuggestResponse(request.index(), request.shardId(), result);
     }
     return new ShardSuggestResponse(request.index(), request.shardId(), new Suggest());
   } catch (Throwable ex) {
     throw new ElasticSearchException("failed to execute suggest", ex);
   } finally {
     searcher.release();
     if (parser != null) {
       parser.close();
     }
   }
 }
  protected ExplainResponse shardOperation(ExplainRequest request, int shardId)
      throws ElasticSearchException {
    IndexService indexService = indicesService.indexService(request.index());
    IndexShard indexShard = indexService.shardSafe(shardId);
    Term uidTerm =
        new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(request.getType(), request.getId()));
    Engine.GetResult result = indexShard.get(new Engine.Get(false, uidTerm));
    if (!result.exists()) {
      return new ExplainResponse(false);
    }

    SearchContext context =
        new SearchContext(
            0,
            new ShardSearchRequest()
                .types(new String[] {request.getType()})
                .filteringAliases(request.getFilteringAlias()),
            null,
            result.searcher(),
            indexService,
            indexShard,
            scriptService);
    SearchContext.setCurrent(context);

    try {
      context.parsedQuery(parseQuery(request, indexService));
      context.preProcess();
      int topLevelDocId = result.docIdAndVersion().docId + result.docIdAndVersion().reader.docBase;
      Explanation explanation;
      if (context.rescore() != null) {
        RescoreSearchContext ctx = context.rescore();
        Rescorer rescorer = ctx.rescorer();
        explanation = rescorer.explain(topLevelDocId, context, ctx);
      } else {
        explanation = context.searcher().explain(context.query(), topLevelDocId);
      }
      if (request.getFields() != null) {
        if (request.getFields().length == 1 && "_source".equals(request.getFields()[0])) {
          request.setFields(null); // Load the _source field
        }
        // Advantage is that we're not opening a second searcher to retrieve the _source. Also
        // because we are working in the same searcher in engineGetResult we can be sure that a
        // doc isn't deleted between the initial get and this call.
        GetResult getResult =
            indexShard
                .getService()
                .get(result, request.getId(), request.getType(), request.getFields());
        return new ExplainResponse(true, explanation, getResult);
      } else {
        return new ExplainResponse(true, explanation);
      }
    } catch (IOException e) {
      throw new ElasticSearchException("Could not explain", e);
    } finally {
      context.release();
      SearchContext.removeCurrent();
    }
  }
  @Override
  protected MultiGetShardResponse shardOperation(MultiGetShardRequest request, int shardId)
      throws ElasticSearchException {
    IndexService indexService = indicesService.indexServiceSafe(request.index());
    IndexShard indexShard = indexService.shardSafe(shardId);

    if (request.refresh() && !request.realtime()) {
      indexShard.refresh(
          new Engine.Refresh("refresh_flag_mget").force(TransportGetAction.REFRESH_FORCE));
    }

    MultiGetShardResponse response = new MultiGetShardResponse();
    for (int i = 0; i < request.locations.size(); i++) {
      String type = request.types.get(i);
      String id = request.ids.get(i);
      String[] fields = request.fields.get(i);

      long version = request.versions.get(i);
      VersionType versionType = request.versionTypes.get(i);
      if (versionType == null) {
        versionType = VersionType.INTERNAL;
      }

      FetchSourceContext fetchSourceContext = request.fetchSourceContexts.get(i);
      try {
        GetResult getResult =
            indexShard
                .getService()
                .get(
                    type, id, fields, request.realtime(), version, versionType, fetchSourceContext);
        response.add(request.locations.get(i), new GetResponse(getResult));
      } catch (Throwable t) {
        if (TransportActions.isShardNotAvailableException(t)) {
          throw (ElasticSearchException) t;
        } else {
          logger.debug(
              "[{}][{}] failed to execute multi_get for [{}]/[{}]",
              t,
              request.index(),
              shardId,
              type,
              id);
          response.add(
              request.locations.get(i),
              new MultiGetResponse.Failure(
                  request.index(), type, id, ExceptionsHelper.detailedMessage(t)));
        }
      }
    }

    return response;
  }
  @Override
  protected MultiTermVectorsShardResponse shardOperation(
      MultiTermVectorsShardRequest request, int shardId) throws ElasticsearchException {

    MultiTermVectorsShardResponse response = new MultiTermVectorsShardResponse();
    for (int i = 0; i < request.locations.size(); i++) {
      TermVectorRequest termVectorRequest = request.requests.get(i);

      try {
        IndexService indexService = indicesService.indexServiceSafe(request.index());
        IndexShard indexShard = indexService.shardSafe(shardId);
        TermVectorResponse termVectorResponse =
            indexShard.termVectorService().getTermVector(termVectorRequest);
        response.add(request.locations.get(i), termVectorResponse);
      } catch (Throwable t) {
        if (TransportActions.isShardNotAvailableException(t)) {
          throw (ElasticsearchException) t;
        } else {
          logger.debug(
              "[{}][{}] failed to execute multi term vectors for [{}]/[{}]",
              t,
              request.index(),
              shardId,
              termVectorRequest.type(),
              termVectorRequest.id());
          response.add(
              request.locations.get(i),
              new MultiTermVectorsResponse.Failure(
                  request.index(),
                  termVectorRequest.type(),
                  termVectorRequest.id(),
                  ExceptionsHelper.detailedMessage(t)));
        }
      }
    }

    return response;
  }
  @Override
  protected GetResponse shardOperation(GetRequest request, int shardId)
      throws ElasticsearchException {
    IndexService indexService = indicesService.indexServiceSafe(request.index());
    IndexShard indexShard = indexService.shardSafe(shardId);

    if (request.refresh() && !request.realtime()) {
      indexShard.refresh(new Engine.Refresh("refresh_flag_get").force(REFRESH_FORCE));
    }

    GetResult result =
        indexShard
            .getService()
            .get(
                request.type(),
                request.id(),
                request.fields(),
                request.realtime(),
                request.version(),
                request.versionType(),
                request.fetchSourceContext());
    return new GetResponse(result);
  }
  @Override
  protected PrimaryResponse<IndexResponse, IndexRequest> shardOperationOnPrimary(
      ClusterState clusterState, PrimaryOperationRequest shardRequest) {
    final IndexRequest request = shardRequest.request;

    // validate, if routing is required, that we got routing
    IndexMetaData indexMetaData = clusterState.metaData().index(shardRequest.shardId.getIndex());
    MappingMetaData mappingMd = indexMetaData.mappingOrDefault(request.type());
    if (mappingMd != null && mappingMd.routing().required()) {
      if (request.routing() == null) {
        throw new RoutingMissingException(
            shardRequest.shardId.getIndex(), request.type(), request.id());
      }
    }

    IndexService indexService = indicesService.indexServiceSafe(shardRequest.shardId.getIndex());
    IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id());
    SourceToParse sourceToParse =
        SourceToParse.source(SourceToParse.Origin.PRIMARY, request.source())
            .type(request.type())
            .id(request.id())
            .routing(request.routing())
            .parent(request.parent())
            .timestamp(request.timestamp())
            .ttl(request.ttl());
    long version;
    boolean created;
    Engine.IndexingOperation op;
    if (request.opType() == IndexRequest.OpType.INDEX) {
      Engine.Index index =
          indexShard.prepareIndex(
              sourceToParse,
              request.version(),
              request.versionType(),
              Engine.Operation.Origin.PRIMARY,
              request.canHaveDuplicates());
      if (index.parsedDoc().mappingsModified()) {
        mappingUpdatedAction.updateMappingOnMaster(
            shardRequest.shardId.getIndex(), index.docMapper(), indexService.indexUUID());
      }
      indexShard.index(index);
      version = index.version();
      op = index;
      created = index.created();
    } else {
      Engine.Create create =
          indexShard.prepareCreate(
              sourceToParse,
              request.version(),
              request.versionType(),
              Engine.Operation.Origin.PRIMARY,
              request.canHaveDuplicates(),
              request.autoGeneratedId());
      if (create.parsedDoc().mappingsModified()) {
        mappingUpdatedAction.updateMappingOnMaster(
            shardRequest.shardId.getIndex(), create.docMapper(), indexService.indexUUID());
      }
      indexShard.create(create);
      version = create.version();
      op = create;
      created = true;
    }
    if (request.refresh()) {
      try {
        indexShard.refresh(new Engine.Refresh("refresh_flag_index").force(false));
      } catch (Throwable e) {
        // ignore
      }
    }

    // update the version on the request, so it will be used for the replicas
    request.version(version);
    request.versionType(request.versionType().versionTypeForReplicationAndRecovery());

    assert request.versionType().validateVersionForWrites(request.version());

    IndexResponse response =
        new IndexResponse(
            shardRequest.shardId.getIndex(), request.type(), request.id(), version, created);
    return new PrimaryResponse<>(shardRequest.request, response, op);
  }
  public PercolateShardResponse percolate(PercolateShardRequest request) {
    IndexService percolateIndexService = indicesService.indexServiceSafe(request.index());
    IndexShard indexShard = percolateIndexService.shardSafe(request.shardId());

    ShardPercolateService shardPercolateService = indexShard.shardPercolateService();
    shardPercolateService.prePercolate();
    long startTime = System.nanoTime();

    SearchShardTarget searchShardTarget =
        new SearchShardTarget(clusterService.localNode().id(), request.index(), request.shardId());
    final PercolateContext context =
        new PercolateContext(
            request,
            searchShardTarget,
            indexShard,
            percolateIndexService,
            cacheRecycler,
            pageCacheRecycler,
            bigArrays,
            scriptService);
    try {
      ParsedDocument parsedDocument = parseRequest(percolateIndexService, request, context);
      if (context.percolateQueries().isEmpty()) {
        return new PercolateShardResponse(context, request.index(), request.shardId());
      }

      if (request.docSource() != null && request.docSource().length() != 0) {
        parsedDocument =
            parseFetchedDoc(
                context, request.docSource(), percolateIndexService, request.documentType());
      } else if (parsedDocument == null) {
        throw new ElasticsearchIllegalArgumentException("Nothing to percolate");
      }

      if (context.percolateQuery() == null
          && (context.trackScores()
              || context.doSort
              || context.facets() != null
              || context.aggregations() != null)) {
        context.percolateQuery(new MatchAllDocsQuery());
      }

      if (context.doSort && !context.limit) {
        throw new ElasticsearchIllegalArgumentException("Can't sort if size isn't specified");
      }

      if (context.highlight() != null && !context.limit) {
        throw new ElasticsearchIllegalArgumentException("Can't highlight if size isn't specified");
      }

      if (context.size() < 0) {
        context.size(0);
      }

      // parse the source either into one MemoryIndex, if it is a single document or index multiple
      // docs if nested
      PercolatorIndex percolatorIndex;
      if (indexShard.mapperService().documentMapper(request.documentType()).hasNestedObjects()) {
        percolatorIndex = multi;
      } else {
        percolatorIndex = single;
      }

      PercolatorType action;
      if (request.onlyCount()) {
        action = context.percolateQuery() != null ? queryCountPercolator : countPercolator;
      } else {
        if (context.doSort) {
          action = topMatchingPercolator;
        } else if (context.percolateQuery() != null) {
          action = context.trackScores() ? scoringPercolator : queryPercolator;
        } else {
          action = matchPercolator;
        }
      }
      context.percolatorTypeId = action.id();

      percolatorIndex.prepare(context, parsedDocument);

      indexShard.readAllowed();
      return action.doPercolate(request, context);
    } finally {
      context.release();
      shardPercolateService.postPercolate(System.nanoTime() - startTime);
    }
  }
  @Override
  protected ExplainResponse shardOperation(ExplainRequest request, ShardId shardId)
      throws ElasticsearchException {
    IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
    IndexShard indexShard = indexService.shardSafe(shardId.id());
    Term uidTerm =
        new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(request.type(), request.id()));
    Engine.GetResult result = indexShard.get(new Engine.Get(false, uidTerm));
    if (!result.exists()) {
      return new ExplainResponse(shardId.getIndex(), request.type(), request.id(), false);
    }

    SearchContext context =
        new DefaultSearchContext(
            0,
            new ShardSearchRequest(request)
                .types(new String[] {request.type()})
                .filteringAliases(request.filteringAlias())
                .nowInMillis(request.nowInMillis),
            null,
            result.searcher(),
            indexService,
            indexShard,
            scriptService,
            pageCacheRecycler,
            bigArrays,
            threadPool.estimatedTimeInMillisCounter());
    SearchContext.setCurrent(context);

    try {
      context.parsedQuery(indexService.queryParserService().parseQuery(request.source()));
      context.preProcess();
      int topLevelDocId = result.docIdAndVersion().docId + result.docIdAndVersion().context.docBase;
      Explanation explanation = context.searcher().explain(context.query(), topLevelDocId);
      for (RescoreSearchContext ctx : context.rescore()) {
        Rescorer rescorer = ctx.rescorer();
        explanation = rescorer.explain(topLevelDocId, context, ctx, explanation);
      }
      if (request.fields() != null
          || (request.fetchSourceContext() != null && request.fetchSourceContext().fetchSource())) {
        // Advantage is that we're not opening a second searcher to retrieve the _source. Also
        // because we are working in the same searcher in engineGetResult we can be sure that a
        // doc isn't deleted between the initial get and this call.
        GetResult getResult =
            indexShard
                .getService()
                .get(
                    result,
                    request.id(),
                    request.type(),
                    request.fields(),
                    request.fetchSourceContext(),
                    false);
        return new ExplainResponse(
            shardId.getIndex(), request.type(), request.id(), true, explanation, getResult);
      } else {
        return new ExplainResponse(
            shardId.getIndex(), request.type(), request.id(), true, explanation);
      }
    } catch (IOException e) {
      throw new ElasticsearchException("Could not explain", e);
    } finally {
      context.close();
      SearchContext.removeCurrent();
    }
  }
  @Override
  protected GetResponse shardOperation(GetRequest request, int shardId)
      throws ElasticSearchException {
    IndexService indexService = indicesService.indexServiceSafe(request.index());
    BloomCache bloomCache = indexService.cache().bloomCache();
    IndexShard indexShard = indexService.shardSafe(shardId);

    DocumentMapper docMapper = indexService.mapperService().documentMapper(request.type());
    if (docMapper == null) {
      throw new TypeMissingException(new Index(request.index()), request.type());
    }

    if (request.refresh()) {
      indexShard.refresh(new Engine.Refresh(false));
    }

    Engine.Searcher searcher = indexShard.searcher();
    boolean exists = false;
    byte[] source = null;
    Map<String, GetField> fields = null;
    long version = -1;
    try {
      UidField.DocIdAndVersion docIdAndVersion =
          loadCurrentVersionFromIndex(
              bloomCache, searcher, docMapper.uidMapper().term(request.type(), request.id()));
      if (docIdAndVersion != null && docIdAndVersion.docId != Lucene.NO_DOC) {
        if (docIdAndVersion.version > 0) {
          version = docIdAndVersion.version;
        }
        exists = true;
        FieldSelector fieldSelector = buildFieldSelectors(docMapper, request.fields());
        if (fieldSelector != null) {
          Document doc = docIdAndVersion.reader.document(docIdAndVersion.docId, fieldSelector);
          source = extractSource(doc, docMapper);

          for (Object oField : doc.getFields()) {
            Fieldable field = (Fieldable) oField;
            String name = field.name();
            Object value = null;
            FieldMappers fieldMappers = docMapper.mappers().indexName(field.name());
            if (fieldMappers != null) {
              FieldMapper mapper = fieldMappers.mapper();
              if (mapper != null) {
                name = mapper.names().fullName();
                value = mapper.valueForSearch(field);
              }
            }
            if (value == null) {
              if (field.isBinary()) {
                value = field.getBinaryValue();
              } else {
                value = field.stringValue();
              }
            }

            if (fields == null) {
              fields = newHashMapWithExpectedSize(2);
            }

            GetField getField = fields.get(name);
            if (getField == null) {
              getField = new GetField(name, new ArrayList<Object>(2));
              fields.put(name, getField);
            }
            getField.values().add(value);
          }
        }

        // now, go and do the script thingy if needed
        if (request.fields() != null && request.fields().length > 0) {
          SearchLookup searchLookup = null;
          for (String field : request.fields()) {
            String script = null;
            if (field.contains("_source.") || field.contains("doc[")) {
              script = field;
            } else {
              FieldMappers x = docMapper.mappers().smartName(field);
              if (x != null && !x.mapper().stored()) {
                script = "_source." + x.mapper().names().fullName();
              }
            }
            if (script != null) {
              if (searchLookup == null) {
                searchLookup =
                    new SearchLookup(
                        indexService.mapperService(), indexService.cache().fieldData());
              }
              SearchScript searchScript = scriptService.search(searchLookup, "mvel", script, null);
              searchScript.setNextReader(docIdAndVersion.reader);
              searchScript.setNextDocId(docIdAndVersion.docId);

              try {
                Object value = searchScript.run();
                if (fields == null) {
                  fields = newHashMapWithExpectedSize(2);
                }
                GetField getField = fields.get(field);
                if (getField == null) {
                  getField = new GetField(field, new ArrayList<Object>(2));
                  fields.put(field, getField);
                }
                getField.values().add(value);
              } catch (RuntimeException e) {
                if (logger.isTraceEnabled()) {
                  logger.trace("failed to execute get request script field [{}]", e, script);
                }
                // ignore
              }
            }
          }
        }
      }
    } catch (IOException e) {
      throw new ElasticSearchException(
          "Failed to get type [" + request.type() + "] and id [" + request.id() + "]", e);
    } finally {
      searcher.release();
    }
    return new GetResponse(
        request.index(), request.type(), request.id(), version, exists, source, fields);
  }
  protected void shardOperation(
      final UpdateRequest request,
      final ActionListener<UpdateResponse> listener,
      final int retryCount)
      throws ElasticSearchException {
    IndexService indexService = indicesService.indexServiceSafe(request.index());
    IndexShard indexShard = indexService.shardSafe(request.shardId());

    long getDate = System.currentTimeMillis();
    final GetResult getResult =
        indexShard
            .getService()
            .get(
                request.type(),
                request.id(),
                new String[] {
                  SourceFieldMapper.NAME,
                  RoutingFieldMapper.NAME,
                  ParentFieldMapper.NAME,
                  TTLFieldMapper.NAME
                },
                true);

    // no doc, what to do, what to do...
    if (!getResult.isExists()) {
      if (request.upsertRequest() == null) {
        listener.onFailure(
            new DocumentMissingException(
                new ShardId(request.index(), request.shardId()), request.type(), request.id()));
        return;
      }
      final IndexRequest indexRequest = request.upsertRequest();
      indexRequest
          .index(request.index())
          .type(request.type())
          .id(request.id())
          // it has to be a "create!"
          .create(true)
          .routing(request.routing())
          .percolate(request.percolate())
          .refresh(request.refresh())
          .replicationType(request.replicationType())
          .consistencyLevel(request.consistencyLevel());
      indexRequest.operationThreaded(false);
      // we fetch it from the index request so we don't generate the bytes twice, its already done
      // in the index request
      final BytesReference updateSourceBytes = indexRequest.source();
      indexAction.execute(
          indexRequest,
          new ActionListener<IndexResponse>() {
            @Override
            public void onResponse(IndexResponse response) {
              UpdateResponse update =
                  new UpdateResponse(
                      response.getIndex(),
                      response.getType(),
                      response.getId(),
                      response.getVersion());
              update.setMatches(response.getMatches());
              if (request.fields() != null && request.fields().length > 0) {
                Tuple<XContentType, Map<String, Object>> sourceAndContent =
                    XContentHelper.convertToMap(updateSourceBytes, true);
                update.setGetResult(
                    extractGetResult(
                        request,
                        response.getVersion(),
                        sourceAndContent.v2(),
                        sourceAndContent.v1(),
                        updateSourceBytes));
              } else {
                update.setGetResult(null);
              }
              listener.onResponse(update);
            }

            @Override
            public void onFailure(Throwable e) {
              e = ExceptionsHelper.unwrapCause(e);
              if (e instanceof VersionConflictEngineException
                  || e instanceof DocumentAlreadyExistsException) {
                if (retryCount < request.retryOnConflict()) {
                  threadPool
                      .executor(executor())
                      .execute(
                          new Runnable() {
                            @Override
                            public void run() {
                              shardOperation(request, listener, retryCount + 1);
                            }
                          });
                  return;
                }
              }
              listener.onFailure(e);
            }
          });
      return;
    }

    if (getResult.internalSourceRef() == null) {
      // no source, we can't do nothing, through a failure...
      listener.onFailure(
          new DocumentSourceMissingException(
              new ShardId(request.index(), request.shardId()), request.type(), request.id()));
      return;
    }

    Tuple<XContentType, Map<String, Object>> sourceAndContent =
        XContentHelper.convertToMap(getResult.internalSourceRef(), true);
    String operation = null;
    String timestamp = null;
    Long ttl = null;
    Object fetchedTTL = null;
    final Map<String, Object> updatedSourceAsMap;
    final XContentType updateSourceContentType = sourceAndContent.v1();
    String routing =
        getResult.getFields().containsKey(RoutingFieldMapper.NAME)
            ? getResult.field(RoutingFieldMapper.NAME).getValue().toString()
            : null;
    String parent =
        getResult.getFields().containsKey(ParentFieldMapper.NAME)
            ? getResult.field(ParentFieldMapper.NAME).getValue().toString()
            : null;

    if (request.script() == null && request.doc() != null) {
      IndexRequest indexRequest = request.doc();
      updatedSourceAsMap = sourceAndContent.v2();
      if (indexRequest.ttl() > 0) {
        ttl = indexRequest.ttl();
      }
      timestamp = indexRequest.timestamp();
      if (indexRequest.routing() != null) {
        routing = indexRequest.routing();
      }
      if (indexRequest.parent() != null) {
        parent = indexRequest.parent();
      }
      XContentHelper.update(updatedSourceAsMap, indexRequest.sourceAsMap());
    } else {
      Map<String, Object> ctx = new HashMap<String, Object>(2);
      ctx.put("_source", sourceAndContent.v2());

      try {
        ExecutableScript script =
            scriptService.executable(request.scriptLang, request.script, request.scriptParams);
        script.setNextVar("ctx", ctx);
        script.run();
        // we need to unwrap the ctx...
        ctx = (Map<String, Object>) script.unwrap(ctx);
      } catch (Exception e) {
        throw new ElasticSearchIllegalArgumentException("failed to execute script", e);
      }

      operation = (String) ctx.get("op");
      timestamp = (String) ctx.get("_timestamp");
      fetchedTTL = ctx.get("_ttl");
      if (fetchedTTL != null) {
        if (fetchedTTL instanceof Number) {
          ttl = ((Number) fetchedTTL).longValue();
        } else {
          ttl = TimeValue.parseTimeValue((String) fetchedTTL, null).millis();
        }
      }

      updatedSourceAsMap = (Map<String, Object>) ctx.get("_source");
    }

    // apply script to update the source
    // No TTL has been given in the update script so we keep previous TTL value if there is one
    if (ttl == null) {
      ttl =
          getResult.getFields().containsKey(TTLFieldMapper.NAME)
              ? (Long) getResult.field(TTLFieldMapper.NAME).getValue()
              : null;
      if (ttl != null) {
        ttl =
            ttl
                - (System.currentTimeMillis()
                    - getDate); // It is an approximation of exact TTL value, could be improved
      }
    }

    // TODO: external version type, does it make sense here? does not seem like it...

    if (operation == null || "index".equals(operation)) {
      final IndexRequest indexRequest =
          Requests.indexRequest(request.index())
              .type(request.type())
              .id(request.id())
              .routing(routing)
              .parent(parent)
              .source(updatedSourceAsMap, updateSourceContentType)
              .version(getResult.getVersion())
              .replicationType(request.replicationType())
              .consistencyLevel(request.consistencyLevel())
              .timestamp(timestamp)
              .ttl(ttl)
              .percolate(request.percolate())
              .refresh(request.refresh());
      indexRequest.operationThreaded(false);
      // we fetch it from the index request so we don't generate the bytes twice, its already done
      // in the index request
      final BytesReference updateSourceBytes = indexRequest.source();
      indexAction.execute(
          indexRequest,
          new ActionListener<IndexResponse>() {
            @Override
            public void onResponse(IndexResponse response) {
              UpdateResponse update =
                  new UpdateResponse(
                      response.getIndex(),
                      response.getType(),
                      response.getId(),
                      response.getVersion());
              update.setMatches(response.getMatches());
              update.setGetResult(
                  extractGetResult(
                      request,
                      response.getVersion(),
                      updatedSourceAsMap,
                      updateSourceContentType,
                      updateSourceBytes));
              listener.onResponse(update);
            }

            @Override
            public void onFailure(Throwable e) {
              e = ExceptionsHelper.unwrapCause(e);
              if (e instanceof VersionConflictEngineException) {
                if (retryCount < request.retryOnConflict()) {
                  threadPool
                      .executor(executor())
                      .execute(
                          new Runnable() {
                            @Override
                            public void run() {
                              shardOperation(request, listener, retryCount + 1);
                            }
                          });
                  return;
                }
              }
              listener.onFailure(e);
            }
          });
    } else if ("delete".equals(operation)) {
      DeleteRequest deleteRequest =
          Requests.deleteRequest(request.index())
              .type(request.type())
              .id(request.id())
              .routing(routing)
              .parent(parent)
              .version(getResult.getVersion())
              .replicationType(request.replicationType())
              .consistencyLevel(request.consistencyLevel());
      deleteRequest.operationThreaded(false);
      deleteAction.execute(
          deleteRequest,
          new ActionListener<DeleteResponse>() {
            @Override
            public void onResponse(DeleteResponse response) {
              UpdateResponse update =
                  new UpdateResponse(
                      response.getIndex(),
                      response.getType(),
                      response.getId(),
                      response.getVersion());
              update.setGetResult(
                  extractGetResult(
                      request,
                      response.getVersion(),
                      updatedSourceAsMap,
                      updateSourceContentType,
                      null));
              listener.onResponse(update);
            }

            @Override
            public void onFailure(Throwable e) {
              e = ExceptionsHelper.unwrapCause(e);
              if (e instanceof VersionConflictEngineException) {
                if (retryCount < request.retryOnConflict()) {
                  threadPool
                      .executor(executor())
                      .execute(
                          new Runnable() {
                            @Override
                            public void run() {
                              shardOperation(request, listener, retryCount + 1);
                            }
                          });
                  return;
                }
              }
              listener.onFailure(e);
            }
          });
    } else if ("none".equals(operation)) {
      UpdateResponse update =
          new UpdateResponse(
              getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion());
      update.setGetResult(
          extractGetResult(
              request, getResult.getVersion(), updatedSourceAsMap, updateSourceContentType, null));
      listener.onResponse(update);
    } else {
      logger.warn(
          "Used update operation [{}] for script [{}], doing nothing...",
          operation,
          request.script);
      listener.onResponse(
          new UpdateResponse(
              getResult.getIndex(),
              getResult.getType(),
              getResult.getId(),
              getResult.getVersion()));
    }
  }
  @SuppressWarnings({"ConstantConditions", "unchecked"})
  @Test
  @BadApple
  public void testIndexShardLifecycleLeak() throws Exception {

    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(
            ImmutableSettings.builder()
                .put("index.number_of_shards", 1)
                .put("index.number_of_replicas", 0))
        .execute()
        .actionGet();

    client().admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();

    IndicesService indicesService = cluster().getInstance(IndicesService.class);
    IndexService indexService = indicesService.indexServiceSafe("test");
    Injector indexInjector = indexService.injector();
    IndexShard shard = indexService.shardSafe(0);
    Injector shardInjector = indexService.shardInjector(0);

    performCommonOperations();

    List<WeakReference> indexReferences = new ArrayList<WeakReference>();
    List<WeakReference> shardReferences = new ArrayList<WeakReference>();

    // TODO if we could iterate over the already created classes on the injector, we can just add
    // them here to the list
    // for now, we simple add some classes that make sense

    // add index references
    indexReferences.add(new WeakReference(indexService));
    indexReferences.add(new WeakReference(indexInjector));
    indexReferences.add(new WeakReference(indexService.mapperService()));
    for (DocumentMapper documentMapper : indexService.mapperService()) {
      indexReferences.add(new WeakReference(documentMapper));
    }
    indexReferences.add(new WeakReference(indexService.aliasesService()));
    indexReferences.add(new WeakReference(indexService.analysisService()));
    indexReferences.add(new WeakReference(indexService.fieldData()));
    indexReferences.add(new WeakReference(indexService.queryParserService()));

    // add shard references
    shardReferences.add(new WeakReference(shard));
    shardReferences.add(new WeakReference(shardInjector));

    indexService = null;
    indexInjector = null;
    shard = null;
    shardInjector = null;

    client().admin().indices().prepareDelete().execute().actionGet();

    for (int i = 0; i < 100; i++) {
      System.gc();
      int indexNotCleared = 0;
      for (WeakReference indexReference : indexReferences) {
        if (indexReference.get() != null) {
          indexNotCleared++;
        }
      }
      int shardNotCleared = 0;
      for (WeakReference shardReference : shardReferences) {
        if (shardReference.get() != null) {
          shardNotCleared++;
        }
      }
      logger.info(
          "round {}, indices {}/{}, shards {}/{}",
          i,
          indexNotCleared,
          indexReferences.size(),
          shardNotCleared,
          shardReferences.size());
      if (indexNotCleared == 0 && shardNotCleared == 0) {
        break;
      }
    }

    // Thread.sleep(1000000);

    for (WeakReference indexReference : indexReferences) {
      assertThat(
          "dangling index reference: " + indexReference.get(), indexReference.get(), nullValue());
    }

    for (WeakReference shardReference : shardReferences) {
      assertThat(
          "dangling shard reference: " + shardReference.get(), shardReference.get(), nullValue());
    }
  }