private synchronized void scheduleSnapshotIfNeeded() {
   if (!shardGateway.requiresSnapshot()) {
     return;
   }
   if (!shardGateway.requiresSnapshotScheduling()) {
     return;
   }
   if (!indexShard.routingEntry().primary()) {
     // we only do snapshotting on the primary shard
     return;
   }
   if (!indexShard.routingEntry().started()) {
     // we only schedule when the cluster assumes we have started
     return;
   }
   if (snapshotScheduleFuture != null) {
     // we are already scheduling this one, ignore
     return;
   }
   if (snapshotInterval.millis() != -1) {
     // we need to schedule snapshot
     if (logger.isDebugEnabled()) {
       logger.debug("scheduling snapshot every [{}]", snapshotInterval);
     }
     snapshotScheduleFuture =
         threadPool.schedule(snapshotInterval, ThreadPool.Names.SNAPSHOT, snapshotRunnable);
   }
 }
  @Override
  protected ShardStatus shardOperation(IndexShardStatusRequest request)
      throws ElasticSearchException {
    InternalIndexService indexService =
        (InternalIndexService) indicesService.indexServiceSafe(request.index());
    InternalIndexShard indexShard = (InternalIndexShard) indexService.shardSafe(request.shardId());
    ShardStatus shardStatus = new ShardStatus(indexShard.routingEntry());
    shardStatus.state = indexShard.state();
    try {
      shardStatus.storeSize = indexShard.store().estimateSize();
    } catch (IOException e) {
      // failure to get the store size...
    }
    if (indexShard.state() == IndexShardState.STARTED) {
      //            shardStatus.estimatedFlushableMemorySize =
      // indexShard.estimateFlushableMemorySize();
      shardStatus.translogId = indexShard.translog().currentId();
      shardStatus.translogOperations = indexShard.translog().estimatedNumberOfOperations();
      Engine.Searcher searcher = indexShard.searcher();
      try {
        shardStatus.docs = new DocsStatus();
        shardStatus.docs.numDocs = searcher.reader().numDocs();
        shardStatus.docs.maxDoc = searcher.reader().maxDoc();
        shardStatus.docs.deletedDocs = searcher.reader().numDeletedDocs();
      } finally {
        searcher.release();
      }

      shardStatus.mergeStats = indexShard.mergeScheduler().stats();
      shardStatus.refreshStats = indexShard.refreshStats();
      shardStatus.flushStats = indexShard.flushStats();
    }

    if (request.recovery) {
      // check on going recovery (from peer or gateway)
      RecoveryStatus peerRecoveryStatus = indexShard.peerRecoveryStatus();
      if (peerRecoveryStatus == null) {
        peerRecoveryStatus = peerRecoveryTarget.peerRecoveryStatus(indexShard.shardId());
      }
      if (peerRecoveryStatus != null) {
        PeerRecoveryStatus.Stage stage;
        switch (peerRecoveryStatus.stage()) {
          case INIT:
            stage = PeerRecoveryStatus.Stage.INIT;
            break;
          case INDEX:
            stage = PeerRecoveryStatus.Stage.INDEX;
            break;
          case TRANSLOG:
            stage = PeerRecoveryStatus.Stage.TRANSLOG;
            break;
          case FINALIZE:
            stage = PeerRecoveryStatus.Stage.FINALIZE;
            break;
          case DONE:
            stage = PeerRecoveryStatus.Stage.DONE;
            break;
          default:
            stage = PeerRecoveryStatus.Stage.INIT;
        }
        shardStatus.peerRecoveryStatus =
            new PeerRecoveryStatus(
                stage,
                peerRecoveryStatus.startTime(),
                peerRecoveryStatus.time(),
                peerRecoveryStatus.phase1TotalSize(),
                peerRecoveryStatus.phase1ExistingTotalSize(),
                peerRecoveryStatus.currentFilesSize(),
                peerRecoveryStatus.currentTranslogOperations());
      }

      IndexShardGatewayService gatewayService =
          indexService.shardInjector(request.shardId()).getInstance(IndexShardGatewayService.class);
      org.elasticsearch.index.gateway.RecoveryStatus gatewayRecoveryStatus =
          gatewayService.recoveryStatus();
      if (gatewayRecoveryStatus != null) {
        GatewayRecoveryStatus.Stage stage;
        switch (gatewayRecoveryStatus.stage()) {
          case INIT:
            stage = GatewayRecoveryStatus.Stage.INIT;
            break;
          case INDEX:
            stage = GatewayRecoveryStatus.Stage.INDEX;
            break;
          case TRANSLOG:
            stage = GatewayRecoveryStatus.Stage.TRANSLOG;
            break;
          case DONE:
            stage = GatewayRecoveryStatus.Stage.DONE;
            break;
          default:
            stage = GatewayRecoveryStatus.Stage.INIT;
        }
        shardStatus.gatewayRecoveryStatus =
            new GatewayRecoveryStatus(
                stage,
                gatewayRecoveryStatus.startTime(),
                gatewayRecoveryStatus.time(),
                gatewayRecoveryStatus.index().totalSize(),
                gatewayRecoveryStatus.index().reusedTotalSize(),
                gatewayRecoveryStatus.index().currentFilesSize(),
                gatewayRecoveryStatus.translog().currentTranslogOperations());
      }
    }

    if (request.snapshot) {
      IndexShardGatewayService gatewayService =
          indexService.shardInjector(request.shardId()).getInstance(IndexShardGatewayService.class);
      SnapshotStatus snapshotStatus = gatewayService.snapshotStatus();
      if (snapshotStatus != null) {
        GatewaySnapshotStatus.Stage stage;
        switch (snapshotStatus.stage()) {
          case DONE:
            stage = GatewaySnapshotStatus.Stage.DONE;
            break;
          case FAILURE:
            stage = GatewaySnapshotStatus.Stage.FAILURE;
            break;
          case TRANSLOG:
            stage = GatewaySnapshotStatus.Stage.TRANSLOG;
            break;
          case FINALIZE:
            stage = GatewaySnapshotStatus.Stage.FINALIZE;
            break;
          case INDEX:
            stage = GatewaySnapshotStatus.Stage.INDEX;
            break;
          default:
            stage = GatewaySnapshotStatus.Stage.NONE;
            break;
        }
        shardStatus.gatewaySnapshotStatus =
            new GatewaySnapshotStatus(
                stage,
                snapshotStatus.startTime(),
                snapshotStatus.time(),
                snapshotStatus.index().totalSize(),
                snapshotStatus.translog().expectedNumberOfOperations());
      }
    }

    return shardStatus;
  }
  /** Snapshots the given shard into the gateway. */
  public synchronized void snapshot(final String reason)
      throws IndexShardGatewaySnapshotFailedException {
    if (!indexShard.routingEntry().primary()) {
      return;
      //            throw new IndexShardGatewaySnapshotNotAllowedException(shardId, "Snapshot not
      // allowed on non primary shard");
    }
    if (indexShard.routingEntry().relocating()) {
      // do not snapshot when in the process of relocation of primaries so we won't get conflicts
      return;
    }
    if (indexShard.state() == IndexShardState.CREATED) {
      // shard has just been created, ignore it and return
      return;
    }
    if (indexShard.state() == IndexShardState.RECOVERING) {
      // shard is recovering, don't snapshot
      return;
    }

    if (snapshotLock == null) {
      try {
        snapshotLock = shardGateway.obtainSnapshotLock();
      } catch (Exception e) {
        logger.warn("failed to obtain snapshot lock, ignoring snapshot", e);
        return;
      }
    }

    try {
      SnapshotStatus snapshotStatus =
          indexShard.snapshot(
              new Engine.SnapshotHandler<SnapshotStatus>() {
                @Override
                public SnapshotStatus snapshot(
                    SnapshotIndexCommit snapshotIndexCommit, Translog.Snapshot translogSnapshot)
                    throws EngineException {
                  if (lastIndexVersion != snapshotIndexCommit.getGeneration()
                      || lastTranslogId != translogSnapshot.translogId()
                      || lastTranslogLength < translogSnapshot.length()) {

                    logger.debug("snapshot ({}) to {} ...", reason, shardGateway);
                    SnapshotStatus snapshotStatus =
                        shardGateway.snapshot(
                            new IndexShardGateway.Snapshot(
                                snapshotIndexCommit,
                                translogSnapshot,
                                lastIndexVersion,
                                lastTranslogId,
                                lastTranslogLength,
                                lastTotalTranslogOperations));

                    lastIndexVersion = snapshotIndexCommit.getGeneration();
                    lastTranslogId = translogSnapshot.translogId();
                    lastTranslogLength = translogSnapshot.length();
                    lastTotalTranslogOperations = translogSnapshot.estimatedTotalOperations();
                    return snapshotStatus;
                  }
                  return null;
                }
              });
      if (snapshotStatus != null) {
        if (logger.isDebugEnabled()) {
          StringBuilder sb = new StringBuilder();
          sb.append("snapshot (")
              .append(reason)
              .append(") completed to ")
              .append(shardGateway)
              .append(", took [")
              .append(TimeValue.timeValueMillis(snapshotStatus.time()))
              .append("]\n");
          sb.append("    index    : version [")
              .append(lastIndexVersion)
              .append("], number_of_files [")
              .append(snapshotStatus.index().numberOfFiles())
              .append("] with total_size [")
              .append(new ByteSizeValue(snapshotStatus.index().totalSize()))
              .append("], took [")
              .append(TimeValue.timeValueMillis(snapshotStatus.index().time()))
              .append("]\n");
          sb.append("    translog : id      [")
              .append(lastTranslogId)
              .append("], number_of_operations [")
              .append(snapshotStatus.translog().expectedNumberOfOperations())
              .append("], took [")
              .append(TimeValue.timeValueMillis(snapshotStatus.translog().time()))
              .append("]");
          logger.debug(sb.toString());
        }
      }
    } catch (SnapshotFailedEngineException e) {
      if (e.getCause() instanceof IllegalStateException) {
        // ignore, that's fine, snapshot has not started yet
      } else {
        throw new IndexShardGatewaySnapshotFailedException(shardId, "Failed to snapshot", e);
      }
    } catch (IllegalIndexShardStateException e) {
      // ignore, that's fine, snapshot has not started yet
    } catch (IndexShardGatewaySnapshotFailedException e) {
      throw e;
    } catch (Exception e) {
      throw new IndexShardGatewaySnapshotFailedException(shardId, "Failed to snapshot", e);
    }
  }
  /** Recovers the state of the shard from the gateway. */
  public void recover(final boolean indexShouldExists, final RecoveryListener listener)
      throws IndexShardGatewayRecoveryException, IgnoreGatewayRecoveryException {
    if (indexShard.state() == IndexShardState.CLOSED) {
      // got closed on us, just ignore this recovery
      listener.onIgnoreRecovery("shard closed");
      return;
    }
    if (!indexShard.routingEntry().primary()) {
      listener.onRecoveryFailed(
          new IndexShardGatewayRecoveryException(
              shardId, "Trying to recover when the shard is in backup state", null));
      return;
    }
    try {
      if (indexShard.routingEntry().restoreSource() != null) {
        indexShard.recovering("from snapshot");
      } else {
        indexShard.recovering("from gateway");
      }
    } catch (IllegalIndexShardStateException e) {
      // that's fine, since we might be called concurrently, just ignore this, we are already
      // recovering
      listener.onIgnoreRecovery("already in recovering process, " + e.getMessage());
      return;
    }

    threadPool
        .generic()
        .execute(
            new Runnable() {
              @Override
              public void run() {
                recoveryStatus = new RecoveryStatus();
                recoveryStatus.updateStage(RecoveryStatus.Stage.INIT);

                try {
                  if (indexShard.routingEntry().restoreSource() != null) {
                    logger.debug(
                        "restoring from {} ...", indexShard.routingEntry().restoreSource());
                    snapshotService.restore(recoveryStatus);
                  } else {
                    logger.debug("starting recovery from {} ...", shardGateway);
                    shardGateway.recover(indexShouldExists, recoveryStatus);
                  }

                  lastIndexVersion = recoveryStatus.index().version();
                  lastTranslogId = -1;
                  lastTranslogLength = 0;
                  lastTotalTranslogOperations =
                      recoveryStatus.translog().currentTranslogOperations();

                  // start the shard if the gateway has not started it already. Note that if the
                  // gateway
                  // moved shard to POST_RECOVERY, it may have been started as well if:
                  // 1) master sent a new cluster state indicating shard is initializing
                  // 2) IndicesClusterStateService#applyInitializingShard will send a shard started
                  // event
                  // 3) Master will mark shard as started and this will be processed locally.
                  IndexShardState shardState = indexShard.state();
                  if (shardState != IndexShardState.POST_RECOVERY
                      && shardState != IndexShardState.STARTED) {
                    indexShard.postRecovery("post recovery from gateway");
                  }
                  // refresh the shard
                  indexShard.refresh(new Engine.Refresh("post_gateway").force(true));

                  recoveryStatus.time(System.currentTimeMillis() - recoveryStatus.startTime());
                  recoveryStatus.updateStage(RecoveryStatus.Stage.DONE);

                  if (logger.isDebugEnabled()) {
                    logger.debug(
                        "recovery completed from [{}], took [{}]",
                        shardGateway,
                        timeValueMillis(recoveryStatus.time()));
                  } else if (logger.isTraceEnabled()) {
                    StringBuilder sb = new StringBuilder();
                    sb.append("recovery completed from ")
                        .append(shardGateway)
                        .append(", took [")
                        .append(timeValueMillis(recoveryStatus.time()))
                        .append("]\n");
                    sb.append("    index    : files           [")
                        .append(recoveryStatus.index().numberOfFiles())
                        .append("] with total_size [")
                        .append(new ByteSizeValue(recoveryStatus.index().totalSize()))
                        .append("], took[")
                        .append(TimeValue.timeValueMillis(recoveryStatus.index().time()))
                        .append("]\n");
                    sb.append("             : recovered_files [")
                        .append(recoveryStatus.index().numberOfRecoveredFiles())
                        .append("] with total_size [")
                        .append(new ByteSizeValue(recoveryStatus.index().recoveredTotalSize()))
                        .append("]\n");
                    sb.append("             : reusing_files   [")
                        .append(recoveryStatus.index().numberOfReusedFiles())
                        .append("] with total_size [")
                        .append(new ByteSizeValue(recoveryStatus.index().reusedTotalSize()))
                        .append("]\n");
                    sb.append("    start    : took [")
                        .append(TimeValue.timeValueMillis(recoveryStatus.start().time()))
                        .append("], check_index [")
                        .append(timeValueMillis(recoveryStatus.start().checkIndexTime()))
                        .append("]\n");
                    sb.append("    translog : number_of_operations [")
                        .append(recoveryStatus.translog().currentTranslogOperations())
                        .append("], took [")
                        .append(TimeValue.timeValueMillis(recoveryStatus.translog().time()))
                        .append("]");
                    logger.trace(sb.toString());
                  }
                  listener.onRecoveryDone();
                  scheduleSnapshotIfNeeded();
                } catch (IndexShardGatewayRecoveryException e) {
                  if (indexShard.state() == IndexShardState.CLOSED) {
                    // got closed on us, just ignore this recovery
                    listener.onIgnoreRecovery("shard closed");
                    return;
                  }
                  if ((e.getCause() instanceof IndexShardClosedException)
                      || (e.getCause() instanceof IndexShardNotStartedException)) {
                    // got closed on us, just ignore this recovery
                    listener.onIgnoreRecovery("shard closed");
                    return;
                  }
                  listener.onRecoveryFailed(e);
                } catch (IndexShardClosedException e) {
                  listener.onIgnoreRecovery("shard closed");
                } catch (IndexShardNotStartedException e) {
                  listener.onIgnoreRecovery("shard closed");
                } catch (Exception e) {
                  if (indexShard.state() == IndexShardState.CLOSED) {
                    // got closed on us, just ignore this recovery
                    listener.onIgnoreRecovery("shard closed");
                    return;
                  }
                  listener.onRecoveryFailed(
                      new IndexShardGatewayRecoveryException(shardId, "failed recovery", e));
                }
              }
            });
  }