private void assignSubscription(String indexerName) {
   try {
     String lock = indexerModel.lockIndexer(indexerName);
     try {
       // Read current situation of record and assure it is still actual
       IndexerDefinition indexer = indexerModel.getFreshIndexer(indexerName);
       if (needsSubscriptionIdAssigned(indexer)) {
         // We assume we are the only process which creates subscriptions which begin with the
         // prefix "Indexer:". This way we are sure there are no naming conflicts or conflicts
         // due to concurrent operations (e.g. someone deleting this subscription right after we
         // created it).
         String subscriptionId = subscriptionId(indexer.getName());
         sepModel.addSubscription(subscriptionId);
         indexer =
             new IndexerDefinitionBuilder()
                 .startFrom(indexer)
                 .subscriptionId(subscriptionId)
                 .build();
         indexerModel.updateIndexerInternal(indexer);
         log.info(
             "Assigned subscription ID '" + subscriptionId + "' to indexer '" + indexerName + "'");
       }
     } finally {
       indexerModel.unlockIndexer(lock);
     }
   } catch (Throwable t) {
     log.error("Error trying to assign a subscription to index " + indexerName, t);
   }
 }
  private void prepareDeleteIndex(String indexerName) {
    // We do not have to take a lock on the indexer, since once in delete state the indexer cannot
    // be modified anymore by ordinary users.
    boolean canBeDeleted = false;
    try {
      // Read current situation of record and assure it is still actual
      IndexerDefinition indexer = indexerModel.getFreshIndexer(indexerName);
      if (indexer.getLifecycleState() == IndexerDefinition.LifecycleState.DELETE_REQUESTED) {
        canBeDeleted = true;

        String queueSubscriptionId = indexer.getSubscriptionId();
        if (queueSubscriptionId != null) {
          sepModel.removeSubscription(indexer.getSubscriptionId());
          // We leave the subscription ID in the indexer definition FYI
        }

        if (indexer.getActiveBatchBuildInfo() != null) {
          JobClient jobClient = getJobClient();
          String jobId = indexer.getActiveBatchBuildInfo().getJobId();
          RunningJob job = jobClient.getJob(jobId);
          if (job != null) {
            job.killJob();
            log.info("Kill indexer build job for indexer " + indexerName + ", job ID =  " + jobId);
          }
          // Just to be sure...
          jobStatusWatcher.assureWatching(
              indexer.getName(), indexer.getActiveBatchBuildInfo().getJobId());
          canBeDeleted = false;
        }

        if (!canBeDeleted) {
          indexer =
              new IndexerDefinitionBuilder()
                  .startFrom(indexer)
                  .lifecycleState(IndexerDefinition.LifecycleState.DELETING)
                  .build();
          indexerModel.updateIndexerInternal(indexer);
        }
      } else if (indexer.getLifecycleState() == IndexerDefinition.LifecycleState.DELETING) {
        // Check if the build job is already finished, if so, allow delete
        if (indexer.getActiveBatchBuildInfo() == null) {
          canBeDeleted = true;
        }
      }
    } catch (Throwable t) {
      log.error("Error preparing deletion of indexer " + indexerName, t);
    }

    if (canBeDeleted) {
      deleteIndexer(indexerName);
    }
  }
    @Override
    public void activateAsLeader() throws Exception {
      log.info("Starting up as indexer master.");

      // Start these processes, but it is not until we have registered our model listener
      // that these will receive work.
      eventWorker.start();
      jobStatusWatcher.start();

      Collection<IndexerDefinition> indexers = indexerModel.getIndexers(listener);

      // Rather than performing any work that might to be done for the indexers here,
      // we push out fake events. This way there's only one place where these actions
      // need to be performed.
      for (IndexerDefinition index : indexers) {
        eventWorker.putEvent(new IndexerModelEvent(INDEXER_UPDATED, index.getName()));
      }

      log.info("Startup as indexer master successful.");
    }
    private void markJobComplete(
        String indexerName, String jobId, boolean success, String jobState, Counters counters) {
      try {
        // Lock internal bypasses the index-in-delete-state check, which does not matter (and might
        // cause
        // failure) in our case.
        String lock = indexerModel.lockIndexerInternal(indexerName, false);
        try {
          // Read current situation of record and assure it is still actual
          IndexerDefinition indexer = indexerModel.getFreshIndexer(indexerName);

          ActiveBatchBuildInfo activeJobInfo = indexer.getActiveBatchBuildInfo();

          if (activeJobInfo == null) {
            // This might happen if we got some older update event on the indexer right after we
            // marked this job as finished.
            log.error(
                "Unexpected situation: indexer build job completed but indexer does not have an active"
                    + " build job. Index: "
                    + indexer.getName()
                    + ", job: "
                    + jobId
                    + ". Ignoring this event.");
            runningJobs.remove(indexerName);
            return;
          } else if (!activeJobInfo.getJobId().equals(jobId)) {
            // I don't think this should ever occur: a new job will never start before we marked
            // this one as finished, especially since we lock when creating/updating indexes.
            log.error(
                "Abnormal situation: indexer is associated with index build job "
                    + activeJobInfo.getJobId()
                    + " but expected job "
                    + jobId
                    + ". Will mark job as"
                    + " done anyway.");
          }

          BatchBuildInfoBuilder jobInfoBuilder = new BatchBuildInfoBuilder();
          jobInfoBuilder.jobState(jobState);
          jobInfoBuilder.success(success);
          jobInfoBuilder.jobId(jobId);
          jobInfoBuilder.batchIndexConfiguration(activeJobInfo.getBatchIndexConfiguration());

          if (activeJobInfo != null) {
            jobInfoBuilder.submitTime(activeJobInfo.getSubmitTime());
            jobInfoBuilder.trackingUrl(activeJobInfo.getTrackingUrl());
          }

          if (counters != null) {
            jobInfoBuilder.counter(
                getCounterKey(Task.Counter.MAP_INPUT_RECORDS),
                counters.getCounter(Task.Counter.MAP_INPUT_RECORDS));
            jobInfoBuilder.counter(
                getCounterKey(JobInProgress.Counter.TOTAL_LAUNCHED_MAPS),
                counters.getCounter(JobInProgress.Counter.TOTAL_LAUNCHED_MAPS));
            jobInfoBuilder.counter(
                getCounterKey(JobInProgress.Counter.NUM_FAILED_MAPS),
                counters.getCounter(JobInProgress.Counter.NUM_FAILED_MAPS));
            // TODO
            //
            // jobInfo.addCounter(getCounterKey(IndexBatchBuildCounters.NUM_FAILED_RECORDS),
            //
            // counters.getCounter(IndexBatchBuildCounters.NUM_FAILED_RECORDS));
          }

          indexer =
              new IndexerDefinitionBuilder()
                  .lastBatchBuildInfo(jobInfoBuilder.build())
                  .activeBatchBuildInfo(null)
                  .batchIndexingState(BatchIndexingState.INACTIVE)
                  .build();

          runningJobs.remove(indexerName);
          indexerModel.updateIndexerInternal(indexer);

          log.info(
              "Marked indexer build job as finished for indexer "
                  + indexerName
                  + ", job ID =  "
                  + jobId);

        } finally {
          indexerModel.unlockIndexer(lock, true);
        }
      } catch (Throwable t) {
        log.error("Error trying to mark index build job as finished for indexer " + indexerName, t);
      }
    }
    @Override
    public void run() {
      long startedAt = System.currentTimeMillis();

      while (!stop && !Thread.interrupted()) {
        try {
          IndexerModelEvent event = null;
          while (!stop && event == null) {
            event = eventQueue.poll(1000, TimeUnit.MILLISECONDS);
          }

          if (stop || event == null || Thread.interrupted()) {
            return;
          }

          // Warn if the queue is getting large, but do not do this just after we started, because
          // on initial startup a fake update event is added for every defined index, which would
          // lead
          // to this message always being printed on startup when more than 10 indexes are defined.
          int queueSize = eventQueue.size();
          if (queueSize >= 10 && (System.currentTimeMillis() - startedAt > 5000)) {
            log.warn("EventWorker queue getting large, size = " + queueSize);
          }

          if (event.getType() == INDEXER_ADDED || event.getType() == INDEXER_UPDATED) {
            IndexerDefinition indexer = null;
            try {
              indexer = indexerModel.getIndexer(event.getIndexerName());
            } catch (IndexerNotFoundException e) {
              // ignore, indexer has meanwhile been deleted, we will get another event for this
            }

            if (indexer != null) {
              if (indexer.getLifecycleState() == IndexerDefinition.LifecycleState.DELETE_REQUESTED
                  || indexer.getLifecycleState() == IndexerDefinition.LifecycleState.DELETING) {
                prepareDeleteIndex(indexer.getName());

                // in case of delete, we do not need to handle any other cases
              } else {
                if (needsSubscriptionIdAssigned(indexer)) {
                  assignSubscription(indexer.getName());
                }

                if (needsSubscriptionIdUnassigned(indexer)) {
                  unassignSubscription(indexer.getName());
                }

                if (needsBatchBuildStart(indexer)) {
                  startFullIndexBuild(indexer.getName());
                }

                if (indexer.getActiveBatchBuildInfo() != null) {
                  jobStatusWatcher.assureWatching(
                      indexer.getName(), indexer.getActiveBatchBuildInfo().getJobId());
                }
              }
            }
          }
          eventCount.incrementAndGet();
        } catch (InterruptedException e) {
          return;
        } catch (Throwable t) {
          log.error("Error processing indexer model event in IndexerMaster.", t);
        }
      }
    }