private void assignSubscription(String indexerName) {
try {
String lock = indexerModel.lockIndexer(indexerName);
try {
// Read current situation of record and assure it is still actual
IndexerDefinition indexer = indexerModel.getFreshIndexer(indexerName);
if (needsSubscriptionIdAssigned(indexer)) {
// We assume we are the only process which creates subscriptions which begin with the
// prefix "Indexer:". This way we are sure there are no naming conflicts or conflicts
// due to concurrent operations (e.g. someone deleting this subscription right after we
// created it).
String subscriptionId = subscriptionId(indexer.getName());
sepModel.addSubscription(subscriptionId);
indexer =
new IndexerDefinitionBuilder()
.startFrom(indexer)
.subscriptionId(subscriptionId)
.build();
indexerModel.updateIndexerInternal(indexer);
log.info(
"Assigned subscription ID '" + subscriptionId + "' to indexer '" + indexerName + "'");
}
} finally {
indexerModel.unlockIndexer(lock);
}
} catch (Throwable t) {
log.error("Error trying to assign a subscription to index " + indexerName, t);
}
}
private void prepareDeleteIndex(String indexerName) {
// We do not have to take a lock on the indexer, since once in delete state the indexer cannot
// be modified anymore by ordinary users.
boolean canBeDeleted = false;
try {
// Read current situation of record and assure it is still actual
IndexerDefinition indexer = indexerModel.getFreshIndexer(indexerName);
if (indexer.getLifecycleState() == IndexerDefinition.LifecycleState.DELETE_REQUESTED) {
canBeDeleted = true;
String queueSubscriptionId = indexer.getSubscriptionId();
if (queueSubscriptionId != null) {
sepModel.removeSubscription(indexer.getSubscriptionId());
// We leave the subscription ID in the indexer definition FYI
}
if (indexer.getActiveBatchBuildInfo() != null) {
JobClient jobClient = getJobClient();
String jobId = indexer.getActiveBatchBuildInfo().getJobId();
RunningJob job = jobClient.getJob(jobId);
if (job != null) {
job.killJob();
log.info("Kill indexer build job for indexer " + indexerName + ", job ID = " + jobId);
}
// Just to be sure...
jobStatusWatcher.assureWatching(
indexer.getName(), indexer.getActiveBatchBuildInfo().getJobId());
canBeDeleted = false;
}
if (!canBeDeleted) {
indexer =
new IndexerDefinitionBuilder()
.startFrom(indexer)
.lifecycleState(IndexerDefinition.LifecycleState.DELETING)
.build();
indexerModel.updateIndexerInternal(indexer);
}
} else if (indexer.getLifecycleState() == IndexerDefinition.LifecycleState.DELETING) {
// Check if the build job is already finished, if so, allow delete
if (indexer.getActiveBatchBuildInfo() == null) {
canBeDeleted = true;
}
}
} catch (Throwable t) {
log.error("Error preparing deletion of indexer " + indexerName, t);
}
if (canBeDeleted) {
deleteIndexer(indexerName);
}
}
@Override
public void activateAsLeader() throws Exception {
log.info("Starting up as indexer master.");
// Start these processes, but it is not until we have registered our model listener
// that these will receive work.
eventWorker.start();
jobStatusWatcher.start();
Collection<IndexerDefinition> indexers = indexerModel.getIndexers(listener);
// Rather than performing any work that might to be done for the indexers here,
// we push out fake events. This way there's only one place where these actions
// need to be performed.
for (IndexerDefinition index : indexers) {
eventWorker.putEvent(new IndexerModelEvent(INDEXER_UPDATED, index.getName()));
}
log.info("Startup as indexer master successful.");
}
private void markJobComplete(
String indexerName, String jobId, boolean success, String jobState, Counters counters) {
try {
// Lock internal bypasses the index-in-delete-state check, which does not matter (and might
// cause
// failure) in our case.
String lock = indexerModel.lockIndexerInternal(indexerName, false);
try {
// Read current situation of record and assure it is still actual
IndexerDefinition indexer = indexerModel.getFreshIndexer(indexerName);
ActiveBatchBuildInfo activeJobInfo = indexer.getActiveBatchBuildInfo();
if (activeJobInfo == null) {
// This might happen if we got some older update event on the indexer right after we
// marked this job as finished.
log.error(
"Unexpected situation: indexer build job completed but indexer does not have an active"
+ " build job. Index: "
+ indexer.getName()
+ ", job: "
+ jobId
+ ". Ignoring this event.");
runningJobs.remove(indexerName);
return;
} else if (!activeJobInfo.getJobId().equals(jobId)) {
// I don't think this should ever occur: a new job will never start before we marked
// this one as finished, especially since we lock when creating/updating indexes.
log.error(
"Abnormal situation: indexer is associated with index build job "
+ activeJobInfo.getJobId()
+ " but expected job "
+ jobId
+ ". Will mark job as"
+ " done anyway.");
}
BatchBuildInfoBuilder jobInfoBuilder = new BatchBuildInfoBuilder();
jobInfoBuilder.jobState(jobState);
jobInfoBuilder.success(success);
jobInfoBuilder.jobId(jobId);
jobInfoBuilder.batchIndexConfiguration(activeJobInfo.getBatchIndexConfiguration());
if (activeJobInfo != null) {
jobInfoBuilder.submitTime(activeJobInfo.getSubmitTime());
jobInfoBuilder.trackingUrl(activeJobInfo.getTrackingUrl());
}
if (counters != null) {
jobInfoBuilder.counter(
getCounterKey(Task.Counter.MAP_INPUT_RECORDS),
counters.getCounter(Task.Counter.MAP_INPUT_RECORDS));
jobInfoBuilder.counter(
getCounterKey(JobInProgress.Counter.TOTAL_LAUNCHED_MAPS),
counters.getCounter(JobInProgress.Counter.TOTAL_LAUNCHED_MAPS));
jobInfoBuilder.counter(
getCounterKey(JobInProgress.Counter.NUM_FAILED_MAPS),
counters.getCounter(JobInProgress.Counter.NUM_FAILED_MAPS));
// TODO
//
// jobInfo.addCounter(getCounterKey(IndexBatchBuildCounters.NUM_FAILED_RECORDS),
//
// counters.getCounter(IndexBatchBuildCounters.NUM_FAILED_RECORDS));
}
indexer =
new IndexerDefinitionBuilder()
.lastBatchBuildInfo(jobInfoBuilder.build())
.activeBatchBuildInfo(null)
.batchIndexingState(BatchIndexingState.INACTIVE)
.build();
runningJobs.remove(indexerName);
indexerModel.updateIndexerInternal(indexer);
log.info(
"Marked indexer build job as finished for indexer "
+ indexerName
+ ", job ID = "
+ jobId);
} finally {
indexerModel.unlockIndexer(lock, true);
}
} catch (Throwable t) {
log.error("Error trying to mark index build job as finished for indexer " + indexerName, t);
}
}
@Override
public void run() {
long startedAt = System.currentTimeMillis();
while (!stop && !Thread.interrupted()) {
try {
IndexerModelEvent event = null;
while (!stop && event == null) {
event = eventQueue.poll(1000, TimeUnit.MILLISECONDS);
}
if (stop || event == null || Thread.interrupted()) {
return;
}
// Warn if the queue is getting large, but do not do this just after we started, because
// on initial startup a fake update event is added for every defined index, which would
// lead
// to this message always being printed on startup when more than 10 indexes are defined.
int queueSize = eventQueue.size();
if (queueSize >= 10 && (System.currentTimeMillis() - startedAt > 5000)) {
log.warn("EventWorker queue getting large, size = " + queueSize);
}
if (event.getType() == INDEXER_ADDED || event.getType() == INDEXER_UPDATED) {
IndexerDefinition indexer = null;
try {
indexer = indexerModel.getIndexer(event.getIndexerName());
} catch (IndexerNotFoundException e) {
// ignore, indexer has meanwhile been deleted, we will get another event for this
}
if (indexer != null) {
if (indexer.getLifecycleState() == IndexerDefinition.LifecycleState.DELETE_REQUESTED
|| indexer.getLifecycleState() == IndexerDefinition.LifecycleState.DELETING) {
prepareDeleteIndex(indexer.getName());
// in case of delete, we do not need to handle any other cases
} else {
if (needsSubscriptionIdAssigned(indexer)) {
assignSubscription(indexer.getName());
}
if (needsSubscriptionIdUnassigned(indexer)) {
unassignSubscription(indexer.getName());
}
if (needsBatchBuildStart(indexer)) {
startFullIndexBuild(indexer.getName());
}
if (indexer.getActiveBatchBuildInfo() != null) {
jobStatusWatcher.assureWatching(
indexer.getName(), indexer.getActiveBatchBuildInfo().getJobId());
}
}
}
}
eventCount.incrementAndGet();
} catch (InterruptedException e) {
return;
} catch (Throwable t) {
log.error("Error processing indexer model event in IndexerMaster.", t);
}
}
}