private void handleFailure(
final SettableFuture<Response> future,
final Supplier<ListenableFuture<Response>> code,
final long deadline,
final long delay,
final TimeUnit timeUnit,
final Throwable t) {
if (clock.now().getMillis() < deadline) {
if (delay > 0) {
executorService.schedule(
new Runnable() {
@Override
public void run() {
startRetry(future, code, deadline - 1, delay, timeUnit);
}
},
delay,
timeUnit);
} else {
startRetry(future, code, deadline - 1, delay, timeUnit);
}
} else {
future.setException(t);
}
}
public void testNoOpScheduledExecutorShutdown() {
ListeningScheduledExecutorService executor = TestingExecutors.noOpScheduledExecutor();
assertFalse(executor.isShutdown());
assertFalse(executor.isTerminated());
executor.shutdown();
assertTrue(executor.isShutdown());
assertTrue(executor.isTerminated());
}
public void testNoOpScheduledExecutorInvokeAll() throws ExecutionException, InterruptedException {
ListeningScheduledExecutorService executor = TestingExecutors.noOpScheduledExecutor();
taskDone = false;
Callable<Boolean> task =
new Callable<Boolean>() {
@Override
public Boolean call() {
taskDone = true;
return taskDone;
}
};
List<Future<Boolean>> futureList =
executor.invokeAll(ImmutableList.of(task), 10, TimeUnit.MILLISECONDS);
Future<Boolean> future = futureList.get(0);
assertFalse(taskDone);
assertTrue(future.isDone());
try {
future.get();
fail();
} catch (CancellationException e) {
// pass
}
}
public NamespaceExtractionCacheManager(
Lifecycle lifecycle,
final ServiceEmitter serviceEmitter,
final Map<Class<? extends ExtractionNamespace>, ExtractionNamespaceCacheFactory<?>>
namespaceFunctionFactoryMap) {
this.listeningScheduledExecutorService =
MoreExecutors.listeningDecorator(
Executors.newScheduledThreadPool(
1,
new ThreadFactoryBuilder()
.setDaemon(true)
.setNameFormat("NamespaceExtractionCacheManager-%d")
.setPriority(Thread.MIN_PRIORITY)
.build()));
ExecutorServices.manageLifecycle(lifecycle, listeningScheduledExecutorService);
this.serviceEmitter = serviceEmitter;
this.namespaceFunctionFactoryMap = namespaceFunctionFactoryMap;
listeningScheduledExecutorService.scheduleAtFixedRate(
new Runnable() {
long priorTasksStarted = 0L;
@Override
public void run() {
try {
final long tasks = tasksStarted.get();
serviceEmitter.emit(
ServiceMetricEvent.builder()
.build("namespace/deltaTasksStarted", tasks - priorTasksStarted));
priorTasksStarted = tasks;
monitor(serviceEmitter);
} catch (Exception e) {
log.error(e, "Error emitting namespace stats");
if (Thread.currentThread().isInterrupted()) {
throw Throwables.propagate(e);
}
}
}
},
1,
10,
TimeUnit.MINUTES);
}
// For testing purposes this is protected
protected <T extends ExtractionNamespace> ListenableFuture<?> schedule(
final String id,
final T namespace,
final ExtractionNamespaceCacheFactory<T> factory,
final Runnable postRunnable,
final String cacheId) {
log.debug("Trying to update namespace [%s]", id);
final NamespaceImplData implDatum = implData.get(id);
if (implDatum != null) {
synchronized (implDatum.enabled) {
if (implDatum.enabled.get()) {
// We also check at the end of the function, but fail fast here
throw new IAE(
"Namespace [%s] already exists! Leaving prior running", namespace.toString());
}
}
}
final long updateMs = namespace.getPollMs();
final CountDownLatch startLatch = new CountDownLatch(1);
final Runnable command =
new Runnable() {
@Override
public void run() {
try {
startLatch.await(); // wait for "election" to leadership or cancellation
if (!Thread.currentThread().isInterrupted()) {
final Map<String, String> cache = getCacheMap(cacheId);
final String preVersion = lastVersion.get(id);
final Callable<String> runnable =
factory.getCachePopulator(id, namespace, preVersion, cache);
tasksStarted.incrementAndGet();
final String newVersion = runnable.call();
if (preVersion != null && preVersion.equals(newVersion)) {
throw new CancellationException(
String.format("Version `%s` already exists", preVersion));
}
if (newVersion != null) {
lastVersion.put(id, newVersion);
}
postRunnable.run();
log.debug("Namespace [%s] successfully updated", id);
}
} catch (Throwable t) {
delete(cacheId);
if (t instanceof CancellationException) {
log.debug(t, "Namespace [%s] cancelled", id);
} else {
log.error(t, "Failed update namespace [%s]", namespace);
}
if (Thread.currentThread().isInterrupted()) {
throw Throwables.propagate(t);
}
}
}
};
ListenableFuture<?> future;
try {
if (updateMs > 0) {
future =
listeningScheduledExecutorService.scheduleAtFixedRate(
command, 0, updateMs, TimeUnit.MILLISECONDS);
} else {
future = listeningScheduledExecutorService.schedule(command, 0, TimeUnit.MILLISECONDS);
}
final NamespaceImplData me = new NamespaceImplData(future, namespace, id);
final NamespaceImplData other = implData.putIfAbsent(id, me);
if (other != null) {
if (!future.isDone() && !future.cancel(true)) {
log.warn("Unable to cancel future for namespace[%s] on race loss", id);
}
throw new IAE("Namespace [%s] already exists! Leaving prior running", namespace);
} else {
if (!me.enabled.compareAndSet(false, true)) {
log.wtf("How did someone enable this before ME?");
}
log.debug("I own namespace [%s]", id);
return future;
}
} finally {
startLatch.countDown();
}
}
protected boolean waitForServiceToEnd(long time, TimeUnit unit) throws InterruptedException {
return listeningScheduledExecutorService.awaitTermination(time, unit);
}
/*
* (non-Javadoc)
*
* @see com.google.common.util.concurrent.AbstractScheduledService#shutDown()
*/
@Override
protected void shutDown() throws Exception {
service.shutdown();
super.shutDown();
}
/**
* Use the provided BulkJobFactory to build and submit BulkJob items as ListenableFuture objects
*/
@ExceptionMetered(name = "BulkJobScheduledService_submitWork_exceptions", group = "scheduler")
private void submitWork(final JobDescriptor jobDescriptor) {
List<Job> jobs;
try {
jobs = jobFactory.jobsFrom(jobDescriptor);
} catch (JobNotFoundException e) {
LOG.error("Could not create jobs", e);
return;
}
for (final Job job : jobs) {
// job execution needs to be external to both the callback and the task.
// This way regardless of any error we can
// mark a job as failed if required
final JobExecution execution = new JobExecutionImpl(jobDescriptor);
// We don't care if this is atomic (not worth using a lock object)
// we just need to prevent NPEs from ever occurring
final JobListener currentListener = this.jobListener;
ListenableFuture<Void> future =
service.submit(
new Callable<Void>() {
@Override
public Void call() throws Exception {
capacitySemaphore.acquire();
execution.start(maxFailCount);
jobAccessor.save(execution);
// this job is dead, treat it as such
if (execution.getStatus() == Status.DEAD) {
return null;
}
// TODO wrap and throw specifically typed exception for onFailure,
// needs jobId
job.execute(execution);
if (currentListener != null) {
currentListener.onSubmit(execution);
}
return null;
}
});
Futures.addCallback(
future,
new FutureCallback<Void>() {
@Override
public void onSuccess(Void param) {
if (execution.getStatus() == Status.IN_PROGRESS) {
LOG.info("Successful completion of bulkJob {}", execution);
execution.completed();
}
jobAccessor.save(execution);
capacitySemaphore.release();
if (currentListener != null) {
currentListener.onSuccess(execution);
}
}
@Override
public void onFailure(Throwable throwable) {
LOG.error("Failed execution for bulkJob", throwable);
// mark it as failed
if (execution.getStatus() == Status.IN_PROGRESS) {
execution.failed();
}
jobAccessor.save(execution);
capacitySemaphore.release();
if (currentListener != null) {
currentListener.onFailure(execution);
}
}
});
}
}