/** Shutdown the scheduler. All subsequent execution request will be rejected. */
public void shutdown() throws InterruptedException {
checkState(!_isShutdown);
_isShutdown = true;
/* Drain jobs from the queue so they will never be started. Has to be done
* before killing jobs as otherwise the queued jobs will immediatley fill
* the freed job slot.
*/
Collection<PrioritizedRequest> toBeCancelled = new ArrayList<>();
_queue.drainTo(toBeCancelled);
/* Kill both the jobs that were queued and which are running. */
_jobs.values().forEach(j -> j.kill("shutdown"));
/* Jobs that were queued were never submitted for execution and thus we
* manually trigger postprocessing.
*/
toBeCancelled.forEach(this::postprocessWithoutJobSlot);
LOGGER.info("Waiting for movers on queue '{}' to finish", _name);
if (!_semaphore.tryAcquire(_semaphore.getMaxPermits(), 2, TimeUnit.SECONDS)) {
// This is often due to a mover not reacting to interrupt or the transfer
// doing a lengthy checksum calculation during post processing.
String versions =
_jobs
.values()
.stream()
.map(PrioritizedRequest::getMover)
.map(Mover::getProtocolInfo)
.map(ProtocolInfo::getVersionString)
.collect(joining(","));
LOGGER.warn("Failed to terminate some movers prior to shutdown: {}", versions);
}
}
/**
* Set maximal number of concurrently running jobs by this scheduler. All pending jobs will be
* executed.
*
* @param maxJobs
*/
public void setMaxActiveJobs(int maxJobs) {
_semaphore.setMaxPermits(maxJobs);
PrioritizedRequest request;
while (_semaphore.tryAcquire() && (request = nextOrRelease()) != null) {
sendToExecution(request);
}
}
public MoverRequestScheduler(String name, int queueId, Order order) {
_name = name;
_queueId = queueId;
_order = order;
_queue = createQueue(order);
_semaphore.setMaxPermits(2);
}
/**
* Returns the next job or releases a job slot. If a non-null value is returned, the caller must
* submit the job to execution. Should only be caller by a caller than currently holds a job slot.
*
* @return
*/
private synchronized PrioritizedRequest nextOrRelease() {
PrioritizedRequest request = _queue.poll();
if (request == null) {
_semaphore.release();
}
return request;
}
/**
* Add a request to the scheduler.
*
* <p>Returns true if the caller acquired a job slot and must send the job to execution.
*
* @param request
* @return
*/
private synchronized boolean submit(PrioritizedRequest request) {
if (_jobs.put(request.getId(), request) != null) {
throw new RuntimeException(
"Duplicate mover id detected. Please report to [email protected].");
}
if (_semaphore.tryAcquire()) {
return true;
} else {
_queue.add(request);
return false;
}
}
/**
* Get mover id for given door request. If there is no mover associated with {@code
* doorUniqueueRequest} a new mover will be created by using provided {@code moverSupplier}.
*
* <p>The returned mover id generated with following encoding: | 31- queue id -24|23- job id -0|
*
* @param moverSupplier {@link MoverSupplier} which can create a mover for given requests.
* @param doorUniqueId unique request identifier generated by the door.
* @param priority
* @return mover id
*/
public int getOrCreateMover(MoverSupplier moverSupplier, String doorUniqueId, IoPriority priority)
throws CacheException {
checkState(!_isShutdown);
try {
/* Create the request if it doesn't already exists.
*/
PrioritizedRequest request =
_moverByRequests.computeIfAbsent(
doorUniqueId,
key -> {
try {
return createRequest(moverSupplier, key, priority);
} catch (CacheException e) {
throw new RuntimeException(e);
}
});
/* If not already queued, submit it.
*/
if (request.queue()) {
if (submit(request)) {
/* There was a free slot in the queue so we submit directly to execution.
*/
sendToExecution(request);
} else if (_semaphore.getMaxPermits() <= 0) {
LOGGER.warn(
"A task was added to queue '{}', however the queue is not "
+ "configured to execute any tasks.",
_name);
}
}
return request.getId();
} catch (RuntimeException e) {
Throwables.propagateIfInstanceOf(e.getCause(), CacheException.class);
throw e;
}
}
/**
* Get the maximal number allowed of concurrently running jobs by this scheduler.
*
* @return maximal number of jobs.
*/
public int getMaxActiveJobs() {
return _semaphore.getMaxPermits();
}