/**
* Unregisters a finished or aborted task.
*
* @param id the ID of the task to be unregistered
*/
private void unregisterTask(final ExecutionVertexID id) {
// Task de-registration must be atomic
synchronized (this) {
final Task task = this.runningTasks.remove(id);
if (task == null) {
LOG.error("Cannot find task with ID " + id + " to unregister");
return;
}
// remove the local tmp file for unregistered tasks.
for (Entry<String, DistributedCacheEntry> e :
DistributedCache.readFileInfoFromConfig(task.getEnvironment().getJobConfiguration())) {
this.fileCache.deleteTmpFile(e.getKey(), e.getValue(), task.getJobID());
}
// Unregister task from the byte buffered channel manager
this.channelManager.unregister(id, task);
// Unregister task from profiling
task.unregisterProfiler(this.profiler);
// Unregister task from memory manager
task.unregisterMemoryManager(this.memoryManager);
// Unregister task from library cache manager
try {
LibraryCacheManager.unregister(task.getJobID());
} catch (IOException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Unregistering the job vertex ID " + id + " caused an IOException");
}
}
}
}
private static Task createTask() {
LibraryCacheManager libCache = mock(LibraryCacheManager.class);
when(libCache.getClassLoader(any(JobID.class))).thenReturn(ClassLoader.getSystemClassLoader());
ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
ResultPartitionConsumableNotifier consumableNotifier =
mock(ResultPartitionConsumableNotifier.class);
NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
when(networkEnvironment.getPartitionManager()).thenReturn(partitionManager);
when(networkEnvironment.getPartitionConsumableNotifier()).thenReturn(consumableNotifier);
when(networkEnvironment.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
TaskDeploymentDescriptor tdd =
new TaskDeploymentDescriptor(
new JobID(),
new JobVertexID(),
new ExecutionAttemptID(),
"Test Task",
0,
1,
new Configuration(),
new Configuration(),
CheckpointsInOrderInvokable.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
Collections.<InputGateDeploymentDescriptor>emptyList(),
Collections.<BlobKey>emptyList(),
0);
ActorGateway taskManagerGateway = DummyActorGateway.INSTANCE;
return new Task(
tdd,
mock(MemoryManager.class),
mock(IOManager.class),
networkEnvironment,
mock(BroadcastVariableManager.class),
taskManagerGateway,
DummyActorGateway.INSTANCE,
new FiniteDuration(60, TimeUnit.SECONDS),
libCache,
mock(FileCache.class),
new TaskManagerRuntimeInfo("localhost", new Configuration()));
}
@Override
public LibraryCacheProfileResponse getLibraryCacheProfile(LibraryCacheProfileRequest request)
throws IOException {
LibraryCacheProfileResponse response = new LibraryCacheProfileResponse(request);
String[] requiredLibraries = request.getRequiredLibraries();
for (int i = 0; i < requiredLibraries.length; i++) {
if (LibraryCacheManager.contains(requiredLibraries[i]) == null) {
response.setCached(i, false);
} else {
response.setCached(i, true);
}
}
return response;
}
@Override
public void invoke() throws Exception {
userCodeClassLoader = LibraryCacheManager.getClassLoader(getEnvironment().getJobID());
TaskConfig taskConfig = new TaskConfig(getTaskConfiguration());
// store all aggregators
this.aggregators = new HashMap<String, Aggregator<?>>();
for (AggregatorWithName<?> aggWithName : taskConfig.getIterationAggregators()) {
aggregators.put(aggWithName.getName(), aggWithName.getAggregator());
}
// store the aggregator convergence criterion
if (taskConfig.usesConvergenceCriterion()) {
convergenceCriterion = taskConfig.getConvergenceCriterion();
convergenceAggregatorName = taskConfig.getConvergenceCriterionAggregatorName();
Preconditions.checkNotNull(convergenceAggregatorName);
}
maxNumberOfIterations = taskConfig.getNumberOfIterations();
// set up the event handler
int numEventsTillEndOfSuperstep = taskConfig.getNumberOfEventsUntilInterruptInIterativeGate(0);
eventHandler =
new SyncEventHandler(numEventsTillEndOfSuperstep, aggregators, userCodeClassLoader);
headEventReader.subscribeToEvent(eventHandler, WorkerDoneEvent.class);
IntegerRecord dummy = new IntegerRecord();
while (!terminationRequested()) {
// notifyMonitor(IterationMonitoring.Event.SYNC_STARTING, currentIteration);
if (log.isInfoEnabled()) {
log.info(formatLogString("starting iteration [" + currentIteration + "]"));
}
// this call listens for events until the end-of-superstep is reached
readHeadEventChannel(dummy);
if (log.isInfoEnabled()) {
log.info(formatLogString("finishing iteration [" + currentIteration + "]"));
}
if (checkForConvergence()) {
if (log.isInfoEnabled()) {
log.info(
formatLogString(
"signaling that all workers are to terminate in iteration ["
+ currentIteration
+ "]"));
}
requestTermination();
sendToAllWorkers(new TerminationEvent());
// notifyMonitor(IterationMonitoring.Event.SYNC_FINISHED, currentIteration);
} else {
if (log.isInfoEnabled()) {
log.info(
formatLogString(
"signaling that all workers are done in iteration [" + currentIteration + "]"));
}
AllWorkersDoneEvent allWorkersDoneEvent = new AllWorkersDoneEvent(aggregators);
sendToAllWorkers(allWorkersDoneEvent);
// reset all aggregators
for (Aggregator<?> agg : aggregators.values()) {
agg.reset();
}
// notifyMonitor(IterationMonitoring.Event.SYNC_FINISHED, currentIteration);
currentIteration++;
}
}
}
