@Override
public List<TaskSubmissionResult> submitTasks(final List<TaskDeploymentDescriptor> tasks)
throws IOException {
final List<TaskSubmissionResult> submissionResultList =
new SerializableArrayList<TaskSubmissionResult>();
final List<Task> tasksToStart = new ArrayList<Task>();
// Make sure all tasks are fully registered before they are started
for (final TaskDeploymentDescriptor tdd : tasks) {
final JobID jobID = tdd.getJobID();
final ExecutionVertexID vertexID = tdd.getVertexID();
RuntimeEnvironment re;
// retrieve the registered cache files from job configuration and create the local tmp file.
Map<String, FutureTask<Path>> cpTasks = new HashMap<String, FutureTask<Path>>();
for (Entry<String, DistributedCacheEntry> e :
DistributedCache.readFileInfoFromConfig(tdd.getJobConfiguration())) {
FutureTask<Path> cp = this.fileCache.createTmpFile(e.getKey(), e.getValue(), jobID);
cpTasks.put(e.getKey(), cp);
}
try {
re =
new RuntimeEnvironment(
tdd,
this.memoryManager,
this.ioManager,
new TaskInputSplitProvider(jobID, vertexID, this.globalInputSplitProvider),
this.accumulatorProtocolProxy,
cpTasks);
} catch (Throwable t) {
final TaskSubmissionResult result =
new TaskSubmissionResult(vertexID, AbstractTaskResult.ReturnCode.DEPLOYMENT_ERROR);
result.setDescription(StringUtils.stringifyException(t));
LOG.error(result.getDescription(), t);
submissionResultList.add(result);
continue;
}
final Configuration jobConfiguration = tdd.getJobConfiguration();
// Register the task
Task task;
try {
task = createAndRegisterTask(vertexID, jobConfiguration, re);
} catch (InsufficientResourcesException e) {
final TaskSubmissionResult result =
new TaskSubmissionResult(
vertexID, AbstractTaskResult.ReturnCode.INSUFFICIENT_RESOURCES);
result.setDescription(e.getMessage());
LOG.error(result.getDescription(), e);
submissionResultList.add(result);
continue;
}
if (task == null) {
final TaskSubmissionResult result =
new TaskSubmissionResult(vertexID, AbstractTaskResult.ReturnCode.TASK_NOT_FOUND);
result.setDescription(
"Task " + re.getTaskNameWithIndex() + " (" + vertexID + ") was already running");
LOG.error(result.getDescription());
submissionResultList.add(result);
continue;
}
submissionResultList.add(
new TaskSubmissionResult(vertexID, AbstractTaskResult.ReturnCode.SUCCESS));
tasksToStart.add(task);
}
// Now start the tasks
for (final Task task : tasksToStart) {
task.startExecution();
}
return submissionResultList;
}
/**
* Constructs a runtime environment from a task deployment description.
*
* @param tdd the task deployment description
* @param memoryManager the task manager's memory manager component
* @param ioManager the task manager's I/O manager component
* @param inputSplitProvider the input split provider for this environment
* @throws Exception thrown if an error occurs while instantiating the invokable class
*/
@SuppressWarnings({"unchecked", "rawtypes"})
public RuntimeEnvironment(
final TaskDeploymentDescriptor tdd,
final MemoryManager memoryManager,
final IOManager ioManager,
final InputSplitProvider inputSplitProvider,
AccumulatorProtocol accumulatorProtocolProxy)
throws Exception {
this.jobID = tdd.getJobID();
this.taskName = tdd.getTaskName();
this.invokableClass = tdd.getInvokableClass();
this.jobConfiguration = tdd.getJobConfiguration();
this.taskConfiguration = tdd.getTaskConfiguration();
this.indexInSubtaskGroup = tdd.getIndexInSubtaskGroup();
this.currentNumberOfSubtasks = tdd.getCurrentNumberOfSubtasks();
this.memoryManager = memoryManager;
this.ioManager = ioManager;
this.inputSplitProvider = inputSplitProvider;
this.accumulatorProtocolProxy = accumulatorProtocolProxy;
this.invokable = this.invokableClass.newInstance();
this.invokable.setEnvironment(this);
this.invokable.registerInputOutput();
if (!this.unboundOutputGateIDs.isEmpty() && LOG.isErrorEnabled()) {
LOG.error(
"Inconsistency: " + this.unboundOutputGateIDs.size() + " unbound output gate IDs left");
}
if (!this.unboundInputGateIDs.isEmpty() && LOG.isErrorEnabled()) {
LOG.error(
"Inconsistency: " + this.unboundInputGateIDs.size() + " unbound output gate IDs left");
}
final int noogdd = tdd.getNumberOfOutputGateDescriptors();
for (int i = 0; i < noogdd; ++i) {
final GateDeploymentDescriptor gdd = tdd.getOutputGateDescriptor(i);
final OutputGate og = this.outputGates.get(i);
final ChannelType channelType = gdd.getChannelType();
og.setChannelType(channelType);
final int nocdd = gdd.getNumberOfChannelDescriptors();
for (int j = 0; j < nocdd; ++j) {
final ChannelDeploymentDescriptor cdd = gdd.getChannelDescriptor(j);
switch (channelType) {
case NETWORK:
og.createNetworkOutputChannel(og, cdd.getOutputChannelID(), cdd.getInputChannelID());
break;
case INMEMORY:
og.createInMemoryOutputChannel(og, cdd.getOutputChannelID(), cdd.getInputChannelID());
break;
default:
throw new IllegalStateException("Unknown channel type");
}
}
}
final int noigdd = tdd.getNumberOfInputGateDescriptors();
for (int i = 0; i < noigdd; ++i) {
final GateDeploymentDescriptor gdd = tdd.getInputGateDescriptor(i);
final InputGate ig = this.inputGates.get(i);
final ChannelType channelType = gdd.getChannelType();
ig.setChannelType(channelType);
final int nicdd = gdd.getNumberOfChannelDescriptors();
for (int j = 0; j < nicdd; ++j) {
final ChannelDeploymentDescriptor cdd = gdd.getChannelDescriptor(j);
switch (channelType) {
case NETWORK:
ig.createNetworkInputChannel(ig, cdd.getInputChannelID(), cdd.getOutputChannelID());
break;
case INMEMORY:
ig.createInMemoryInputChannel(ig, cdd.getInputChannelID(), cdd.getOutputChannelID());
break;
default:
throw new IllegalStateException("Unknown channel type");
}
}
}
}