Ejemplo n.º 1
0
  @Override
  public List<TaskSubmissionResult> submitTasks(final List<TaskDeploymentDescriptor> tasks)
      throws IOException {

    final List<TaskSubmissionResult> submissionResultList =
        new SerializableArrayList<TaskSubmissionResult>();
    final List<Task> tasksToStart = new ArrayList<Task>();

    // Make sure all tasks are fully registered before they are started
    for (final TaskDeploymentDescriptor tdd : tasks) {

      final JobID jobID = tdd.getJobID();
      final ExecutionVertexID vertexID = tdd.getVertexID();
      RuntimeEnvironment re;

      // retrieve the registered cache files from job configuration and create the local tmp file.
      Map<String, FutureTask<Path>> cpTasks = new HashMap<String, FutureTask<Path>>();
      for (Entry<String, DistributedCacheEntry> e :
          DistributedCache.readFileInfoFromConfig(tdd.getJobConfiguration())) {
        FutureTask<Path> cp = this.fileCache.createTmpFile(e.getKey(), e.getValue(), jobID);
        cpTasks.put(e.getKey(), cp);
      }

      try {
        re =
            new RuntimeEnvironment(
                tdd,
                this.memoryManager,
                this.ioManager,
                new TaskInputSplitProvider(jobID, vertexID, this.globalInputSplitProvider),
                this.accumulatorProtocolProxy,
                cpTasks);
      } catch (Throwable t) {
        final TaskSubmissionResult result =
            new TaskSubmissionResult(vertexID, AbstractTaskResult.ReturnCode.DEPLOYMENT_ERROR);
        result.setDescription(StringUtils.stringifyException(t));
        LOG.error(result.getDescription(), t);
        submissionResultList.add(result);
        continue;
      }

      final Configuration jobConfiguration = tdd.getJobConfiguration();

      // Register the task
      Task task;
      try {
        task = createAndRegisterTask(vertexID, jobConfiguration, re);
      } catch (InsufficientResourcesException e) {
        final TaskSubmissionResult result =
            new TaskSubmissionResult(
                vertexID, AbstractTaskResult.ReturnCode.INSUFFICIENT_RESOURCES);
        result.setDescription(e.getMessage());
        LOG.error(result.getDescription(), e);
        submissionResultList.add(result);
        continue;
      }

      if (task == null) {
        final TaskSubmissionResult result =
            new TaskSubmissionResult(vertexID, AbstractTaskResult.ReturnCode.TASK_NOT_FOUND);
        result.setDescription(
            "Task " + re.getTaskNameWithIndex() + " (" + vertexID + ") was already running");
        LOG.error(result.getDescription());
        submissionResultList.add(result);
        continue;
      }

      submissionResultList.add(
          new TaskSubmissionResult(vertexID, AbstractTaskResult.ReturnCode.SUCCESS));
      tasksToStart.add(task);
    }

    // Now start the tasks
    for (final Task task : tasksToStart) {
      task.startExecution();
    }

    return submissionResultList;
  }
Ejemplo n.º 2
0
  /**
   * Constructs a runtime environment from a task deployment description.
   *
   * @param tdd the task deployment description
   * @param memoryManager the task manager's memory manager component
   * @param ioManager the task manager's I/O manager component
   * @param inputSplitProvider the input split provider for this environment
   * @throws Exception thrown if an error occurs while instantiating the invokable class
   */
  @SuppressWarnings({"unchecked", "rawtypes"})
  public RuntimeEnvironment(
      final TaskDeploymentDescriptor tdd,
      final MemoryManager memoryManager,
      final IOManager ioManager,
      final InputSplitProvider inputSplitProvider,
      AccumulatorProtocol accumulatorProtocolProxy)
      throws Exception {

    this.jobID = tdd.getJobID();
    this.taskName = tdd.getTaskName();
    this.invokableClass = tdd.getInvokableClass();
    this.jobConfiguration = tdd.getJobConfiguration();
    this.taskConfiguration = tdd.getTaskConfiguration();
    this.indexInSubtaskGroup = tdd.getIndexInSubtaskGroup();
    this.currentNumberOfSubtasks = tdd.getCurrentNumberOfSubtasks();
    this.memoryManager = memoryManager;
    this.ioManager = ioManager;
    this.inputSplitProvider = inputSplitProvider;
    this.accumulatorProtocolProxy = accumulatorProtocolProxy;

    this.invokable = this.invokableClass.newInstance();
    this.invokable.setEnvironment(this);
    this.invokable.registerInputOutput();

    if (!this.unboundOutputGateIDs.isEmpty() && LOG.isErrorEnabled()) {
      LOG.error(
          "Inconsistency: " + this.unboundOutputGateIDs.size() + " unbound output gate IDs left");
    }

    if (!this.unboundInputGateIDs.isEmpty() && LOG.isErrorEnabled()) {
      LOG.error(
          "Inconsistency: " + this.unboundInputGateIDs.size() + " unbound output gate IDs left");
    }

    final int noogdd = tdd.getNumberOfOutputGateDescriptors();
    for (int i = 0; i < noogdd; ++i) {
      final GateDeploymentDescriptor gdd = tdd.getOutputGateDescriptor(i);
      final OutputGate og = this.outputGates.get(i);
      final ChannelType channelType = gdd.getChannelType();
      og.setChannelType(channelType);

      final int nocdd = gdd.getNumberOfChannelDescriptors();
      for (int j = 0; j < nocdd; ++j) {

        final ChannelDeploymentDescriptor cdd = gdd.getChannelDescriptor(j);
        switch (channelType) {
          case NETWORK:
            og.createNetworkOutputChannel(og, cdd.getOutputChannelID(), cdd.getInputChannelID());
            break;
          case INMEMORY:
            og.createInMemoryOutputChannel(og, cdd.getOutputChannelID(), cdd.getInputChannelID());
            break;
          default:
            throw new IllegalStateException("Unknown channel type");
        }
      }
    }

    final int noigdd = tdd.getNumberOfInputGateDescriptors();
    for (int i = 0; i < noigdd; ++i) {
      final GateDeploymentDescriptor gdd = tdd.getInputGateDescriptor(i);
      final InputGate ig = this.inputGates.get(i);
      final ChannelType channelType = gdd.getChannelType();
      ig.setChannelType(channelType);

      final int nicdd = gdd.getNumberOfChannelDescriptors();
      for (int j = 0; j < nicdd; ++j) {

        final ChannelDeploymentDescriptor cdd = gdd.getChannelDescriptor(j);
        switch (channelType) {
          case NETWORK:
            ig.createNetworkInputChannel(ig, cdd.getInputChannelID(), cdd.getOutputChannelID());
            break;
          case INMEMORY:
            ig.createInMemoryInputChannel(ig, cdd.getInputChannelID(), cdd.getOutputChannelID());
            break;
          default:
            throw new IllegalStateException("Unknown channel type");
        }
      }
    }
  }