Ejemplo n.º 1
0
  /**
   * Creates new shuffle job.
   *
   * @param jobId Job ID.
   * @return Created shuffle job.
   * @throws GridException If job creation failed.
   */
  private GridHadoopShuffleJob<UUID> newJob(GridHadoopJobId jobId) throws GridException {
    GridHadoopMapReducePlan plan = ctx.jobTracker().plan(jobId);

    GridHadoopShuffleJob<UUID> job =
        new GridHadoopShuffleJob<>(
            ctx.localNodeId(),
            ctx.localNodeId(),
            log,
            ctx.jobTracker().job(jobId, null),
            mem,
            plan.reducers(),
            plan.reducers(ctx.localNodeId()));

    UUID[] rdcAddrs = new UUID[plan.reducers()];

    for (int i = 0; i < rdcAddrs.length; i++) {
      UUID nodeId = plan.nodeForReducer(i);

      assert nodeId != null : "Plan is missing node for reducer [plan=" + plan + ", rdc=" + i + ']';

      rdcAddrs[i] = nodeId;
    }

    boolean init = job.initializeReduceAddresses(rdcAddrs);

    assert init;

    return job;
  }
Ejemplo n.º 2
0
  /**
   * Flushes all the outputs for the given job to remote nodes.
   *
   * @param jobId Job ID.
   * @return Future.
   */
  public GridFuture<?> flush(GridHadoopJobId jobId) {
    GridHadoopShuffleJob job = jobs.get(jobId);

    if (job == null) return new GridFinishedFutureEx<>();

    try {
      return job.flush();
    } catch (GridException e) {
      return new GridFinishedFutureEx<>(e);
    }
  }
Ejemplo n.º 3
0
  /** @param jobId Job id. */
  public void jobFinished(GridHadoopJobId jobId) {
    GridHadoopShuffleJob job = jobs.remove(jobId);

    if (job != null) {
      try {
        job.close();
      } catch (GridException e) {
        U.error(log, "Failed to close job: " + jobId, e);
      }
    }
  }
Ejemplo n.º 4
0
  /**
   * Stops shuffle.
   *
   * @param cancel If should cancel all ongoing activities.
   */
  @Override
  public void stop(boolean cancel) {
    for (GridHadoopShuffleJob job : jobs.values()) {
      try {
        job.close();
      } catch (GridException e) {
        U.error(log, "Failed to close job.", e);
      }
    }

    jobs.clear();
  }
Ejemplo n.º 5
0
  /**
   * @param jobId Task info.
   * @return Shuffle job.
   */
  private GridHadoopShuffleJob<UUID> job(GridHadoopJobId jobId) throws GridException {
    GridHadoopShuffleJob<UUID> res = jobs.get(jobId);

    if (res == null) {
      res = newJob(jobId);

      GridHadoopShuffleJob<UUID> old = jobs.putIfAbsent(jobId, res);

      if (old != null) {
        res.close();

        res = old;
      } else if (res.reducersInitialized()) startSending(res);
    }

    return res;
  }
Ejemplo n.º 6
0
 /**
  * Starts message sending thread.
  *
  * @param shuffleJob Job to start sending for.
  */
 private void startSending(GridHadoopShuffleJob<UUID> shuffleJob) {
   shuffleJob.startSending(
       ctx.kernalContext().gridName(),
       new GridInClosure2X<UUID, GridHadoopShuffleMessage>() {
         @Override
         public void applyx(UUID dest, GridHadoopShuffleMessage msg) throws GridException {
           send0(dest, msg);
         }
       });
 }