Пример #1
0
 public void setTaskSchedulerForTag(Object tag, Class<? extends TaskScheduler> scheduler) {
   synchronized (schedulerByTag) {
     TaskScheduler old = getTaskSchedulerForTag(tag);
     if (old != null) {
       if (scheduler.isAssignableFrom(old.getClass())) {
         /* already have such an instance */
         return;
       }
       // might support multiple in future...
       throw new IllegalStateException(
           "Not allowed to set multiple TaskSchedulers on ExecutionManager tag (tag "
               + tag
               + ", has "
               + old
               + ", setting new "
               + scheduler
               + ")");
     }
     try {
       TaskScheduler schedulerI = scheduler.newInstance();
       // allow scheduler to have a nice name, for logging etc
       if (schedulerI instanceof CanSetName) ((CanSetName) schedulerI).setName("" + tag);
       setTaskSchedulerForTag(tag, schedulerI);
     } catch (InstantiationException e) {
       throw Throwables.propagate(e);
     } catch (IllegalAccessException e) {
       throw Throwables.propagate(e);
     }
   }
 }
Пример #2
0
  @Test
  public void testTaskDeletedBeforeEvaluating() {
    final IScheduledTask task = makeTask(TASK_A_ID);
    expect(rateLimiter.acquire()).andReturn(0D);
    expect(taskScheduler.schedule(Tasks.id(task)))
        .andAnswer(
            new IAnswer<Boolean>() {
              @Override
              public Boolean answer() {
                // Test a corner case where a task is deleted while it is being evaluated by the
                // task
                // scheduler.  If not handled carefully, this could result in the scheduler trying
                // again
                // later to satisfy the deleted task.
                taskGroups.tasksDeleted(new TasksDeleted(ImmutableSet.of(task)));

                return false;
              }
            });
    expect(backoffStrategy.calculateBackoffMs(FIRST_SCHEDULE_DELAY.as(Time.MILLISECONDS)))
        .andReturn(0L);

    control.replay();

    taskGroups.taskChangedState(TaskStateChange.transition(makeTask(Tasks.id(task)), INIT));
    clock.advance(FIRST_SCHEDULE_DELAY);
  }
Пример #3
0
  public boolean enqueueTaskReservations(TEnqueueTaskReservationsRequest request) {
    LOG.debug(Logging.functionCall(request));
    AUDIT_LOG.info(
        Logging.auditEventString(
            "node_monitor_enqueue_task_reservation", ipAddress, request.requestId));
    LOG.info(
        "Received enqueue task reservation request from "
            + ipAddress
            + " for request "
            + request.requestId);

    InetSocketAddress schedulerAddress =
        new InetSocketAddress(
            request.getSchedulerAddress().getHost(), request.getSchedulerAddress().getPort());
    requestSchedulers.put(request.getRequestId(), schedulerAddress);

    InetSocketAddress socket = appSockets.get(request.getAppId());
    if (socket == null) {
      LOG.error(
          "No socket stored for "
              + request.getAppId()
              + " (never registered?). "
              + "Can't launch task.");
      return false;
    }
    scheduler.submitTaskReservations(request, socket);
    return true;
  }
 @Override
 public synchronized void start() throws IOException {
   super.start();
   taskTrackerManager.addJobInProgressListener(jobQueueJobInProgressListener);
   eagerTaskInitializationListener.setTaskTrackerManager(taskTrackerManager);
   eagerTaskInitializationListener.start();
   taskTrackerManager.addJobInProgressListener(eagerTaskInitializationListener);
 }
Пример #5
0
  @Test
  public void testEvaluatedAfterFirstSchedulePenalty() {
    expect(rateLimiter.acquire()).andReturn(0D);
    expect(taskScheduler.schedule(TASK_A_ID)).andReturn(true);

    control.replay();

    taskGroups.taskChangedState(TaskStateChange.transition(makeTask(TASK_A_ID), INIT));
    clock.advance(FIRST_SCHEDULE_DELAY);
  }
 @Override
 public synchronized void terminate() throws IOException {
   if (jobQueueJobInProgressListener != null) {
     taskTrackerManager.removeJobInProgressListener(jobQueueJobInProgressListener);
   }
   if (eagerTaskInitializationListener != null) {
     taskTrackerManager.removeJobInProgressListener(eagerTaskInitializationListener);
     eagerTaskInitializationListener.terminate();
   }
   super.terminate();
 }
Пример #7
0
  @Test
  public void testEvaluatedOnStartup() {
    expect(rateLimiter.acquire()).andReturn(0D);
    expect(rescheduleCalculator.getStartupScheduleDelayMs(makeTask(TASK_A_ID))).andReturn(1L);
    expect(taskScheduler.schedule(TASK_A_ID)).andReturn(true);

    control.replay();

    taskGroups.taskChangedState(TaskStateChange.initialized(makeTask(TASK_A_ID)));
    clock.advance(FIRST_SCHEDULE_DELAY);
    clock.advance(RESCHEDULE_DELAY);
  }
Пример #8
0
  /**
   * Defines a {@link TaskScheduler} to run on all subsequently submitted jobs with the given tag.
   *
   * <p>Maximum of one allowed currently. Resubmissions of the same scheduler (or scheduler class)
   * allowed. If changing, you must call {@link #clearTaskSchedulerForTag(Object)} between the two.
   *
   * @see #setTaskSchedulerForTag(Object, Class)
   */
  public void setTaskSchedulerForTag(Object tag, TaskScheduler scheduler) {
    synchronized (schedulerByTag) {
      scheduler.injectExecutor(runner);

      Object old = schedulerByTag.put(tag, scheduler);
      if (old != null && old != scheduler) {
        // might support multiple in future...
        throw new IllegalStateException(
            "Not allowed to set multiple TaskSchedulers on ExecutionManager tag (tag " + tag + ")");
      }
    }
  }
Пример #9
0
  @Test
  public void testResistStarvation() {
    expect(rateLimiter.acquire()).andReturn(0D).times(2);
    expect(taskScheduler.schedule("a0")).andReturn(true);
    expect(taskScheduler.schedule("b0")).andReturn(true);

    control.replay();

    taskGroups.taskChangedState(TaskStateChange.transition(makeTask(JOB_A, "a0", 0), INIT));
    taskGroups.taskChangedState(TaskStateChange.transition(makeTask(JOB_A, "a1", 1), INIT));
    taskGroups.taskChangedState(TaskStateChange.transition(makeTask(JOB_A, "a2", 2), INIT));
    taskGroups.taskChangedState(
        TaskStateChange.transition(
            makeTask(IJobKey.build(JOB_A.newBuilder().setName("jobB")), "b0", 0), INIT));

    clock.advance(FIRST_SCHEDULE_DELAY);
  }
Пример #10
0
  public void initialize(Configuration conf, int nodeMonitorInternalPort)
      throws UnknownHostException {
    String mode = conf.getString(SparrowConf.DEPLYOMENT_MODE, "unspecified");
    if (mode.equals("standalone")) {
      state = new StandaloneNodeMonitorState();
    } else if (mode.equals("configbased")) {
      state = new ConfigNodeMonitorState();
    } else if (mode.equals("production")) {
      state = new StateStoreNodeMonitorState();
    } else {
      throw new RuntimeException("Unsupported deployment mode: " + mode);
    }
    try {
      state.initialize(conf);
    } catch (IOException e) {
      LOG.fatal("Error initializing node monitor state.", e);
    }
    capacity = new TResourceVector();
    int mem = Resources.getSystemMemoryMb(conf);
    capacity.setMemory(mem);
    LOG.info("Using memory allocation: " + mem);

    ipAddress = Network.getIPAddress(conf);

    int cores = Resources.getSystemCPUCount(conf);
    capacity.setCores(cores);
    LOG.info("Using core allocation: " + cores);

    String task_scheduler_type = conf.getString(SparrowConf.NM_TASK_SCHEDULER_TYPE, "fifo");
    if (task_scheduler_type.equals("round_robin")) {
      scheduler = new RoundRobinTaskScheduler(cores);
    } else if (task_scheduler_type.equals("fifo")) {
      scheduler = new FifoTaskScheduler(cores);
    } else if (task_scheduler_type.equals("priority")) {
      scheduler = new PriorityTaskScheduler(cores);
    } else {
      throw new RuntimeException("Unsupported task scheduler type: " + mode);
    }
    scheduler.initialize(capacity, conf, nodeMonitorInternalPort);
    taskLauncherService = new TaskLauncherService();
    taskLauncherService.initialize(conf, scheduler, nodeMonitorInternalPort);
  }
Пример #11
0
 /** Account for tasks which have finished. */
 public void tasksFinished(List<TFullTaskId> tasks) {
   LOG.debug(Logging.functionCall(tasks));
   scheduler.tasksFinished(tasks);
 }
 @Override
 public synchronized void setConf(Configuration conf) {
   super.setConf(conf);
   padFraction = conf.getFloat("mapred.jobtracker.taskalloc.capacitypad", 0.01f);
   this.eagerTaskInitializationListener = new EagerTaskInitializationListener(conf);
 }
Пример #13
0
 @Override
 public void accountDeleted(AccountKey key) {
   scheduler.cancelTask(key);
 }
Пример #14
0
 @Override
 public void accountCreated(AccountKey key) {
   FetchingTask task = taskFactory.createIncrementalTask(key);
   scheduler.scheduleTask(task);
 }
Пример #15
0
 /**
  * 放入任务
  *
  * @param task
  */
 public void addTask(Task task) {
   if (task != null) {
     taskScheduler.addTask(task);
   }
 }