Пример #1
0
  /**
   * Unregisters a finished or aborted task.
   *
   * @param id the ID of the task to be unregistered
   */
  private void unregisterTask(final ExecutionVertexID id) {

    // Task de-registration must be atomic
    synchronized (this) {
      final Task task = this.runningTasks.remove(id);
      if (task == null) {
        LOG.error("Cannot find task with ID " + id + " to unregister");
        return;
      }

      // remove the local tmp file for unregistered tasks.
      for (Entry<String, DistributedCacheEntry> e :
          DistributedCache.readFileInfoFromConfig(task.getEnvironment().getJobConfiguration())) {
        this.fileCache.deleteTmpFile(e.getKey(), e.getValue(), task.getJobID());
      }
      // Unregister task from the byte buffered channel manager
      this.channelManager.unregister(id, task);

      // Unregister task from profiling
      task.unregisterProfiler(this.profiler);

      // Unregister task from memory manager
      task.unregisterMemoryManager(this.memoryManager);

      // Unregister task from library cache manager
      try {
        LibraryCacheManager.unregister(task.getJobID());
      } catch (IOException e) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Unregistering the job vertex ID " + id + " caused an IOException");
        }
      }
    }
  }
Пример #2
0
  private static Task createTask() {
    LibraryCacheManager libCache = mock(LibraryCacheManager.class);
    when(libCache.getClassLoader(any(JobID.class))).thenReturn(ClassLoader.getSystemClassLoader());

    ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    ResultPartitionConsumableNotifier consumableNotifier =
        mock(ResultPartitionConsumableNotifier.class);
    NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
    when(networkEnvironment.getPartitionManager()).thenReturn(partitionManager);
    when(networkEnvironment.getPartitionConsumableNotifier()).thenReturn(consumableNotifier);
    when(networkEnvironment.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);

    TaskDeploymentDescriptor tdd =
        new TaskDeploymentDescriptor(
            new JobID(),
            new JobVertexID(),
            new ExecutionAttemptID(),
            "Test Task",
            0,
            1,
            new Configuration(),
            new Configuration(),
            CheckpointsInOrderInvokable.class.getName(),
            Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
            Collections.<InputGateDeploymentDescriptor>emptyList(),
            Collections.<BlobKey>emptyList(),
            0);

    ActorGateway taskManagerGateway = DummyActorGateway.INSTANCE;
    return new Task(
        tdd,
        mock(MemoryManager.class),
        mock(IOManager.class),
        networkEnvironment,
        mock(BroadcastVariableManager.class),
        taskManagerGateway,
        DummyActorGateway.INSTANCE,
        new FiniteDuration(60, TimeUnit.SECONDS),
        libCache,
        mock(FileCache.class),
        new TaskManagerRuntimeInfo("localhost", new Configuration()));
  }
Пример #3
0
  @Override
  public LibraryCacheProfileResponse getLibraryCacheProfile(LibraryCacheProfileRequest request)
      throws IOException {

    LibraryCacheProfileResponse response = new LibraryCacheProfileResponse(request);
    String[] requiredLibraries = request.getRequiredLibraries();

    for (int i = 0; i < requiredLibraries.length; i++) {
      if (LibraryCacheManager.contains(requiredLibraries[i]) == null) {
        response.setCached(i, false);
      } else {
        response.setCached(i, true);
      }
    }

    return response;
  }
  @Override
  public void invoke() throws Exception {
    userCodeClassLoader = LibraryCacheManager.getClassLoader(getEnvironment().getJobID());
    TaskConfig taskConfig = new TaskConfig(getTaskConfiguration());

    // store all aggregators
    this.aggregators = new HashMap<String, Aggregator<?>>();
    for (AggregatorWithName<?> aggWithName : taskConfig.getIterationAggregators()) {
      aggregators.put(aggWithName.getName(), aggWithName.getAggregator());
    }

    // store the aggregator convergence criterion
    if (taskConfig.usesConvergenceCriterion()) {
      convergenceCriterion = taskConfig.getConvergenceCriterion();
      convergenceAggregatorName = taskConfig.getConvergenceCriterionAggregatorName();
      Preconditions.checkNotNull(convergenceAggregatorName);
    }

    maxNumberOfIterations = taskConfig.getNumberOfIterations();

    // set up the event handler
    int numEventsTillEndOfSuperstep = taskConfig.getNumberOfEventsUntilInterruptInIterativeGate(0);
    eventHandler =
        new SyncEventHandler(numEventsTillEndOfSuperstep, aggregators, userCodeClassLoader);
    headEventReader.subscribeToEvent(eventHandler, WorkerDoneEvent.class);

    IntegerRecord dummy = new IntegerRecord();

    while (!terminationRequested()) {

      //			notifyMonitor(IterationMonitoring.Event.SYNC_STARTING, currentIteration);
      if (log.isInfoEnabled()) {
        log.info(formatLogString("starting iteration [" + currentIteration + "]"));
      }

      // this call listens for events until the end-of-superstep is reached
      readHeadEventChannel(dummy);

      if (log.isInfoEnabled()) {
        log.info(formatLogString("finishing iteration [" + currentIteration + "]"));
      }

      if (checkForConvergence()) {
        if (log.isInfoEnabled()) {
          log.info(
              formatLogString(
                  "signaling that all workers are to terminate in iteration ["
                      + currentIteration
                      + "]"));
        }

        requestTermination();
        sendToAllWorkers(new TerminationEvent());
        //				notifyMonitor(IterationMonitoring.Event.SYNC_FINISHED, currentIteration);
      } else {
        if (log.isInfoEnabled()) {
          log.info(
              formatLogString(
                  "signaling that all workers are done in iteration [" + currentIteration + "]"));
        }

        AllWorkersDoneEvent allWorkersDoneEvent = new AllWorkersDoneEvent(aggregators);
        sendToAllWorkers(allWorkersDoneEvent);

        // reset all aggregators
        for (Aggregator<?> agg : aggregators.values()) {
          agg.reset();
        }

        //				notifyMonitor(IterationMonitoring.Event.SYNC_FINISHED, currentIteration);
        currentIteration++;
      }
    }
  }