예제 #1
0
  protected LinkedList<TaskRequestEvent> createTaskRequest(final int incompleteTaskNum)
      throws Exception {
    LinkedList<TaskRequestEvent> taskRequestEvents = new LinkedList<>();

    // If scheduled tasks is long-term task, cluster resource can be the worst load balance.
    // This part is to throttle the maximum required container per request
    int requestContainerNum = Math.min(incompleteTaskNum, maximumRequestContainer);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Try to schedule task resources: " + requestContainerNum);
    }

    ServiceTracker serviceTracker =
        context.getMasterContext().getQueryMasterContext().getWorkerContext().getServiceTracker();
    NettyClientBase tmClient =
        RpcClientManager.getInstance()
            .getClient(
                serviceTracker.getUmbilicalAddress(),
                QueryCoordinatorProtocol.class,
                true,
                rpcParams);
    QueryCoordinatorProtocolService masterClientService = tmClient.getStub();

    CallFuture<NodeResourceResponse> callBack = new CallFuture<>();
    NodeResourceRequest.Builder request = NodeResourceRequest.newBuilder();
    request
        .setCapacity(NodeResources.createResource(minTaskMemory, isLeaf ? 1 : 0).getProto())
        .setNumContainers(requestContainerNum)
        .setPriority(stage.getPriority())
        .setQueryId(context.getMasterContext().getQueryId().getProto())
        .setType(isLeaf ? ResourceType.LEAF : ResourceType.INTERMEDIATE)
        .setUserId(context.getMasterContext().getQueryContext().getUser())
        .setRunningTasks(stage.getTotalScheduledObjectsCount() - stage.getCompletedTaskCount())
        .addAllCandidateNodes(candidateWorkers)
        .setQueue(
            context.getMasterContext().getQueryContext().get("queue", "default")); // TODO set queue

    masterClientService.reserveNodeResources(callBack.getController(), request.build(), callBack);
    NodeResourceResponse response =
        callBack.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS);

    for (AllocationResourceProto resource : response.getResourceList()) {
      taskRequestEvents.add(
          new TaskRequestEvent(resource.getWorkerId(), resource, context.getBlockId()));
    }

    return taskRequestEvents;
  }
예제 #2
0
  public void cancel(TaskAttempt taskAttempt) {

    if (taskAttempt.isLeafTask()) {
      releaseTaskAttempt(taskAttempt);

      List<DataLocation> locations = taskAttempt.getTask().getDataLocations();

      for (DataLocation location : locations) {
        HostVolumeMapping volumeMapping =
            scheduledRequests.leafTaskHostMapping.get(location.getHost());
        volumeMapping.addTaskAttempt(location.getVolumeId(), taskAttempt);
      }

      scheduledRequests.leafTasks.add(taskAttempt.getId());
    } else {
      scheduledRequests.nonLeafTasks.add(taskAttempt.getId());
    }

    context
        .getMasterContext()
        .getEventHandler()
        .handle(new TaskAttemptEvent(taskAttempt.getId(), TaskAttemptEventType.TA_ASSIGN_CANCEL));
  }