示例#1
0
  protected void unregister() {
    try {
      FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
      JobImpl jobImpl = (JobImpl) job;
      if (jobImpl.getInternalState() == JobStateInternal.SUCCEEDED) {
        finishState = FinalApplicationStatus.SUCCEEDED;
      } else if (jobImpl.getInternalState() == JobStateInternal.KILLED
          || (jobImpl.getInternalState() == JobStateInternal.RUNNING && isSignalled)) {
        finishState = FinalApplicationStatus.KILLED;
      } else if (jobImpl.getInternalState() == JobStateInternal.FAILED
          || jobImpl.getInternalState() == JobStateInternal.ERROR) {
        finishState = FinalApplicationStatus.FAILED;
      }
      StringBuffer sb = new StringBuffer();
      for (String s : job.getDiagnostics()) {
        sb.append(s).append("\n");
      }
      LOG.info("Setting job diagnostics to " + sb.toString());

      String historyUrl = JobHistoryUtils.getHistoryUrl(getConfig(), context.getApplicationID());
      LOG.info("History url is " + historyUrl);

      FinishApplicationMasterRequest request =
          FinishApplicationMasterRequest.newInstance(finishState, sb.toString(), historyUrl);
      scheduler.finishApplicationMaster(request);
    } catch (Exception are) {
      LOG.error("Exception while unregistering ", are);
    }
  }
 @Override
 public void unregisterApplicationMaster(
     FinalApplicationStatus appStatus, String appMessage, String appTrackingUrl)
     throws YarnException, IOException {
   Preconditions.checkArgument(appStatus != null, "AppStatus should not be null.");
   FinishApplicationMasterRequest request =
       FinishApplicationMasterRequest.newInstance(appStatus, appMessage, appTrackingUrl);
   try {
     while (true) {
       FinishApplicationMasterResponse response = rmClient.finishApplicationMaster(request);
       if (response.getIsUnregistered()) {
         break;
       }
       LOG.info("Waiting for application to be successfully unregistered.");
       Thread.sleep(100);
     }
   } catch (InterruptedException e) {
     LOG.info("Interrupted while waiting for application" + " to be removed from RMStateStore");
   } catch (ApplicationMasterNotRegisteredException e) {
     LOG.warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing.");
     // re register with RM
     registerApplicationMaster();
     unregisterApplicationMaster(appStatus, appMessage, appTrackingUrl);
   }
 }
示例#3
0
 protected void register() {
   // Register
   InetSocketAddress serviceAddr = null;
   if (clientService != null) {
     serviceAddr = clientService.getBindAddress();
   }
   try {
     RegisterApplicationMasterRequest request =
         recordFactory.newRecordInstance(RegisterApplicationMasterRequest.class);
     if (serviceAddr != null) {
       request.setHost(serviceAddr.getHostName());
       request.setRpcPort(serviceAddr.getPort());
       request.setTrackingUrl(serviceAddr.getHostName() + ":" + clientService.getHttpPort());
     }
     RegisterApplicationMasterResponse response = scheduler.registerApplicationMaster(request);
     maxContainerCapability = response.getMaximumResourceCapability();
     this.context.getClusterInfo().setMaxContainerCapability(maxContainerCapability);
     if (UserGroupInformation.isSecurityEnabled()) {
       setClientToAMToken(response.getClientToAMTokenMasterKey());
     }
     this.applicationACLs = response.getApplicationACLs();
     LOG.info("maxContainerCapability: " + maxContainerCapability.getMemory());
   } catch (Exception are) {
     LOG.error("Exception while registering", are);
     throw new YarnRuntimeException(are);
   }
 }
 @Override
 public FinishApplicationMasterResponseProto finishApplicationMaster(
     RpcController arg0, FinishApplicationMasterRequestProto proto) throws ServiceException {
   FinishApplicationMasterRequestPBImpl request = new FinishApplicationMasterRequestPBImpl(proto);
   try {
     FinishApplicationMasterResponse response = real.finishApplicationMaster(request);
     return ((FinishApplicationMasterResponsePBImpl) response).getProto();
   } catch (YarnException e) {
     throw new ServiceException(e);
   } catch (IOException e) {
     throw new ServiceException(e);
   }
 }
 @Override
 public AllocateResponseProto allocate(RpcController arg0, AllocateRequestProto proto)
     throws ServiceException {
   AllocateRequestPBImpl request = new AllocateRequestPBImpl(proto);
   try {
     AllocateResponse response = real.allocate(request);
     return ((AllocateResponsePBImpl) response).getProto();
   } catch (YarnException e) {
     throw new ServiceException(e);
   } catch (IOException e) {
     throw new ServiceException(e);
   }
 }
 private RegisterApplicationMasterResponse registerApplicationMaster()
     throws YarnException, IOException {
   RegisterApplicationMasterRequest request =
       RegisterApplicationMasterRequest.newInstance(
           this.appHostName, this.appHostPort, this.appTrackingUrl);
   RegisterApplicationMasterResponse response = rmClient.registerApplicationMaster(request);
   synchronized (this) {
     lastResponseId = 0;
     if (!response.getNMTokensFromPreviousAttempts().isEmpty()) {
       populateNMTokens(response.getNMTokensFromPreviousAttempts());
     }
   }
   return response;
 }
 @Override
 public RegisterApplicationMasterResponseProto registerApplicationMaster(
     RpcController arg0, RegisterApplicationMasterRequestProto proto) throws ServiceException {
   RegisterApplicationMasterRequestPBImpl request =
       new RegisterApplicationMasterRequestPBImpl(proto);
   try {
     RegisterApplicationMasterResponse response = real.registerApplicationMaster(request);
     return ((RegisterApplicationMasterResponsePBImpl) response).getProto();
   } catch (YarnException e) {
     throw new ServiceException(e);
   } catch (IOException e) {
     throw new ServiceException(e);
   }
 }
  @Override
  public AllocateResponse allocate(float progressIndicator) throws YarnException, IOException {
    Preconditions.checkArgument(
        progressIndicator >= 0, "Progress indicator should not be negative");
    AllocateResponse allocateResponse = null;
    List<ResourceRequest> askList = null;
    List<ContainerId> releaseList = null;
    AllocateRequest allocateRequest = null;
    List<String> blacklistToAdd = new ArrayList<String>();
    List<String> blacklistToRemove = new ArrayList<String>();

    try {
      synchronized (this) {
        askList = new ArrayList<ResourceRequest>(ask.size());
        for (ResourceRequest r : ask) {
          // create a copy of ResourceRequest as we might change it while the
          // RPC layer is using it to send info across
          askList.add(
              ResourceRequest.newInstance(
                  r.getPriority(),
                  r.getResourceName(),
                  r.getCapability(),
                  r.getNumContainers(),
                  r.getRelaxLocality(),
                  r.getNodeLabelExpression()));
        }
        releaseList = new ArrayList<ContainerId>(release);
        // optimistically clear this collection assuming no RPC failure
        ask.clear();
        release.clear();

        blacklistToAdd.addAll(blacklistAdditions);
        blacklistToRemove.addAll(blacklistRemovals);

        ResourceBlacklistRequest blacklistRequest =
            (blacklistToAdd != null) || (blacklistToRemove != null)
                ? ResourceBlacklistRequest.newInstance(blacklistToAdd, blacklistToRemove)
                : null;

        allocateRequest =
            AllocateRequest.newInstance(
                lastResponseId, progressIndicator, askList, releaseList, blacklistRequest);
        // clear blacklistAdditions and blacklistRemovals before
        // unsynchronized part
        blacklistAdditions.clear();
        blacklistRemovals.clear();
      }

      try {
        allocateResponse = rmClient.allocate(allocateRequest);
      } catch (ApplicationMasterNotRegisteredException e) {
        LOG.warn("ApplicationMaster is out of sync with ResourceManager," + " hence resyncing.");
        synchronized (this) {
          release.addAll(this.pendingRelease);
          blacklistAdditions.addAll(this.blacklistedNodes);
          for (Map<String, TreeMap<Resource, ResourceRequestInfo>> rr :
              remoteRequestsTable.values()) {
            for (Map<Resource, ResourceRequestInfo> capabalities : rr.values()) {
              for (ResourceRequestInfo request : capabalities.values()) {
                addResourceRequestToAsk(request.remoteRequest);
              }
            }
          }
        }
        // re register with RM
        registerApplicationMaster();
        allocateResponse = allocate(progressIndicator);
        return allocateResponse;
      }

      synchronized (this) {
        // update these on successful RPC
        clusterNodeCount = allocateResponse.getNumClusterNodes();
        lastResponseId = allocateResponse.getResponseId();
        clusterAvailableResources = allocateResponse.getAvailableResources();
        if (!allocateResponse.getNMTokens().isEmpty()) {
          populateNMTokens(allocateResponse.getNMTokens());
        }
        if (allocateResponse.getAMRMToken() != null) {
          updateAMRMToken(allocateResponse.getAMRMToken());
        }
        if (!pendingRelease.isEmpty()
            && !allocateResponse.getCompletedContainersStatuses().isEmpty()) {
          removePendingReleaseRequests(allocateResponse.getCompletedContainersStatuses());
        }
      }
    } finally {
      // TODO how to differentiate remote yarn exception vs error in rpc
      if (allocateResponse == null) {
        // we hit an exception in allocate()
        // preserve ask and release for next call to allocate()
        synchronized (this) {
          release.addAll(releaseList);
          // requests could have been added or deleted during call to allocate
          // If requests were added/removed then there is nothing to do since
          // the ResourceRequest object in ask would have the actual new value.
          // If ask does not have this ResourceRequest then it was unchanged and
          // so we can add the value back safely.
          // This assumes that there will no concurrent calls to allocate() and
          // so we dont have to worry about ask being changed in the
          // synchronized block at the beginning of this method.
          for (ResourceRequest oldAsk : askList) {
            if (!ask.contains(oldAsk)) {
              ask.add(oldAsk);
            }
          }

          blacklistAdditions.addAll(blacklistToAdd);
          blacklistRemovals.addAll(blacklistToRemove);
        }
      }
    }
    return allocateResponse;
  }