void launchWorkerNode(Container container) throws IOException, YarnException {
Map<String, String> env = System.getenv();
ContainerLaunchContext workerContext = Records.newRecord(ContainerLaunchContext.class);
LocalResource workerJar = Records.newRecord(LocalResource.class);
setupWorkerJar(workerJar);
workerContext.setLocalResources(Collections.singletonMap("socialite.jar", workerJar));
System.out.println("[Master] workerJar:" + workerJar);
Map<String, String> workerEnv = new HashMap<String, String>(env);
setupWorkerEnv(workerEnv);
workerContext.setEnvironment(workerEnv);
String opts = "-Dsocialite.master=" + NetUtils.getHostname().split("/")[1] + " ";
opts += "-Dsocialite.worker.num_threads=" + ClusterConf.get().getNumWorkerThreads() + " ";
workerContext.setCommands(
Collections.singletonList(
"$JAVA_HOME/bin/java -ea "
+ env.get("JVM_OPTS")
+ " "
+ env.get("SOCIALITE_OPTS")
+ " "
+ opts
+ " "
+ "socialite.dist.worker.WorkerNode"
+ " 1>"
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR
+ "/stdout"
+ " 2>"
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR
+ "/stderr"));
nmClient.startContainer(container, workerContext);
L.info("Launched worker node (container:" + container.getId() + ")");
}
private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) throws Exception {
YarnClientApplication newApp = rmClient.createApplication();
ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();
// Create launch context for app master
ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
// set the application id
appContext.setApplicationId(appId);
// set the application name
appContext.setApplicationName("test");
// Set the priority for the application master
Priority pri = Records.newRecord(Priority.class);
pri.setPriority(1);
appContext.setPriority(pri);
// Set the queue to which this application is to be submitted in the RM
appContext.setQueue("default");
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
appContext.setAMContainerSpec(amContainer);
appContext.setResource(Resource.newInstance(1024, 1));
appContext.setUnmanagedAM(unmanaged);
// Submit the application to the applications manager
rmClient.submitApplication(appContext);
return appId;
}
public NodeHeartbeatResponse nodeHeartbeat(
Map<ApplicationId, List<ContainerStatus>> conts, boolean isHealthy, int resId)
throws Exception {
NodeHeartbeatRequest req = Records.newRecord(NodeHeartbeatRequest.class);
NodeStatus status = Records.newRecord(NodeStatus.class);
status.setResponseId(resId);
status.setNodeId(nodeId);
for (Map.Entry<ApplicationId, List<ContainerStatus>> entry : conts.entrySet()) {
Log.info("entry.getValue() " + entry.getValue());
status.setContainersStatuses(entry.getValue());
}
NodeHealthStatus healthStatus = Records.newRecord(NodeHealthStatus.class);
healthStatus.setHealthReport("");
healthStatus.setIsNodeHealthy(isHealthy);
healthStatus.setLastHealthReportTime(1);
status.setNodeHealthStatus(healthStatus);
req.setNodeStatus(status);
req.setLastKnownContainerTokenMasterKey(this.currentContainerTokenMasterKey);
req.setLastKnownNMTokenMasterKey(this.currentNMTokenMasterKey);
NodeHeartbeatResponse heartbeatResponse = resourceTracker.nodeHeartbeat(req);
MasterKey masterKeyFromRM = heartbeatResponse.getContainerTokenMasterKey();
if (masterKeyFromRM != null
&& masterKeyFromRM.getKeyId() != this.currentContainerTokenMasterKey.getKeyId()) {
this.currentContainerTokenMasterKey = masterKeyFromRM;
}
masterKeyFromRM = heartbeatResponse.getNMTokenMasterKey();
if (masterKeyFromRM != null
&& masterKeyFromRM.getKeyId() != this.currentNMTokenMasterKey.getKeyId()) {
this.currentNMTokenMasterKey = masterKeyFromRM;
}
return heartbeatResponse;
}
@Test
public void testReportDiagnostics() throws Exception {
JobID jobID = JobID.forName("job_1234567890000_0001");
JobId jobId = TypeConverter.toYarn(jobID);
final String diagMsg = "some diagnostic message";
final JobDiagnosticsUpdateEvent diagUpdateEvent = new JobDiagnosticsUpdateEvent(jobId, diagMsg);
MRAppMetrics mrAppMetrics = MRAppMetrics.create();
AppContext mockContext = mock(AppContext.class);
when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
JobImpl job =
new JobImpl(
jobId,
Records.newRecord(ApplicationAttemptId.class),
new Configuration(),
mock(EventHandler.class),
null,
mock(JobTokenSecretManager.class),
null,
new SystemClock(),
null,
mrAppMetrics,
null,
true,
null,
0,
null,
mockContext,
null,
null);
job.handle(diagUpdateEvent);
String diagnostics = job.getReport().getDiagnostics();
Assert.assertNotNull(diagnostics);
Assert.assertTrue(diagnostics.contains(diagMsg));
job =
new JobImpl(
jobId,
Records.newRecord(ApplicationAttemptId.class),
new Configuration(),
mock(EventHandler.class),
null,
mock(JobTokenSecretManager.class),
null,
new SystemClock(),
null,
mrAppMetrics,
null,
true,
null,
0,
null,
mockContext,
null,
null);
job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
job.handle(diagUpdateEvent);
diagnostics = job.getReport().getDiagnostics();
Assert.assertNotNull(diagnostics);
Assert.assertTrue(diagnostics.contains(diagMsg));
}
public void submitApplication(
ApplicationId appId,
String appName,
Map<String, String> env,
Map<String, LocalResource> localResources,
List<String> commands,
int memory)
throws URISyntaxException, IOException {
if (clientResourceManager == null)
throw new IllegalStateException(
"Cannot submit an application without connecting to resource manager!");
ApplicationSubmissionContext appCtx = Records.newRecord(ApplicationSubmissionContext.class);
appCtx.setApplicationId(appId);
appCtx.setApplicationName(appName);
appCtx.setQueue("default");
appCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
// System.out.println( "Based on my current user I am: " +
// UserGroupInformation.getCurrentUser().getShortUserName() );
Priority prio = Records.newRecord(Priority.class);
prio.setPriority(0);
appCtx.setPriority(prio);
/*
for (Map.Entry<String, LocalResource> entry : localResources.entrySet())
{
System.out.println("IR:RM: " + entry.getKey() + "/" + entry.getValue().getResource());
}
*/
/*
for (Map.Entry<String, String> entry : env.entrySet())
{
System.out.println("IR:ResourceManager -> Env vars: " + entry.getKey() + "/" + entry.getValue() );
}
*/
// Launch ctx
ContainerLaunchContext containerCtx = Records.newRecord(ContainerLaunchContext.class);
containerCtx.setLocalResources(localResources);
containerCtx.setCommands(commands);
containerCtx.setEnvironment(env);
containerCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(memory);
containerCtx.setResource(capability);
appCtx.setAMContainerSpec(containerCtx);
SubmitApplicationRequest submitReq = Records.newRecord(SubmitApplicationRequest.class);
submitReq.setApplicationSubmissionContext(appCtx);
LOG.info("Submitting application to ASM");
clientResourceManager.submitApplication(submitReq);
// Don't return anything, ASM#submit returns an empty response
}
public void launchSupervisorOnContainer(Container container) throws IOException {
LOG.info("Connecting to ContainerManager for containerid=" + container.getId());
String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
LOG.info("Connecting to ContainerManager at " + cmIpPortStr);
ContainerManager proxy =
((ContainerManager) rpc.getProxy(ContainerManager.class, cmAddress, hadoopConf));
LOG.info("launchSupervisorOnContainer( id:" + container.getId() + " )");
ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);
launchContext.setContainerId(container.getId());
launchContext.setResource(container.getResource());
try {
launchContext.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
} catch (IOException e) {
LOG.info(
"Getting current user info failed when trying to launch the container" + e.getMessage());
}
Map<String, String> env = new HashMap<String, String>();
launchContext.setEnvironment(env);
Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
String stormVersion = Util.getStormVersion(this.storm_conf);
Path zip = new Path("/lib/storm/" + stormVersion + "/storm.zip");
FileSystem fs = FileSystem.get(this.hadoopConf);
localResources.put(
"storm",
Util.newYarnAppResource(
fs, zip, LocalResourceType.ARCHIVE, LocalResourceVisibility.PUBLIC));
String appHome = Util.getApplicationHomeForId(this.appAttemptId.toString());
Path dirDst = Util.createConfigurationFileInFs(fs, appHome, this.storm_conf, this.hadoopConf);
localResources.put("conf", Util.newYarnAppResource(fs, dirDst));
launchContext.setLocalResources(localResources);
List<String> supervisorArgs = Util.buildSupervisorCommands(this.storm_conf);
launchContext.setCommands(supervisorArgs);
StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class);
startRequest.setContainerLaunchContext(launchContext);
LOG.info(
"launchSupervisorOnContainer: startRequest prepared, calling startContainer. "
+ startRequest);
try {
StartContainerResponse response = proxy.startContainer(startRequest);
LOG.info("Got a start container response " + response);
} catch (Exception e) {
LOG.error("Caught an exception while trying to start a container", e);
System.exit(-1);
}
}
private GetJobReportResponse getJobReportResponse() {
GetJobReportResponse jobReportResponse = Records.newRecord(GetJobReportResponse.class);
JobReport jobReport = Records.newRecord(JobReport.class);
jobReport.setJobId(jobId);
jobReport.setJobState(JobState.SUCCEEDED);
jobReportResponse.setJobReport(jobReport);
return jobReportResponse;
}
@Test
public void testNodeHeartBeatWithInvalidLabels() throws Exception {
writeToHostsFile("host2");
Configuration conf = new Configuration();
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath());
conf.set(
YarnConfiguration.NODELABEL_CONFIGURATION_TYPE,
YarnConfiguration.DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE);
final RMNodeLabelsManager nodeLabelsMgr = new NullRMNodeLabelsManager();
rm =
new MockRM(conf) {
@Override
protected RMNodeLabelsManager createNodeLabelManager() {
return nodeLabelsMgr;
}
};
rm.start();
try {
nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C"));
} catch (IOException e) {
Assert.fail("Caught Exception while intializing");
e.printStackTrace();
}
ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
RegisterNodeManagerRequest registerReq = Records.newRecord(RegisterNodeManagerRequest.class);
NodeId nodeId = NodeId.newInstance("host2", 1234);
Resource capability = BuilderUtils.newResource(1024, 1);
registerReq.setResource(capability);
registerReq.setNodeId(nodeId);
registerReq.setHttpPort(1234);
registerReq.setNMVersion(YarnVersionInfo.getVersion());
registerReq.setNodeLabels(toNodeLabelSet("A"));
RegisterNodeManagerResponse registerResponse =
resourceTrackerService.registerNodeManager(registerReq);
NodeHeartbeatRequest heartbeatReq = Records.newRecord(NodeHeartbeatRequest.class);
heartbeatReq.setNodeLabels(toNodeLabelSet("B", "#C")); // Invalid heart beat labels
heartbeatReq.setNodeStatus(getNodeStatusObject(nodeId));
heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse.getNMTokenMasterKey());
heartbeatReq.setLastKnownContainerTokenMasterKey(registerResponse.getContainerTokenMasterKey());
NodeHeartbeatResponse nodeHeartbeatResponse =
resourceTrackerService.nodeHeartbeat(heartbeatReq);
// response should be NORMAL when RM heartbeat labels are rejected
Assert.assertEquals(
"Response should be NORMAL when RM heartbeat labels" + " are rejected",
NodeAction.NORMAL,
nodeHeartbeatResponse.getNodeAction());
Assert.assertFalse(nodeHeartbeatResponse.getAreNodeLabelsAcceptedByRM());
Assert.assertNotNull(nodeHeartbeatResponse.getDiagnosticsMessage());
rm.stop();
}
public void killApplication(String appId) throws Exception {
KillApplicationRequest killRequest = Records.newRecord(KillApplicationRequest.class);
ApplicationId aid = Records.newRecord(ApplicationId.class);
long ts = Long.parseLong(appId.substring(appId.indexOf('_') + 1, appId.lastIndexOf('_')));
aid.setClusterTimestamp(ts);
int id = Integer.parseInt(appId.substring(appId.lastIndexOf('_') + 1));
aid.setId(id);
killRequest.setApplicationId(aid);
crmp.forceKillApplication(killRequest);
}
private GetJobReportResponse getJobReportResponseFromHistoryServer() {
GetJobReportResponse jobReportResponse = Records.newRecord(GetJobReportResponse.class);
JobReport jobReport = Records.newRecord(JobReport.class);
jobReport.setJobId(jobId);
jobReport.setJobState(JobState.SUCCEEDED);
jobReport.setMapProgress(1.0f);
jobReport.setReduceProgress(1.0f);
jobReport.setJobFile("TestJobFilePath");
jobReport.setTrackingUrl("http://TestTrackingUrl");
jobReportResponse.setJobReport(jobReport);
return jobReportResponse;
}
@Test
public void testNodeHeartbeatWithCentralLabelConfig() throws Exception {
writeToHostsFile("host2");
Configuration conf = new Configuration();
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath());
conf.set(
YarnConfiguration.NODELABEL_CONFIGURATION_TYPE,
YarnConfiguration.DEFAULT_NODELABEL_CONFIGURATION_TYPE);
final RMNodeLabelsManager nodeLabelsMgr = new NullRMNodeLabelsManager();
rm =
new MockRM(conf) {
@Override
protected RMNodeLabelsManager createNodeLabelManager() {
return nodeLabelsMgr;
}
};
rm.start();
ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
RegisterNodeManagerRequest req = Records.newRecord(RegisterNodeManagerRequest.class);
NodeId nodeId = NodeId.newInstance("host2", 1234);
Resource capability = BuilderUtils.newResource(1024, 1);
req.setResource(capability);
req.setNodeId(nodeId);
req.setHttpPort(1234);
req.setNMVersion(YarnVersionInfo.getVersion());
req.setNodeLabels(toNodeLabelSet("A", "B", "C"));
RegisterNodeManagerResponse registerResponse = resourceTrackerService.registerNodeManager(req);
NodeHeartbeatRequest heartbeatReq = Records.newRecord(NodeHeartbeatRequest.class);
heartbeatReq.setNodeLabels(toNodeLabelSet("B")); // Valid heart beat labels
heartbeatReq.setNodeStatus(getNodeStatusObject(nodeId));
heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse.getNMTokenMasterKey());
heartbeatReq.setLastKnownContainerTokenMasterKey(registerResponse.getContainerTokenMasterKey());
NodeHeartbeatResponse nodeHeartbeatResponse =
resourceTrackerService.nodeHeartbeat(heartbeatReq);
// response should be ok but the RMacceptNodeLabelsUpdate should be false
Assert.assertEquals(NodeAction.NORMAL, nodeHeartbeatResponse.getNodeAction());
// no change in the labels,
Assert.assertNull(nodeLabelsMgr.getNodeLabels().get(nodeId));
// heartbeat labels rejected
Assert.assertFalse(
"Invalid Node Labels should not accepted by RM",
nodeHeartbeatResponse.getAreNodeLabelsAcceptedByRM());
if (rm != null) {
rm.stop();
}
}
/**
* Setup a container request on specified node
*
* @param node the specified node
* @return ResourceRequest sent to RM
*/
private ResourceRequest setupAContainerAskForRM(String node) {
ResourceRequest request = Records.newRecord(ResourceRequest.class);
request.setResourceName(node);
request.setNumContainers(1); // important
Priority priority = Records.newRecord(Priority.class);
priority.setPriority(requestPriority);
request.setPriority(priority);
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(containerMemory);
request.setCapability(capability);
return request;
}
/**
* Utility method creating a {@link ResourceRequest}.
*
* @param numContainers number of containers to request
* @return request to be sent to resource manager
*/
private ResourceRequest getContainerResourceRequest(
int numContainers, String hostName, boolean relaxLocality) {
ResourceRequest request = Records.newRecord(ResourceRequest.class);
request.setRelaxLocality(relaxLocality);
request.setResourceName(hostName);
request.setNumContainers(numContainers);
Priority pri = Records.newRecord(Priority.class);
pri.setPriority(priority);
request.setPriority(pri);
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(memory);
ResourceCompat.setVirtualCores(capability, virtualcores);
request.setCapability(capability);
return request;
}
private GetCountersResponse getCountersResponseFromHistoryServer() {
GetCountersResponse countersResponse = Records.newRecord(GetCountersResponse.class);
Counter counter = Records.newRecord(Counter.class);
CounterGroup counterGroup = Records.newRecord(CounterGroup.class);
Counters counters = Records.newRecord(Counters.class);
counter.setDisplayName("dummyCounter");
counter.setName("dummyCounter");
counter.setValue(1001);
counterGroup.setName("dummyCounters");
counterGroup.setDisplayName("dummyCounters");
counterGroup.setCounter("dummyCounter", counter);
counters.setCounterGroup("dummyCounters", counterGroup);
countersResponse.setCounters(counters);
return countersResponse;
}
public void runMainLoop() throws Exception {
rmClient.registerApplicationMaster("", 0, "");
initRegistry();
startMasterNode();
Priority priority = Records.newRecord(Priority.class);
priority.setPriority(0);
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(ClusterConf.get().getWorkerHeapSize());
capability.setVirtualCores(ClusterConf.get().getNumWorkerThreads());
List<ContainerRequest> containerReq = new ArrayList<ContainerRequest>();
for (int i = 0; i < ClusterConf.get().getNumWorkers(); ++i) {
ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
rmClient.addContainerRequest(containerAsk);
containerReq.add(containerAsk);
}
while (alive) {
Thread.sleep(1000);
}
finish();
/*
for (ContainerRequest req : containerReq) {
rmClient.removeContainerRequest(req);
}
int containersToAdd = 2;
numContainersToWaitFor = containersToAdd;
System.out.println("[Am] finished all containers. Asking for more containers, total=" + numContainersToWaitFor);
for (int i = 0; i < containersToAdd; ++i) {
ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
System.out.println("[AM] Making res-req " + i);
rmClient.addContainerRequest(containerAsk);
}
System.out.println("[AM] waiting for containers to finish once more!!!");
while (!doneWithContainers()) {
Thread.sleep(100);
}
*/
}
private ContainerLaunchContext newContainerLaunchContext(
Container container, String helixInstanceName) throws IOException {
Path appWorkDir =
GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, this.applicationId);
Path containerWorkDir =
new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME);
Map<String, LocalResource> resourceMap = Maps.newHashMap();
addContainerLocalResources(
new Path(appWorkDir, GobblinYarnConfigurationKeys.LIB_JARS_DIR_NAME), resourceMap);
addContainerLocalResources(
new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME), resourceMap);
addContainerLocalResources(
new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME), resourceMap);
if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY)) {
addRemoteAppFiles(
this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY),
resourceMap);
}
ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
containerLaunchContext.setLocalResources(resourceMap);
containerLaunchContext.setEnvironment(
YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
containerLaunchContext.setCommands(
Lists.newArrayList(buildContainerCommand(container, helixInstanceName)));
if (UserGroupInformation.isSecurityEnabled()) {
containerLaunchContext.setTokens(this.tokens.duplicate());
}
return containerLaunchContext;
}
@Override
public void killApplication(ApplicationId applicationId) throws YarnException, IOException {
KillApplicationRequest request = Records.newRecord(KillApplicationRequest.class);
request.setApplicationId(applicationId);
try {
int pollCount = 0;
long startTime = System.currentTimeMillis();
while (true) {
KillApplicationResponse response = rmClient.forceKillApplication(request);
if (response.getIsKillCompleted()) {
LOG.info("Killed application " + applicationId);
break;
}
long elapsedMillis = System.currentTimeMillis() - startTime;
if (enforceAsyncAPITimeout() && elapsedMillis >= this.asyncApiPollTimeoutMillis) {
throw new YarnException(
"Timed out while waiting for application " + applicationId + " to be killed.");
}
if (++pollCount % 10 == 0) {
LOG.info("Waiting for application " + applicationId + " to be killed.");
}
Thread.sleep(asyncApiPollIntervalMillis);
}
} catch (InterruptedException e) {
LOG.error("Interrupted while waiting for application " + applicationId + " to be killed.");
}
}
public ChildExecutor(ChildExecutionContext context) {
this.context = context;
this.taskAttemptId = context.getTaskAttemptId();
this.taskType = context.getTaskType();
attemptReport = Records.newRecord(TaskAttemptReport.class);
gcUpdater = new GcTimeUpdater();
}
@Public
@Stable
public static KillApplicationRequest newInstance(ApplicationId applicationId) {
KillApplicationRequest request = Records.newRecord(KillApplicationRequest.class);
request.setApplicationId(applicationId);
return request;
}
@Test
public void testNodeRegistrationVersionLessThanRM() throws Exception {
writeToHostsFile("host2");
Configuration conf = new Configuration();
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile.getAbsolutePath());
conf.set(YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, "EqualToRM");
rm = new MockRM(conf);
rm.start();
String nmVersion = "1.9.9";
ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
RegisterNodeManagerRequest req = Records.newRecord(RegisterNodeManagerRequest.class);
NodeId nodeId = NodeId.newInstance("host2", 1234);
Resource capability = BuilderUtils.newResource(1024, 1);
req.setResource(capability);
req.setNodeId(nodeId);
req.setHttpPort(1234);
req.setNMVersion(nmVersion);
// trying to register a invalid node.
RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
Assert.assertEquals(NodeAction.SHUTDOWN, response.getNodeAction());
Assert.assertTrue(
"Diagnostic message did not contain: 'Disallowed NodeManager "
+ "Version "
+ nmVersion
+ ", is less than the minimum version'",
response
.getDiagnosticsMessage()
.contains(
"Disallowed NodeManager Version "
+ nmVersion
+ ", is less than the minimum version "));
}
@Override
public Token getRMDelegationToken(Text renewer) throws YarnException, IOException {
/* get the token from RM */
GetDelegationTokenRequest rmDTRequest = Records.newRecord(GetDelegationTokenRequest.class);
rmDTRequest.setRenewer(renewer.toString());
GetDelegationTokenResponse response = rmClient.getDelegationToken(rmDTRequest);
return response.getRMDelegationToken();
}
@Override
public YarnClientApplication createApplication() throws YarnException, IOException {
ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class);
GetNewApplicationResponse newApp = getNewApplication();
ApplicationId appId = newApp.getApplicationId();
context.setApplicationId(appId);
return new YarnClientApplication(newApp, context);
}
@Public
@Unstable
public static ApplicationAttemptId newInstance(ApplicationId appId, int attemptId) {
ApplicationAttemptId appAttemptId = Records.newRecord(ApplicationAttemptId.class);
appAttemptId.setApplicationId(appId);
appAttemptId.setAttemptId(attemptId);
appAttemptId.build();
return appAttemptId;
}
public List<NodeReport> getClusterNodes() throws YarnRemoteException {
if (clientResourceManager == null)
throw new IllegalArgumentException("Can't get report without connecting first!");
GetClusterNodesRequest req = Records.newRecord(GetClusterNodesRequest.class);
GetClusterNodesResponse res = clientResourceManager.getClusterNodes(req);
return res.getNodeReports();
}
/**
* Changed the return type to AllocateResponse which use to hold a reference to AMResponse.
*
* <p>AMResponse seems to have disappeared in CDH 4.6
*
* @param requestedContainers
* @param releasedContainers
* @return
* @throws YarnRemoteException
*/
public AllocateResponse allocateRequest(
List<ResourceRequest> requestedContainers, List<ContainerId> releasedContainers)
throws YarnRemoteException {
if (amResourceManager == null)
throw new IllegalStateException(
"Cannot send allocation request before connecting to the resource manager!");
LOG.info(
"Sending allocation request"
+ ", requestedSize="
+ requestedContainers.size()
+ ", releasedSize="
+ releasedContainers.size());
for (ResourceRequest req : requestedContainers)
LOG.info(
"Requesting container, host="
+ req.getHostName()
+ ", amount="
+ req.getNumContainers()
+ ", memory="
+ req.getCapability().getMemory()
+ ", priority="
+ req.getPriority().getPriority());
for (ContainerId rel : releasedContainers) LOG.info("Releasing container: " + rel.getId());
AllocateRequest request = Records.newRecord(AllocateRequest.class);
request.setResponseId(rmRequestId.incrementAndGet());
request.setApplicationAttemptId(appAttemptId);
request.addAllAsks(requestedContainers);
request.addAllReleases(releasedContainers);
AllocateResponse response = amResourceManager.allocate(request);
// response.getAllocatedContainers()
LOG.debug(
"Got an allocation response, "
+ ", responseId="
+ response.getResponseId()
+ ", numClusterNodes="
+ response.getNumClusterNodes()
+ ", headroom="
+ response.getAvailableResources().getMemory()
+ ", allocatedSize="
+ response.getAllocatedContainers().size()
+ ", updatedNodes="
+ response.getUpdatedNodes().size()
+ ", reboot="
+ response.getReboot()
+ ", completedSize="
+ response.getCompletedContainersStatuses().size());
return response;
}
@Public
@Stable
public static ResourceBlacklistRequest newInstance(
List<String> additions, List<String> removals) {
ResourceBlacklistRequest blacklistRequest = Records.newRecord(ResourceBlacklistRequest.class);
blacklistRequest.setBlacklistAdditions(additions);
blacklistRequest.setBlacklistRemovals(removals);
return blacklistRequest;
}
@Private
@Unstable
public static PreemptionContract newInstance(
List<PreemptionResourceRequest> req, Set<PreemptionContainer> containers) {
PreemptionContract contract = Records.newRecord(PreemptionContract.class);
contract.setResourceRequest(req);
contract.setContainers(containers);
return contract;
}
/**
* Setup the request that will be sent to the RM for the container ask.
*
* @return the setup ResourceRequest to be sent to RM
*/
private ContainerRequest setupContainerAskForRM() {
// setup requirements for hosts
// using * as any host will do for the distributed shell app
// set the priority for the request
Priority pri = Records.newRecord(Priority.class);
// TODO - what is the range for priority? how to decide?
pri.setPriority(requestPriority);
// Set up resource type requirements
// For now, memory and CPU are supported so we set memory and cpu requirements
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(containerMemory);
capability.setVirtualCores(containerVirtualCores);
ContainerRequest request = new ContainerRequest(capability, null, null, pri);
LOG.info("Requested container ask: " + request.toString());
return request;
}
@SuppressWarnings("unchecked")
private NodeStatus getNodeStatusObject(NodeId nodeId) {
NodeStatus status = Records.newRecord(NodeStatus.class);
status.setNodeId(nodeId);
status.setResponseId(0);
status.setContainersStatuses(Collections.EMPTY_LIST);
status.setKeepAliveApplications(Collections.EMPTY_LIST);
return status;
}
@Public
@Unstable
public static GetApplicationAttemptReportResponse newInstance(
ApplicationAttemptReport ApplicationAttemptReport) {
GetApplicationAttemptReportResponse response =
Records.newRecord(GetApplicationAttemptReportResponse.class);
response.setApplicationAttemptReport(ApplicationAttemptReport);
return response;
}
