public static Resource newResource(int memory, int vCores, int gCores) {
Resource resource = recordFactory.newRecordInstance(Resource.class);
resource.setMemory(memory);
resource.setVirtualCores(vCores);
resource.setGPUCores(gCores);
return resource;
}
@Test
public void testNodeRegistrationWithMinimumAllocations() throws Exception {
Configuration conf = new Configuration();
conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048");
conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4");
rm = new MockRM(conf);
rm.start();
ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
RegisterNodeManagerRequest req = Records.newRecord(RegisterNodeManagerRequest.class);
NodeId nodeId = BuilderUtils.newNodeId("host", 1234);
req.setNodeId(nodeId);
Resource capability = BuilderUtils.newResource(1024, 1);
req.setResource(capability);
RegisterNodeManagerResponse response1 = resourceTrackerService.registerNodeManager(req);
Assert.assertEquals(NodeAction.SHUTDOWN, response1.getNodeAction());
capability.setMemory(2048);
capability.setVirtualCores(1);
req.setResource(capability);
RegisterNodeManagerResponse response2 = resourceTrackerService.registerNodeManager(req);
Assert.assertEquals(NodeAction.SHUTDOWN, response2.getNodeAction());
capability.setMemory(1024);
capability.setVirtualCores(4);
req.setResource(capability);
RegisterNodeManagerResponse response3 = resourceTrackerService.registerNodeManager(req);
Assert.assertEquals(NodeAction.SHUTDOWN, response3.getNodeAction());
capability.setMemory(2048);
capability.setVirtualCores(4);
req.setResource(capability);
RegisterNodeManagerResponse response4 = resourceTrackerService.registerNodeManager(req);
Assert.assertEquals(NodeAction.NORMAL, response4.getNodeAction());
}
public void runMainLoop() throws Exception {
rmClient.registerApplicationMaster("", 0, "");
initRegistry();
startMasterNode();
Priority priority = Records.newRecord(Priority.class);
priority.setPriority(0);
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(ClusterConf.get().getWorkerHeapSize());
capability.setVirtualCores(ClusterConf.get().getNumWorkerThreads());
List<ContainerRequest> containerReq = new ArrayList<ContainerRequest>();
for (int i = 0; i < ClusterConf.get().getNumWorkers(); ++i) {
ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
rmClient.addContainerRequest(containerAsk);
containerReq.add(containerAsk);
}
while (alive) {
Thread.sleep(1000);
}
finish();
/*
for (ContainerRequest req : containerReq) {
rmClient.removeContainerRequest(req);
}
int containersToAdd = 2;
numContainersToWaitFor = containersToAdd;
System.out.println("[Am] finished all containers. Asking for more containers, total=" + numContainersToWaitFor);
for (int i = 0; i < containersToAdd; ++i) {
ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
System.out.println("[AM] Making res-req " + i);
rmClient.addContainerRequest(containerAsk);
}
System.out.println("[AM] waiting for containers to finish once more!!!");
while (!doneWithContainers()) {
Thread.sleep(100);
}
*/
}
/**
* Setup the request that will be sent to the RM for the container ask.
*
* @return the setup ResourceRequest to be sent to RM
*/
private ContainerRequest setupContainerAskForRM() {
// setup requirements for hosts
// using * as any host will do for the distributed shell app
// set the priority for the request
Priority pri = Records.newRecord(Priority.class);
// TODO - what is the range for priority? how to decide?
pri.setPriority(requestPriority);
// Set up resource type requirements
// For now, memory and CPU are supported so we set memory and cpu requirements
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(containerMemory);
capability.setVirtualCores(containerVirtualCores);
ContainerRequest request = new ContainerRequest(capability, null, null, pri);
LOG.info("Requested container ask: " + request.toString());
return request;
}
private void requestContainer(Optional<String> preferredNode) {
Priority priority = Records.newRecord(Priority.class);
priority.setPriority(0);
Resource capability = Records.newRecord(Resource.class);
int maxMemoryCapacity = this.maxResourceCapacity.get().getMemory();
capability.setMemory(
this.requestedContainerMemoryMbs <= maxMemoryCapacity
? this.requestedContainerMemoryMbs
: maxMemoryCapacity);
int maxCoreCapacity = this.maxResourceCapacity.get().getVirtualCores();
capability.setVirtualCores(
this.requestedContainerCores <= maxCoreCapacity
? this.requestedContainerCores
: maxCoreCapacity);
String[] preferredNodes = preferredNode.isPresent() ? new String[] {preferredNode.get()} : null;
this.amrmClientAsync.addContainerRequest(
new AMRMClient.ContainerRequest(capability, preferredNodes, null, priority));
}
public void run(String[] args) throws Exception {
final String command = args[0];
final int n = Integer.valueOf(args[1]);
final Path jarPath = new Path(args[2]);
// Create yarnClient
YarnConfiguration conf = new YarnConfiguration();
YarnClient yarnClient = YarnClient.createYarnClient();
yarnClient.init(conf);
yarnClient.start();
// Create application via yarnClient
YarnClientApplication app = yarnClient.createApplication();
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
amContainer.setCommands(
Collections.singletonList(
"$JAVA_HOME/bin/java"
+ " -Xmx256M"
+ " com.hortonworks.simpleyarnapp.ApplicationMaster"
+ " "
+ command
+ " "
+ String.valueOf(n)
+ " 1>"
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR
+ "/stdout"
+ " 2>"
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR
+ "/stderr"));
// Setup jar for ApplicationMaster
LocalResource appMasterJar = Records.newRecord(LocalResource.class);
setupAppMasterJar(jarPath, appMasterJar);
System.out.println("Jar name is " + jarPath.getName());
amContainer.setLocalResources(Collections.singletonMap(jarPath.getName(), appMasterJar));
// Setup CLASSPATH for ApplicationMaster
Map<String, String> appMasterEnv = new HashMap<String, String>();
setupAppMasterEnv(appMasterEnv);
amContainer.setEnvironment(appMasterEnv);
// Set up resource type requirements for ApplicationMaster
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(256);
capability.setVirtualCores(1);
// Finally, set-up ApplicationSubmissionContext for the application
ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
appContext.setApplicationName("apache-yarn-example"); // application name
appContext.setAMContainerSpec(amContainer);
appContext.setResource(capability);
appContext.setQueue("default"); // queue
// Submit application
ApplicationId appId = appContext.getApplicationId();
System.out.println("Submitting application " + appId);
yarnClient.submitApplication(appContext);
ApplicationReport appReport = yarnClient.getApplicationReport(appId);
YarnApplicationState appState = appReport.getYarnApplicationState();
while (appState != YarnApplicationState.FINISHED
&& appState != YarnApplicationState.KILLED
&& appState != YarnApplicationState.FAILED) {
System.out.println("App Status = " + appState);
Thread.sleep(100);
appReport = yarnClient.getApplicationReport(appId);
appState = appReport.getYarnApplicationState();
}
System.out.println(
"Application "
+ appId
+ " finished with"
+ " state "
+ appState
+ " at "
+ appReport.getFinishTime());
}
public void clearPreemptedResources() {
preemptedResources.setMemory(0);
preemptedResources.setVirtualCores(0);
}