public void runMainLoop() throws Exception {
rmClient.registerApplicationMaster("", 0, "");
initRegistry();
startMasterNode();
Priority priority = Records.newRecord(Priority.class);
priority.setPriority(0);
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(ClusterConf.get().getWorkerHeapSize());
capability.setVirtualCores(ClusterConf.get().getNumWorkerThreads());
List<ContainerRequest> containerReq = new ArrayList<ContainerRequest>();
for (int i = 0; i < ClusterConf.get().getNumWorkers(); ++i) {
ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
rmClient.addContainerRequest(containerAsk);
containerReq.add(containerAsk);
}
while (alive) {
Thread.sleep(1000);
}
finish();
/*
for (ContainerRequest req : containerReq) {
rmClient.removeContainerRequest(req);
}
int containersToAdd = 2;
numContainersToWaitFor = containersToAdd;
System.out.println("[Am] finished all containers. Asking for more containers, total=" + numContainersToWaitFor);
for (int i = 0; i < containersToAdd; ++i) {
ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
System.out.println("[AM] Making res-req " + i);
rmClient.addContainerRequest(containerAsk);
}
System.out.println("[AM] waiting for containers to finish once more!!!");
while (!doneWithContainers()) {
Thread.sleep(100);
}
*/
}
void launchWorkerNode(Container container) throws IOException, YarnException {
Map<String, String> env = System.getenv();
ContainerLaunchContext workerContext = Records.newRecord(ContainerLaunchContext.class);
LocalResource workerJar = Records.newRecord(LocalResource.class);
setupWorkerJar(workerJar);
workerContext.setLocalResources(Collections.singletonMap("socialite.jar", workerJar));
System.out.println("[Master] workerJar:" + workerJar);
Map<String, String> workerEnv = new HashMap<String, String>(env);
setupWorkerEnv(workerEnv);
workerContext.setEnvironment(workerEnv);
String opts = "-Dsocialite.master=" + NetUtils.getHostname().split("/")[1] + " ";
opts += "-Dsocialite.worker.num_threads=" + ClusterConf.get().getNumWorkerThreads() + " ";
workerContext.setCommands(
Collections.singletonList(
"$JAVA_HOME/bin/java -ea "
+ env.get("JVM_OPTS")
+ " "
+ env.get("SOCIALITE_OPTS")
+ " "
+ opts
+ " "
+ "socialite.dist.worker.WorkerNode"
+ " 1>"
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR
+ "/stdout"
+ " 2>"
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR
+ "/stderr"));
nmClient.startContainer(container, workerContext);
L.info("Launched worker node (container:" + container.getId() + ")");
}
public boolean allContainersAllocated() {
return numAllocedContainers == ClusterConf.get().getNumWorkers();
}