private void waitForWorkersStartup(WorkerJvm worker, int workerTimeoutSec) {
int loopCount =
(int) TimeUnit.SECONDS.toMillis(workerTimeoutSec) / WAIT_FOR_WORKER_STARTUP_INTERVAL_MILLIS;
for (int i = 0; i < loopCount; i++) {
if (hasExited(worker)) {
throw new SpawnWorkerFailedException(
format(
"Startup of Worker on host %s failed, check log files in %s for more information!",
agent.getPublicAddress(), worker.getWorkerHome()));
}
String address = readAddress(worker);
if (address != null) {
worker.setHzAddress(address);
LOGGER.info(format("Worker %s started", worker.getId()));
return;
}
sleepMillis(WAIT_FOR_WORKER_STARTUP_INTERVAL_MILLIS);
}
throw new SpawnWorkerFailedException(
format(
"Worker %s of Testsuite %s on Agent %s didn't start within %s seconds",
worker.getId(),
agent.getTestSuite().getId(),
agent.getPublicAddress(),
workerTimeoutSec));
}
private WorkerJvm startWorkerJvm() throws IOException {
int workerIndex = workerJvmSettings.getWorkerIndex();
WorkerType type = workerJvmSettings.getWorkerType();
SimulatorAddress workerAddress =
new SimulatorAddress(AddressLevel.WORKER, agent.getAddressIndex(), workerIndex, 0);
String workerId =
"worker-" + agent.getPublicAddress() + '-' + workerIndex + '-' + type.toLowerCase();
File workerHome = new File(testSuiteDir, workerId);
ensureExistingDirectory(workerHome);
WorkerJvm workerJvm = new WorkerJvm(workerAddress, workerId, workerHome);
generateWorkerStartScript(type, workerJvm);
ProcessBuilder processBuilder =
new ProcessBuilder(new String[] {"bash", "worker.sh"})
.directory(workerHome)
.redirectErrorStream(true);
Map<String, String> environment = processBuilder.environment();
String javaHome = getJavaHome();
String path = javaHome + File.pathSeparator + "bin:" + environment.get("PATH");
environment.put("PATH", path);
environment.put("JAVA_HOME", javaHome);
Process process = processBuilder.start();
workerJvm.setProcess(process);
copyResourcesToWorkerId(workerId);
workerJvmManager.add(workerAddress, workerJvm);
return workerJvm;
}
private void addProfilerSettings(WorkerJvm workerJvm, List<String> args) {
String javaExecutable = "java";
switch (workerJvmSettings.getProfiler()) {
case YOURKIT:
args.add(javaExecutable);
String agentSetting =
workerJvmSettings
.getProfilerSettings()
.replace("${SIMULATOR_HOME}", getSimulatorHome().getAbsolutePath())
.replace("${WORKER_HOME}", workerJvm.getWorkerHome().getAbsolutePath());
args.add(agentSetting);
break;
case FLIGHTRECORDER:
case HPROF:
args.add(javaExecutable);
args.add(workerJvmSettings.getProfilerSettings());
break;
case PERF:
case VTUNE:
// perf and vtune command always need to be in front of the java command
args.add(workerJvmSettings.getProfilerSettings());
args.add(javaExecutable);
break;
default:
args.add(javaExecutable);
}
}
private String[] buildArgs(WorkerJvm workerJvm, WorkerType type) {
List<String> args = new LinkedList<String>();
int workerIndex = workerJvmSettings.getWorkerIndex();
int workerPort = agent.getPort() + workerIndex;
addNumaCtlSettings(args);
addProfilerSettings(workerJvm, args);
args.add("-classpath");
args.add(getClasspath());
args.addAll(getJvmOptions());
args.add("-XX:OnOutOfMemoryError=\"touch worker.oome\"");
args.add("-Dhazelcast.logging.type=log4j");
args.add("-Dlog4j.configuration=file:" + log4jFile.getAbsolutePath());
args.add("-DSIMULATOR_HOME=" + getSimulatorHome());
args.add("-DworkerId=" + workerJvm.getId());
args.add("-DworkerType=" + type);
args.add("-DpublicAddress=" + agent.getPublicAddress());
args.add("-DagentIndex=" + agent.getAddressIndex());
args.add("-DworkerIndex=" + workerIndex);
args.add("-DworkerPort=" + workerPort);
args.add("-DautoCreateHzInstance=" + workerJvmSettings.isAutoCreateHzInstance());
args.add(
"-DworkerPerformanceMonitorIntervalSeconds="
+ workerJvmSettings.getWorkerPerformanceMonitorIntervalSeconds());
args.add("-DhzConfigFile=" + hzConfigFile.getAbsolutePath());
// add class name to start correct worker type
args.add(type.getClassName());
return args.toArray(new String[args.size()]);
}
private boolean hasExited(WorkerJvm workerJvm) {
try {
workerJvm.getProcess().exitValue();
return true;
} catch (IllegalThreadStateException e) {
return false;
}
}
private String readAddress(WorkerJvm jvm) {
File file = new File(jvm.getWorkerHome(), "worker.address");
if (!file.exists()) {
return null;
}
String address = readObject(file);
deleteQuiet(file);
return address;
}
private void generateWorkerStartScript(WorkerType type, WorkerJvm workerJvm) {
String[] args = buildArgs(workerJvm, type);
File startScript = new File(workerJvm.getWorkerHome(), "worker.sh");
StringBuilder sb = new StringBuilder();
sb.append("#!/bin/bash").append(NEW_LINE);
for (String arg : args) {
sb.append(arg).append(' ');
}
sb.append("> worker.out 2> worker.err").append(NEW_LINE);
writeText(sb.toString(), startScript);
}
