private void replaceBadHosts(int expectedNumHosts) throws Exception {
Set<Host> goodHosts = Sets.newHashSet();
for (HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) {
if (hostExecutor.isBad()) {
logger.info("Removing host during execution phase: " + hostExecutor.getHost());
executionContext.addBadHost(hostExecutor.getHost());
hostExecutors.remove(hostExecutor);
} else {
goodHosts.add(hostExecutor.getHost());
}
}
long start = System.currentTimeMillis();
while (hostExecutors.size() < expectedNumHosts) {
if (System.currentTimeMillis() - start > FOUR_HOURS) {
throw new RuntimeException(
"Waited over fours for hosts, still have only "
+ hostExecutors.size()
+ " hosts out of an expected "
+ expectedNumHosts);
}
logger.warn(
"Only "
+ hostExecutors.size()
+ " hosts out of an expected "
+ expectedNumHosts
+ ", attempting to replace bad hosts");
TimeUnit.MINUTES.sleep(1);
executionContext.replaceBadHosts();
for (Host host : executionContext.getHosts()) {
if (!goodHosts.contains(host)) {
HostExecutor hostExecutor = hostExecutorBuilder.build(host);
initalizeHost(hostExecutor);
if (hostExecutor.isBad()) {
executionContext.addBadHost(hostExecutor.getHost());
} else {
logger.info("Adding new host during execution phase: " + host);
hostExecutors.add(hostExecutor);
}
}
}
}
}
public PTest(
final TestConfiguration configuration,
final ExecutionContext executionContext,
final String buildTag,
final File logDir,
final LocalCommandFactory localCommandFactory,
final SSHCommandExecutor sshCommandExecutor,
final RSyncCommandExecutor rsyncCommandExecutor,
final Logger logger)
throws Exception {
mConfiguration = configuration;
mLogger = logger;
mBuildTag = buildTag;
mExecutedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
mFailedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
mExecutionContext = executionContext;
mSshCommandExecutor = sshCommandExecutor;
mRsyncCommandExecutor = rsyncCommandExecutor;
mExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool());
final File failedLogDir = Dirs.create(new File(logDir, "failed"));
final File succeededLogDir = Dirs.create(new File(logDir, "succeeded"));
final File scratchDir =
Dirs.createEmpty(new File(mExecutionContext.getLocalWorkingDirectory(), "scratch"));
File patchDir = Dirs.createEmpty(new File(logDir, "patches"));
File patchFile = null;
if (!configuration.getPatch().isEmpty()) {
patchFile = new File(patchDir, buildTag + ".patch");
Files.write(Resources.toByteArray(new URL(configuration.getPatch())), patchFile);
}
ImmutableMap.Builder<String, String> templateDefaultsBuilder = ImmutableMap.builder();
templateDefaultsBuilder
.put("repository", configuration.getRepository())
.put("repositoryName", configuration.getRepositoryName())
.put("repositoryType", configuration.getRepositoryType())
.put("buildTool", configuration.getBuildTool())
.put("branch", configuration.getBranch())
.put("clearLibraryCache", String.valueOf(configuration.isClearLibraryCache()))
.put("workingDir", mExecutionContext.getLocalWorkingDirectory())
.put("buildTag", buildTag)
.put("logDir", logDir.getAbsolutePath())
.put("javaHome", configuration.getJavaHome())
.put("javaHomeForTests", configuration.getJavaHomeForTests())
.put("antEnvOpts", configuration.getAntEnvOpts())
.put("antArgs", configuration.getAntArgs())
.put("antTestArgs", configuration.getAntTestArgs())
.put("antTestTarget", configuration.getAntTestTarget())
.put("mavenEnvOpts", configuration.getMavenEnvOpts())
.put("mavenArgs", configuration.getMavenArgs())
.put("mavenBuildArgs", configuration.getMavenBuildArgs())
.put("mavenTestArgs", configuration.getMavenTestArgs());
final ImmutableMap<String, String> templateDefaults = templateDefaultsBuilder.build();
TestParser testParser =
new TestParser(
configuration.getContext(),
configuration.getTestCasePropertyName(),
new File(
mExecutionContext.getLocalWorkingDirectory(),
configuration.getRepositoryName() + "-source"),
logger);
HostExecutorBuilder hostExecutorBuilder =
new HostExecutorBuilder() {
@Override
public HostExecutor build(Host host) {
return new HostExecutor(
host,
executionContext.getPrivateKey(),
mExecutor,
sshCommandExecutor,
rsyncCommandExecutor,
templateDefaults,
scratchDir,
succeededLogDir,
failedLogDir,
10,
logger);
}
};
List<HostExecutor> hostExecutors = new ArrayList<HostExecutor>();
for (Host host : mExecutionContext.getHosts()) {
hostExecutors.add(hostExecutorBuilder.build(host));
}
mHostExecutors = new CopyOnWriteArrayList<HostExecutor>(hostExecutors);
mPhases = Lists.newArrayList();
mPhases.add(
new PrepPhase(
mHostExecutors, localCommandFactory, templateDefaults, scratchDir, patchFile, logger));
mPhases.add(
new ExecutionPhase(
mHostExecutors,
mExecutionContext,
hostExecutorBuilder,
localCommandFactory,
templateDefaults,
succeededLogDir,
failedLogDir,
testParser.parse(),
mExecutedTests,
mFailedTests,
logger));
mPhases.add(new ReportingPhase(mHostExecutors, localCommandFactory, templateDefaults, logger));
}