private void replaceBadHosts(int expectedNumHosts) throws Exception { Set<Host> goodHosts = Sets.newHashSet(); for (HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) { if (hostExecutor.isBad()) { logger.info("Removing host during execution phase: " + hostExecutor.getHost()); executionContext.addBadHost(hostExecutor.getHost()); hostExecutors.remove(hostExecutor); } else { goodHosts.add(hostExecutor.getHost()); } } long start = System.currentTimeMillis(); while (hostExecutors.size() < expectedNumHosts) { if (System.currentTimeMillis() - start > FOUR_HOURS) { throw new RuntimeException( "Waited over fours for hosts, still have only " + hostExecutors.size() + " hosts out of an expected " + expectedNumHosts); } logger.warn( "Only " + hostExecutors.size() + " hosts out of an expected " + expectedNumHosts + ", attempting to replace bad hosts"); TimeUnit.MINUTES.sleep(1); executionContext.replaceBadHosts(); for (Host host : executionContext.getHosts()) { if (!goodHosts.contains(host)) { HostExecutor hostExecutor = hostExecutorBuilder.build(host); initalizeHost(hostExecutor); if (hostExecutor.isBad()) { executionContext.addBadHost(hostExecutor.getHost()); } else { logger.info("Adding new host during execution phase: " + host); hostExecutors.add(hostExecutor); } } } } }
public PTest( final TestConfiguration configuration, final ExecutionContext executionContext, final String buildTag, final File logDir, final LocalCommandFactory localCommandFactory, final SSHCommandExecutor sshCommandExecutor, final RSyncCommandExecutor rsyncCommandExecutor, final Logger logger) throws Exception { mConfiguration = configuration; mLogger = logger; mBuildTag = buildTag; mExecutedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>()); mFailedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>()); mExecutionContext = executionContext; mSshCommandExecutor = sshCommandExecutor; mRsyncCommandExecutor = rsyncCommandExecutor; mExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); final File failedLogDir = Dirs.create(new File(logDir, "failed")); final File succeededLogDir = Dirs.create(new File(logDir, "succeeded")); final File scratchDir = Dirs.createEmpty(new File(mExecutionContext.getLocalWorkingDirectory(), "scratch")); File patchDir = Dirs.createEmpty(new File(logDir, "patches")); File patchFile = null; if (!configuration.getPatch().isEmpty()) { patchFile = new File(patchDir, buildTag + ".patch"); Files.write(Resources.toByteArray(new URL(configuration.getPatch())), patchFile); } ImmutableMap.Builder<String, String> templateDefaultsBuilder = ImmutableMap.builder(); templateDefaultsBuilder .put("repository", configuration.getRepository()) .put("repositoryName", configuration.getRepositoryName()) .put("repositoryType", configuration.getRepositoryType()) .put("buildTool", configuration.getBuildTool()) .put("branch", configuration.getBranch()) .put("clearLibraryCache", String.valueOf(configuration.isClearLibraryCache())) .put("workingDir", mExecutionContext.getLocalWorkingDirectory()) .put("buildTag", buildTag) .put("logDir", logDir.getAbsolutePath()) .put("javaHome", configuration.getJavaHome()) .put("javaHomeForTests", configuration.getJavaHomeForTests()) .put("antEnvOpts", configuration.getAntEnvOpts()) .put("antArgs", configuration.getAntArgs()) .put("antTestArgs", configuration.getAntTestArgs()) .put("antTestTarget", configuration.getAntTestTarget()) .put("mavenEnvOpts", configuration.getMavenEnvOpts()) .put("mavenArgs", configuration.getMavenArgs()) .put("mavenBuildArgs", configuration.getMavenBuildArgs()) .put("mavenTestArgs", configuration.getMavenTestArgs()); final ImmutableMap<String, String> templateDefaults = templateDefaultsBuilder.build(); TestParser testParser = new TestParser( configuration.getContext(), configuration.getTestCasePropertyName(), new File( mExecutionContext.getLocalWorkingDirectory(), configuration.getRepositoryName() + "-source"), logger); HostExecutorBuilder hostExecutorBuilder = new HostExecutorBuilder() { @Override public HostExecutor build(Host host) { return new HostExecutor( host, executionContext.getPrivateKey(), mExecutor, sshCommandExecutor, rsyncCommandExecutor, templateDefaults, scratchDir, succeededLogDir, failedLogDir, 10, logger); } }; List<HostExecutor> hostExecutors = new ArrayList<HostExecutor>(); for (Host host : mExecutionContext.getHosts()) { hostExecutors.add(hostExecutorBuilder.build(host)); } mHostExecutors = new CopyOnWriteArrayList<HostExecutor>(hostExecutors); mPhases = Lists.newArrayList(); mPhases.add( new PrepPhase( mHostExecutors, localCommandFactory, templateDefaults, scratchDir, patchFile, logger)); mPhases.add( new ExecutionPhase( mHostExecutors, mExecutionContext, hostExecutorBuilder, localCommandFactory, templateDefaults, succeededLogDir, failedLogDir, testParser.parse(), mExecutedTests, mFailedTests, logger)); mPhases.add(new ReportingPhase(mHostExecutors, localCommandFactory, templateDefaults, logger)); }