private void replaceBadHosts(int expectedNumHosts) throws Exception {
   Set<Host> goodHosts = Sets.newHashSet();
   for (HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) {
     if (hostExecutor.isBad()) {
       logger.info("Removing host during execution phase: " + hostExecutor.getHost());
       executionContext.addBadHost(hostExecutor.getHost());
       hostExecutors.remove(hostExecutor);
     } else {
       goodHosts.add(hostExecutor.getHost());
     }
   }
   long start = System.currentTimeMillis();
   while (hostExecutors.size() < expectedNumHosts) {
     if (System.currentTimeMillis() - start > FOUR_HOURS) {
       throw new RuntimeException(
           "Waited over fours for hosts, still have only "
               + hostExecutors.size()
               + " hosts out of an expected "
               + expectedNumHosts);
     }
     logger.warn(
         "Only "
             + hostExecutors.size()
             + " hosts out of an expected "
             + expectedNumHosts
             + ", attempting to replace bad hosts");
     TimeUnit.MINUTES.sleep(1);
     executionContext.replaceBadHosts();
     for (Host host : executionContext.getHosts()) {
       if (!goodHosts.contains(host)) {
         HostExecutor hostExecutor = hostExecutorBuilder.build(host);
         initalizeHost(hostExecutor);
         if (hostExecutor.isBad()) {
           executionContext.addBadHost(hostExecutor.getHost());
         } else {
           logger.info("Adding new host during execution phase: " + host);
           hostExecutors.add(hostExecutor);
         }
       }
     }
   }
 }
 public static void main(String[] args) throws Exception {
   LOG.info("Args " + Arrays.toString(args));
   CommandLineParser parser = new GnuParser();
   Options options = new Options();
   options.addOption(null, PROPERTIES, true, "properties file");
   options.addOption(null, REPOSITORY, true, "Overrides git repository in properties file");
   options.addOption(
       null, REPOSITORY_NAME, true, "Overrides git repository *name* in properties file");
   options.addOption(null, BRANCH, true, "Overrides git branch in properties file");
   options.addOption(null, PATCH, true, "URI to patch, either file:/// or http(s)://");
   options.addOption(ANT_ARG, null, true, "Supplemntal ant arguments");
   options.addOption(
       null,
       JAVA_HOME,
       true,
       "Java Home for compiling and running tests (unless " + JAVA_HOME_TEST + " is specified)");
   options.addOption(null, JAVA_HOME_TEST, true, "Java Home for running tests (optional)");
   options.addOption(null, ANT_TEST_ARGS, true, "Arguments to ant test on slave nodes only");
   options.addOption(null, ANT_ENV_OPTS, true, "ANT_OPTS environment variable setting");
   CommandLine commandLine = parser.parse(options, args);
   if (!commandLine.hasOption(PROPERTIES)) {
     throw new IllegalArgumentException(
         Joiner.on(" ").join(PTest.class.getName(), "--" + PROPERTIES, "config.properties"));
   }
   String testConfigurationFile = commandLine.getOptionValue(PROPERTIES);
   ExecutionContextConfiguration executionContextConfiguration =
       ExecutionContextConfiguration.fromFile(testConfigurationFile);
   String buildTag =
       System.getenv("BUILD_TAG") == null
           ? "undefined-" + System.currentTimeMillis()
           : System.getenv("BUILD_TAG");
   File logDir =
       Dirs.create(new File(executionContextConfiguration.getGlobalLogDirectory(), buildTag));
   LogDirectoryCleaner cleaner =
       new LogDirectoryCleaner(new File(executionContextConfiguration.getGlobalLogDirectory()), 5);
   cleaner.setName("LogCleaner-" + executionContextConfiguration.getGlobalLogDirectory());
   cleaner.setDaemon(true);
   cleaner.start();
   TestConfiguration conf = TestConfiguration.fromFile(testConfigurationFile, LOG);
   String repository = Strings.nullToEmpty(commandLine.getOptionValue(REPOSITORY)).trim();
   if (!repository.isEmpty()) {
     conf.setRepository(repository);
   }
   String repositoryName = Strings.nullToEmpty(commandLine.getOptionValue(REPOSITORY_NAME)).trim();
   if (!repositoryName.isEmpty()) {
     conf.setRepositoryName(repositoryName);
   }
   String branch = Strings.nullToEmpty(commandLine.getOptionValue(BRANCH)).trim();
   if (!branch.isEmpty()) {
     conf.setBranch(branch);
   }
   String patch = Strings.nullToEmpty(commandLine.getOptionValue(PATCH)).trim();
   if (!patch.isEmpty()) {
     conf.setPatch(patch);
   }
   String javaHome = Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME)).trim();
   if (!javaHome.isEmpty()) {
     conf.setJavaHome(javaHome);
   }
   String javaHomeForTests =
       Strings.nullToEmpty(commandLine.getOptionValue(JAVA_HOME_TEST)).trim();
   if (!javaHomeForTests.isEmpty()) {
     conf.setJavaHomeForTests(javaHomeForTests);
   }
   String antTestArgs = Strings.nullToEmpty(commandLine.getOptionValue(ANT_TEST_ARGS)).trim();
   if (!antTestArgs.isEmpty()) {
     conf.setAntTestArgs(antTestArgs);
   }
   String antEnvOpts = Strings.nullToEmpty(commandLine.getOptionValue(ANT_ENV_OPTS)).trim();
   if (!antEnvOpts.isEmpty()) {
     conf.setAntEnvOpts(antEnvOpts);
   }
   String antTestTarget = Strings.nullToEmpty(commandLine.getOptionValue(ANT_TEST_TARGET)).trim();
   if (!antTestTarget.isEmpty()) {
     conf.setAntTestTarget(antTestTarget);
   }
   String[] supplementalAntArgs = commandLine.getOptionValues(ANT_ARG);
   if (supplementalAntArgs != null && supplementalAntArgs.length > 0) {
     String antArgs = Strings.nullToEmpty(conf.getAntArgs());
     if (!(antArgs.isEmpty() || antArgs.endsWith(" "))) {
       antArgs += " ";
     }
     antArgs += "-" + ANT_ARG + Joiner.on(" -" + ANT_ARG).join(supplementalAntArgs);
     conf.setAntArgs(antArgs);
   }
   ExecutionContextProvider executionContextProvider = null;
   ExecutionContext executionContext = null;
   int exitCode = 0;
   try {
     executionContextProvider = executionContextConfiguration.getExecutionContextProvider();
     executionContext = executionContextProvider.createExecutionContext();
     LocalCommandFactory localCommandFactory = new LocalCommandFactory(LOG);
     PTest ptest =
         new PTest(
             conf,
             executionContext,
             buildTag,
             logDir,
             localCommandFactory,
             new SSHCommandExecutor(LOG, localCommandFactory, conf.getSshOpts()),
             new RSyncCommandExecutor(LOG, 10, localCommandFactory),
             LOG);
     exitCode = ptest.run();
   } finally {
     if (executionContext != null) {
       executionContext.terminate();
     }
     if (executionContextProvider != null) {
       executionContextProvider.close();
     }
   }
   System.exit(exitCode);
 }
  public PTest(
      final TestConfiguration configuration,
      final ExecutionContext executionContext,
      final String buildTag,
      final File logDir,
      final LocalCommandFactory localCommandFactory,
      final SSHCommandExecutor sshCommandExecutor,
      final RSyncCommandExecutor rsyncCommandExecutor,
      final Logger logger)
      throws Exception {
    mConfiguration = configuration;
    mLogger = logger;
    mBuildTag = buildTag;
    mExecutedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
    mFailedTests = Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
    mExecutionContext = executionContext;
    mSshCommandExecutor = sshCommandExecutor;
    mRsyncCommandExecutor = rsyncCommandExecutor;
    mExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool());
    final File failedLogDir = Dirs.create(new File(logDir, "failed"));
    final File succeededLogDir = Dirs.create(new File(logDir, "succeeded"));
    final File scratchDir =
        Dirs.createEmpty(new File(mExecutionContext.getLocalWorkingDirectory(), "scratch"));
    File patchDir = Dirs.createEmpty(new File(logDir, "patches"));
    File patchFile = null;
    if (!configuration.getPatch().isEmpty()) {
      patchFile = new File(patchDir, buildTag + ".patch");
      Files.write(Resources.toByteArray(new URL(configuration.getPatch())), patchFile);
    }
    ImmutableMap.Builder<String, String> templateDefaultsBuilder = ImmutableMap.builder();
    templateDefaultsBuilder
        .put("repository", configuration.getRepository())
        .put("repositoryName", configuration.getRepositoryName())
        .put("repositoryType", configuration.getRepositoryType())
        .put("buildTool", configuration.getBuildTool())
        .put("branch", configuration.getBranch())
        .put("clearLibraryCache", String.valueOf(configuration.isClearLibraryCache()))
        .put("workingDir", mExecutionContext.getLocalWorkingDirectory())
        .put("buildTag", buildTag)
        .put("logDir", logDir.getAbsolutePath())
        .put("javaHome", configuration.getJavaHome())
        .put("javaHomeForTests", configuration.getJavaHomeForTests())
        .put("antEnvOpts", configuration.getAntEnvOpts())
        .put("antArgs", configuration.getAntArgs())
        .put("antTestArgs", configuration.getAntTestArgs())
        .put("antTestTarget", configuration.getAntTestTarget())
        .put("mavenEnvOpts", configuration.getMavenEnvOpts())
        .put("mavenArgs", configuration.getMavenArgs())
        .put("mavenBuildArgs", configuration.getMavenBuildArgs())
        .put("mavenTestArgs", configuration.getMavenTestArgs());
    final ImmutableMap<String, String> templateDefaults = templateDefaultsBuilder.build();
    TestParser testParser =
        new TestParser(
            configuration.getContext(),
            configuration.getTestCasePropertyName(),
            new File(
                mExecutionContext.getLocalWorkingDirectory(),
                configuration.getRepositoryName() + "-source"),
            logger);

    HostExecutorBuilder hostExecutorBuilder =
        new HostExecutorBuilder() {
          @Override
          public HostExecutor build(Host host) {
            return new HostExecutor(
                host,
                executionContext.getPrivateKey(),
                mExecutor,
                sshCommandExecutor,
                rsyncCommandExecutor,
                templateDefaults,
                scratchDir,
                succeededLogDir,
                failedLogDir,
                10,
                logger);
          }
        };
    List<HostExecutor> hostExecutors = new ArrayList<HostExecutor>();
    for (Host host : mExecutionContext.getHosts()) {
      hostExecutors.add(hostExecutorBuilder.build(host));
    }
    mHostExecutors = new CopyOnWriteArrayList<HostExecutor>(hostExecutors);
    mPhases = Lists.newArrayList();
    mPhases.add(
        new PrepPhase(
            mHostExecutors, localCommandFactory, templateDefaults, scratchDir, patchFile, logger));
    mPhases.add(
        new ExecutionPhase(
            mHostExecutors,
            mExecutionContext,
            hostExecutorBuilder,
            localCommandFactory,
            templateDefaults,
            succeededLogDir,
            failedLogDir,
            testParser.parse(),
            mExecutedTests,
            mFailedTests,
            logger));
    mPhases.add(new ReportingPhase(mHostExecutors, localCommandFactory, templateDefaults, logger));
  }
 public int run() {
   int result = 0;
   boolean error = false;
   List<String> messages = Lists.newArrayList();
   Map<String, Long> elapsedTimes = Maps.newTreeMap();
   try {
     mLogger.info("Running tests with " + mConfiguration);
     for (Phase phase : mPhases) {
       String msg = "Executing " + phase.getClass().getName();
       mLogger.info(msg);
       messages.add(msg);
       long start = System.currentTimeMillis();
       try {
         phase.execute();
       } finally {
         long elapsedTime =
             TimeUnit.MINUTES.convert((System.currentTimeMillis() - start), TimeUnit.MILLISECONDS);
         elapsedTimes.put(phase.getClass().getSimpleName(), elapsedTime);
       }
     }
     if (!mFailedTests.isEmpty()) {
       throw new TestsFailedException(mFailedTests.size() + " tests failed");
     }
   } catch (Throwable throwable) {
     mLogger.error("Test run exited with an unexpected error", throwable);
     // NonZeroExitCodeExceptions can have long messages and should be
     // trimmable when published to the JIRA via the JiraService
     if (throwable instanceof NonZeroExitCodeException) {
       messages.add("Tests exited with: " + throwable.getClass().getSimpleName());
       for (String line : Strings.nullToEmpty(throwable.getMessage()).split("\n")) {
         messages.add(line);
       }
     } else {
       messages.add(
           "Tests exited with: "
               + throwable.getClass().getSimpleName()
               + ": "
               + throwable.getMessage());
     }
     error = true;
   } finally {
     for (HostExecutor hostExecutor : mHostExecutors) {
       hostExecutor.shutdownNow();
       if (hostExecutor.isBad()) {
         mExecutionContext.addBadHost(hostExecutor.getHost());
       }
     }
     mSshCommandExecutor.shutdownNow();
     mRsyncCommandExecutor.shutdownNow();
     mExecutor.shutdownNow();
     SortedSet<String> failedTests = new TreeSet<String>(mFailedTests);
     if (failedTests.isEmpty()) {
       mLogger.info(String.format("%d failed tests", failedTests.size()));
     } else {
       mLogger.warn(String.format("%d failed tests", failedTests.size()));
     }
     for (String failingTestName : failedTests) {
       mLogger.warn(failingTestName);
     }
     mLogger.info("Executed " + mExecutedTests.size() + " tests");
     for (Map.Entry<String, Long> entry : elapsedTimes.entrySet()) {
       mLogger.info(
           String.format("PERF: Phase %s took %d minutes", entry.getKey(), entry.getValue()));
     }
     publishJiraComment(error, messages, failedTests);
     if (error || !mFailedTests.isEmpty()) {
       result = 1;
     }
   }
   return result;
 }