@Override public void execute() throws Throwable { long start = System.currentTimeMillis(); List<TestBatch> testBatches = Lists.newArrayList(); for (TestBatch batch : testBatchSupplier.get()) { testBatches.add(batch); if (batch.isParallel()) { parallelWorkQueue.add(batch); } else { isolatedWorkQueue.add(batch); } } try { int expectedNumHosts = hostExecutors.size(); initalizeHosts(); do { replaceBadHosts(expectedNumHosts); List<ListenableFuture<Void>> results = Lists.newArrayList(); for (HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) { results.add( hostExecutor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults)); } Futures.allAsList(results).get(); } while (!(parallelWorkQueue.isEmpty() && isolatedWorkQueue.isEmpty())); for (TestBatch batch : testBatches) { File batchLogDir; if (failedTestResults.contains(batch)) { batchLogDir = new File(failedLogDir, batch.getName()); } else { batchLogDir = new File(succeededLogDir, batch.getName()); } JUnitReportParser parser = new JUnitReportParser(logger, batchLogDir); executedTests.addAll(parser.getExecutedTests()); failedTests.addAll(parser.getFailedTests()); } } finally { long elapsed = System.currentTimeMillis() - start; logger.info( "PERF: exec phase " + TimeUnit.MINUTES.convert(elapsed, TimeUnit.MILLISECONDS) + " minutes"); } }
private void replaceBadHosts(int expectedNumHosts) throws Exception { Set<Host> goodHosts = Sets.newHashSet(); for (HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) { if (hostExecutor.isBad()) { logger.info("Removing host during execution phase: " + hostExecutor.getHost()); executionContext.addBadHost(hostExecutor.getHost()); hostExecutors.remove(hostExecutor); } else { goodHosts.add(hostExecutor.getHost()); } } long start = System.currentTimeMillis(); while (hostExecutors.size() < expectedNumHosts) { if (System.currentTimeMillis() - start > FOUR_HOURS) { throw new RuntimeException( "Waited over fours for hosts, still have only " + hostExecutors.size() + " hosts out of an expected " + expectedNumHosts); } logger.warn( "Only " + hostExecutors.size() + " hosts out of an expected " + expectedNumHosts + ", attempting to replace bad hosts"); TimeUnit.MINUTES.sleep(1); executionContext.replaceBadHosts(); for (Host host : executionContext.getHosts()) { if (!goodHosts.contains(host)) { HostExecutor hostExecutor = hostExecutorBuilder.build(host); initalizeHost(hostExecutor); if (hostExecutor.isBad()) { executionContext.addBadHost(hostExecutor.getHost()); } else { logger.info("Adding new host during execution phase: " + host); hostExecutors.add(hostExecutor); } } } } }
public int run() { int result = 0; boolean error = false; List<String> messages = Lists.newArrayList(); Map<String, Long> elapsedTimes = Maps.newTreeMap(); try { mLogger.info("Running tests with " + mConfiguration); for (Phase phase : mPhases) { String msg = "Executing " + phase.getClass().getName(); mLogger.info(msg); messages.add(msg); long start = System.currentTimeMillis(); try { phase.execute(); } finally { long elapsedTime = TimeUnit.MINUTES.convert((System.currentTimeMillis() - start), TimeUnit.MILLISECONDS); elapsedTimes.put(phase.getClass().getSimpleName(), elapsedTime); } } if (!mFailedTests.isEmpty()) { throw new TestsFailedException(mFailedTests.size() + " tests failed"); } } catch (Throwable throwable) { mLogger.error("Test run exited with an unexpected error", throwable); // NonZeroExitCodeExceptions can have long messages and should be // trimmable when published to the JIRA via the JiraService if (throwable instanceof NonZeroExitCodeException) { messages.add("Tests exited with: " + throwable.getClass().getSimpleName()); for (String line : Strings.nullToEmpty(throwable.getMessage()).split("\n")) { messages.add(line); } } else { messages.add( "Tests exited with: " + throwable.getClass().getSimpleName() + ": " + throwable.getMessage()); } error = true; } finally { for (HostExecutor hostExecutor : mHostExecutors) { hostExecutor.shutdownNow(); if (hostExecutor.isBad()) { mExecutionContext.addBadHost(hostExecutor.getHost()); } } mSshCommandExecutor.shutdownNow(); mRsyncCommandExecutor.shutdownNow(); mExecutor.shutdownNow(); SortedSet<String> failedTests = new TreeSet<String>(mFailedTests); if (failedTests.isEmpty()) { mLogger.info(String.format("%d failed tests", failedTests.size())); } else { mLogger.warn(String.format("%d failed tests", failedTests.size())); } for (String failingTestName : failedTests) { mLogger.warn(failingTestName); } mLogger.info("Executed " + mExecutedTests.size() + " tests"); for (Map.Entry<String, Long> entry : elapsedTimes.entrySet()) { mLogger.info( String.format("PERF: Phase %s took %d minutes", entry.getKey(), entry.getValue())); } publishJiraComment(error, messages, failedTests); if (error || !mFailedTests.isEmpty()) { result = 1; } } return result; }