@Override
public void execute() throws Throwable {
long start = System.currentTimeMillis();
List<TestBatch> testBatches = Lists.newArrayList();
for (TestBatch batch : testBatchSupplier.get()) {
testBatches.add(batch);
if (batch.isParallel()) {
parallelWorkQueue.add(batch);
} else {
isolatedWorkQueue.add(batch);
}
}
try {
int expectedNumHosts = hostExecutors.size();
initalizeHosts();
do {
replaceBadHosts(expectedNumHosts);
List<ListenableFuture<Void>> results = Lists.newArrayList();
for (HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) {
results.add(
hostExecutor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults));
}
Futures.allAsList(results).get();
} while (!(parallelWorkQueue.isEmpty() && isolatedWorkQueue.isEmpty()));
for (TestBatch batch : testBatches) {
File batchLogDir;
if (failedTestResults.contains(batch)) {
batchLogDir = new File(failedLogDir, batch.getName());
} else {
batchLogDir = new File(succeededLogDir, batch.getName());
}
JUnitReportParser parser = new JUnitReportParser(logger, batchLogDir);
executedTests.addAll(parser.getExecutedTests());
failedTests.addAll(parser.getFailedTests());
}
} finally {
long elapsed = System.currentTimeMillis() - start;
logger.info(
"PERF: exec phase "
+ TimeUnit.MINUTES.convert(elapsed, TimeUnit.MILLISECONDS)
+ " minutes");
}
}
private void replaceBadHosts(int expectedNumHosts) throws Exception {
Set<Host> goodHosts = Sets.newHashSet();
for (HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) {
if (hostExecutor.isBad()) {
logger.info("Removing host during execution phase: " + hostExecutor.getHost());
executionContext.addBadHost(hostExecutor.getHost());
hostExecutors.remove(hostExecutor);
} else {
goodHosts.add(hostExecutor.getHost());
}
}
long start = System.currentTimeMillis();
while (hostExecutors.size() < expectedNumHosts) {
if (System.currentTimeMillis() - start > FOUR_HOURS) {
throw new RuntimeException(
"Waited over fours for hosts, still have only "
+ hostExecutors.size()
+ " hosts out of an expected "
+ expectedNumHosts);
}
logger.warn(
"Only "
+ hostExecutors.size()
+ " hosts out of an expected "
+ expectedNumHosts
+ ", attempting to replace bad hosts");
TimeUnit.MINUTES.sleep(1);
executionContext.replaceBadHosts();
for (Host host : executionContext.getHosts()) {
if (!goodHosts.contains(host)) {
HostExecutor hostExecutor = hostExecutorBuilder.build(host);
initalizeHost(hostExecutor);
if (hostExecutor.isBad()) {
executionContext.addBadHost(hostExecutor.getHost());
} else {
logger.info("Adding new host during execution phase: " + host);
hostExecutors.add(hostExecutor);
}
}
}
}
}
public int run() {
int result = 0;
boolean error = false;
List<String> messages = Lists.newArrayList();
Map<String, Long> elapsedTimes = Maps.newTreeMap();
try {
mLogger.info("Running tests with " + mConfiguration);
for (Phase phase : mPhases) {
String msg = "Executing " + phase.getClass().getName();
mLogger.info(msg);
messages.add(msg);
long start = System.currentTimeMillis();
try {
phase.execute();
} finally {
long elapsedTime =
TimeUnit.MINUTES.convert((System.currentTimeMillis() - start), TimeUnit.MILLISECONDS);
elapsedTimes.put(phase.getClass().getSimpleName(), elapsedTime);
}
}
if (!mFailedTests.isEmpty()) {
throw new TestsFailedException(mFailedTests.size() + " tests failed");
}
} catch (Throwable throwable) {
mLogger.error("Test run exited with an unexpected error", throwable);
// NonZeroExitCodeExceptions can have long messages and should be
// trimmable when published to the JIRA via the JiraService
if (throwable instanceof NonZeroExitCodeException) {
messages.add("Tests exited with: " + throwable.getClass().getSimpleName());
for (String line : Strings.nullToEmpty(throwable.getMessage()).split("\n")) {
messages.add(line);
}
} else {
messages.add(
"Tests exited with: "
+ throwable.getClass().getSimpleName()
+ ": "
+ throwable.getMessage());
}
error = true;
} finally {
for (HostExecutor hostExecutor : mHostExecutors) {
hostExecutor.shutdownNow();
if (hostExecutor.isBad()) {
mExecutionContext.addBadHost(hostExecutor.getHost());
}
}
mSshCommandExecutor.shutdownNow();
mRsyncCommandExecutor.shutdownNow();
mExecutor.shutdownNow();
SortedSet<String> failedTests = new TreeSet<String>(mFailedTests);
if (failedTests.isEmpty()) {
mLogger.info(String.format("%d failed tests", failedTests.size()));
} else {
mLogger.warn(String.format("%d failed tests", failedTests.size()));
}
for (String failingTestName : failedTests) {
mLogger.warn(failingTestName);
}
mLogger.info("Executed " + mExecutedTests.size() + " tests");
for (Map.Entry<String, Long> entry : elapsedTimes.entrySet()) {
mLogger.info(
String.format("PERF: Phase %s took %d minutes", entry.getKey(), entry.getValue()));
}
publishJiraComment(error, messages, failedTests);
if (error || !mFailedTests.isEmpty()) {
result = 1;
}
}
return result;
}
