private String formatMockTestRules(List<TestRule> testRules) {
List<String> testNames = Lists.newArrayList();
for (TestRule testRule : testRules) {
testNames.add(testRule.getBuildTarget().getFullyQualifiedName());
}
return "{" + Joiner.on(", ").join(testNames) + "}";
}
@VisibleForTesting
static boolean isTestRunRequiredForTest(
TestRule test,
BuildEngine cachingBuildEngine,
ExecutionContext executionContext,
TestRuleKeyFileHelper testRuleKeyFileHelper,
boolean isResultsCacheEnabled,
boolean isRunningWithTestSelectors)
throws IOException, ExecutionException, InterruptedException {
boolean isTestRunRequired;
BuildResult result;
if (executionContext.isDebugEnabled()) {
// If debug is enabled, then we should always run the tests as the user is expecting to
// hook up a debugger.
isTestRunRequired = true;
} else if (isRunningWithTestSelectors) {
// As a feature to aid developers, we'll assume that when we are using test selectors,
// we should always run each test (and never look at the cache.)
// TODO(user) When #3090004 and #3436849 are closed we can respect the cache again.
isTestRunRequired = true;
} else if (((result = cachingBuildEngine.getBuildRuleResult(test.getBuildTarget())) != null)
&& result.getSuccess() == BuildRuleSuccessType.MATCHING_RULE_KEY
&& isResultsCacheEnabled
&& test.hasTestResultFiles(executionContext)
&& testRuleKeyFileHelper.isRuleKeyInDir(test)) {
// If this build rule's artifacts (which includes the rule's output and its test result
// files) are up to date, then no commands are necessary to run the tests. The test result
// files will be read from the XML files in interpretTestResults().
isTestRunRequired = false;
} else {
isTestRunRequired = true;
}
return isTestRunRequired;
}
/** Generates the set of Java library rules under test. */
private static ImmutableSet<JavaLibrary> getRulesUnderTest(Iterable<TestRule> tests) {
ImmutableSet.Builder<JavaLibrary> rulesUnderTest = ImmutableSet.builder();
// Gathering all rules whose source will be under test.
for (TestRule test : tests) {
if (test instanceof JavaTest) {
JavaTest javaTest = (JavaTest) test;
ImmutableSet<BuildRule> sourceUnderTest = javaTest.getSourceUnderTest();
for (BuildRule buildRule : sourceUnderTest) {
if (buildRule instanceof JavaLibrary) {
JavaLibrary javaLibrary = (JavaLibrary) buildRule;
rulesUnderTest.add(javaLibrary);
} else {
throw new HumanReadableException(
"Test '%s' is a java_test() "
+ "but it is testing module '%s' "
+ "which is not a java_library()!",
test.getBuildTarget(), buildRule.getBuildTarget());
}
}
}
}
return rulesUnderTest.build();
}
private int runTestsExternal(
final CommandRunnerParams params,
Build build,
Iterable<String> command,
Iterable<TestRule> testRules)
throws InterruptedException, IOException {
TestRunningOptions options = getTestRunningOptions(params);
// Walk the test rules, collecting all the specs.
List<ExternalTestRunnerTestSpec> specs = Lists.newArrayList();
for (TestRule testRule : testRules) {
if (!(testRule instanceof ExternalTestRunnerRule)) {
params
.getBuckEventBus()
.post(
ConsoleEvent.severe(
String.format(
"Test %s does not support external test running",
testRule.getBuildTarget())));
return 1;
}
ExternalTestRunnerRule rule = (ExternalTestRunnerRule) testRule;
specs.add(rule.getExternalTestRunnerSpec(build.getExecutionContext(), options));
}
// Serialize the specs to a file to pass into the test runner.
Path infoFile =
params
.getCell()
.getFilesystem()
.resolve(BuckConstant.SCRATCH_PATH.resolve("external_runner_specs.json"));
Files.createDirectories(infoFile.getParent());
Files.deleteIfExists(infoFile);
params.getObjectMapper().writerWithDefaultPrettyPrinter().writeValue(infoFile.toFile(), specs);
// Launch and run the external test runner, forwarding it's stdout/stderr to the console.
// We wait for it to complete then returns its error code.
ListeningProcessExecutor processExecutor = new ListeningProcessExecutor();
ProcessExecutorParams processExecutorParams =
ProcessExecutorParams.builder()
.addAllCommand(command)
.addAllCommand(withDashArguments)
.addCommand("--buck-test-info", infoFile.toString())
.setDirectory(params.getCell().getFilesystem().getRootPath().toFile())
.build();
ForwardingProcessListener processListener =
new ForwardingProcessListener(
Channels.newChannel(params.getConsole().getStdOut()),
Channels.newChannel(params.getConsole().getStdErr()));
ListeningProcessExecutor.LaunchedProcess process =
processExecutor.launchProcess(processExecutorParams, processListener);
try {
return processExecutor.waitForProcess(process, Long.MAX_VALUE, TimeUnit.DAYS);
} finally {
processExecutor.destroyProcess(process, /* force */ false);
processExecutor.waitForProcess(process, Long.MAX_VALUE, TimeUnit.DAYS);
}
}
private void printMatchingTestRules(Console console, Iterable<TestRule> testRules) {
PrintStream out = console.getStdOut();
ImmutableList<TestRule> list = ImmutableList.copyOf(testRules);
out.println(String.format("MATCHING TEST RULES (%d):", list.size()));
out.println("");
if (list.isEmpty()) {
out.println(" (none)");
} else {
for (TestRule testRule : testRules) {
out.println(" " + testRule.getBuildTarget());
}
}
out.println("");
}
@Test
public void testIsTestRunRequiredForTestBuiltLocally() throws IOException {
ExecutionContext executionContext = createMock(ExecutionContext.class);
expect(executionContext.isDebugEnabled()).andReturn(false);
TestRule testRule = createMock(TestRule.class);
expect(testRule.getBuildResultType()).andReturn(BuildRuleSuccess.Type.BUILT_LOCALLY);
replay(executionContext, testRule);
assertTrue(
"A test built locally should always run regardless of any cached result. ",
TestCommand.isTestRunRequiredForTest(
testRule, executionContext, createMock(TestRuleKeyFileHelper.class)));
verify(executionContext, testRule);
}
@Test
public void testIsTestRunRequiredForTestBuiltFromCacheIfHasTestResultFiles() throws IOException {
ExecutionContext executionContext = createMock(ExecutionContext.class);
expect(executionContext.isDebugEnabled()).andReturn(false);
TestRule testRule = createMock(TestRule.class);
expect(testRule.getBuildResultType()).andReturn(BuildRuleSuccess.Type.FETCHED_FROM_CACHE);
replay(executionContext, testRule);
assertTrue(
"A cache hit updates the build artifact but not the test results. "
+ "Therefore, the test should be re-run to ensure the test results are up to date.",
TestCommand.isTestRunRequiredForTest(
testRule, executionContext, createMock(TestRuleKeyFileHelper.class)));
verify(executionContext, testRule);
}
@VisibleForTesting
Iterable<TestRule> filterTestRules(
BuckConfig buckConfig,
ImmutableSet<BuildTarget> explicitBuildTargets,
Iterable<TestRule> testRules) {
ImmutableSortedSet.Builder<TestRule> builder =
ImmutableSortedSet.orderedBy(
new Comparator<TestRule>() {
@Override
public int compare(TestRule o1, TestRule o2) {
return o1.getBuildTarget()
.getFullyQualifiedName()
.compareTo(o2.getBuildTarget().getFullyQualifiedName());
}
});
for (TestRule rule : testRules) {
boolean explicitArgument = explicitBuildTargets.contains(rule.getBuildTarget());
boolean matchesLabel = isMatchedByLabelOptions(buckConfig, rule.getLabels());
// We always want to run the rules that are given on the command line. Always. Unless we don't
// want to.
if (shouldExcludeWin() && !matchesLabel) {
continue;
}
// The testRules Iterable contains transitive deps of the arguments given on the command line,
// filter those out if such is the user's will.
if (shouldExcludeTransitiveTests() && !explicitArgument) {
continue;
}
// Normal behavior is to include all rules that match the given label as well as any that
// were explicitly specified by the user.
if (explicitArgument || matchesLabel) {
builder.add(rule);
}
}
return builder.build();
}
@Test
public void testIsTestRunRequiredIfRuleKeyNotPresent() throws IOException {
ExecutionContext executionContext = createMock(ExecutionContext.class);
expect(executionContext.isDebugEnabled()).andReturn(false);
TestRule testRule = createNiceMock(TestRule.class);
expect(testRule.getBuildResultType()).andReturn(BuildRuleSuccess.Type.MATCHING_RULE_KEY);
expect(testRule.hasTestResultFiles(executionContext)).andReturn(true);
TestRuleKeyFileHelper testRuleKeyFileHelper = createNiceMock(TestRuleKeyFileHelper.class);
expect(testRuleKeyFileHelper.isRuleKeyInDir(testRule)).andReturn(false);
replay(executionContext, testRule, testRuleKeyFileHelper);
assertTrue(
"A cached build should run the tests if the test output directory\'s rule key is not "
+ "present or does not matche the rule key for the test.",
TestCommand.isTestRunRequiredForTest(testRule, executionContext, testRuleKeyFileHelper));
verify(executionContext, testRule, testRuleKeyFileHelper);
}
@SuppressWarnings("PMD.EmptyCatchBlock")
public static int runTests(
final CommandRunnerParams params,
Iterable<TestRule> tests,
BuildContext buildContext,
ExecutionContext executionContext,
final TestRunningOptions options,
ListeningExecutorService service,
BuildEngine buildEngine,
final StepRunner stepRunner)
throws IOException, ExecutionException, InterruptedException {
if (options.isUsingOneTimeOutputDirectories()) {
BuckConstant.setOneTimeTestSubdirectory(UUID.randomUUID().toString());
}
ImmutableSet<JavaLibrary> rulesUnderTest;
// If needed, we first run instrumentation on the class files.
if (options.isCodeCoverageEnabled()) {
rulesUnderTest = getRulesUnderTest(tests);
if (!rulesUnderTest.isEmpty()) {
try {
stepRunner.runStepForBuildTarget(
new MakeCleanDirectoryStep(JUnitStep.JACOCO_OUTPUT_DIR),
Optional.<BuildTarget>absent());
} catch (StepFailedException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
}
} else {
rulesUnderTest = ImmutableSet.of();
}
final ImmutableSet<String> testTargets =
FluentIterable.from(tests)
.transform(HasBuildTarget.TO_TARGET)
.transform(Functions.toStringFunction())
.toSet();
final int totalNumberOfTests = Iterables.size(tests);
params
.getBuckEventBus()
.post(
TestRunEvent.started(
options.isRunAllTests(),
options.getTestSelectorList(),
options.shouldExplainTestSelectorList(),
testTargets));
// Start running all of the tests. The result of each java_test() rule is represented as a
// ListenableFuture.
List<ListenableFuture<TestResults>> results = Lists.newArrayList();
// Unless `--verbose 0` is specified, print out test results as they become available.
// Failures with the ListenableFuture should always be printed, as they indicate an error with
// Buck, not the test being run.
Verbosity verbosity = params.getConsole().getVerbosity();
final boolean printTestResults = (verbosity != Verbosity.SILENT);
// For grouping results!
final TestResultsGrouper grouper;
if (options.isIgnoreFailingDependencies()) {
grouper = new TestResultsGrouper(tests);
} else {
grouper = null;
}
TestRuleKeyFileHelper testRuleKeyFileHelper =
new TestRuleKeyFileHelper(executionContext.getProjectFilesystem(), buildEngine);
final AtomicInteger lastReportedTestSequenceNumber = new AtomicInteger();
final List<TestRun> separateTestRuns = Lists.newArrayList();
List<TestRun> parallelTestRuns = Lists.newArrayList();
for (final TestRule test : tests) {
// Determine whether the test needs to be executed.
boolean isTestRunRequired;
isTestRunRequired =
isTestRunRequiredForTest(
test,
buildEngine,
executionContext,
testRuleKeyFileHelper,
options.isResultsCacheEnabled(),
!options.getTestSelectorList().isEmpty());
List<Step> steps;
if (isTestRunRequired) {
params.getBuckEventBus().post(IndividualTestEvent.started(testTargets));
ImmutableList.Builder<Step> stepsBuilder = ImmutableList.builder();
Preconditions.checkState(buildEngine.isRuleBuilt(test.getBuildTarget()));
final Map<String, UUID> testUUIDMap = new HashMap<>();
List<Step> testSteps =
test.runTests(
buildContext,
executionContext,
options.isDryRun(),
options.isShufflingTests(),
options.getTestSelectorList(),
new TestRule.TestReportingCallback() {
@Override
public void testsDidBegin() {
LOG.debug("Tests for rule %s began", test.getBuildTarget());
}
@Override
public void testDidBegin(String testCaseName, String testName) {
LOG.debug(
"Test rule %s test case %s test name %s began",
test.getBuildTarget(), testCaseName, testName);
UUID testUUID = UUID.randomUUID();
// UUID is immutable and thread-safe as of Java 7, so it's
// safe to stash in a map and use later:
//
// http://bugs.java.com/view_bug.do?bug_id=6611830
testUUIDMap.put(testCaseName + ":" + testName, testUUID);
params
.getBuckEventBus()
.post(TestSummaryEvent.started(testUUID, testCaseName, testName));
}
@Override
public void testDidEnd(TestResultSummary testResultSummary) {
LOG.debug(
"Test rule %s test did end: %s", test.getBuildTarget(), testResultSummary);
UUID testUUID =
testUUIDMap.get(
testResultSummary.getTestCaseName()
+ ":"
+ testResultSummary.getTestName());
Preconditions.checkNotNull(testUUID);
params
.getBuckEventBus()
.post(TestSummaryEvent.finished(testUUID, testResultSummary));
}
@Override
public void testsDidEnd(List<TestCaseSummary> testCaseSummaries) {
LOG.debug(
"Test rule %s tests did end: %s", test.getBuildTarget(), testCaseSummaries);
}
});
if (!testSteps.isEmpty()) {
stepsBuilder.addAll(testSteps);
stepsBuilder.add(testRuleKeyFileHelper.createRuleKeyInDirStep(test));
}
steps = stepsBuilder.build();
} else {
steps = ImmutableList.of();
}
TestRun testRun =
TestRun.of(
test,
steps,
getCachingStatusTransformingCallable(
isTestRunRequired,
test.interpretTestResults(
executionContext,
/*isUsingTestSelectors*/ !options.getTestSelectorList().isEmpty(),
/*isDryRun*/ options.isDryRun())));
// Always run the commands, even if the list of commands as empty. There may be zero
// commands because the rule is cached, but its results must still be processed.
if (test.runTestSeparately()) {
LOG.debug("Running test %s in serial", test);
separateTestRuns.add(testRun);
} else {
LOG.debug("Running test %s in parallel", test);
parallelTestRuns.add(testRun);
}
}
final StepRunner.StepRunningCallback testStepRunningCallback =
new StepRunner.StepRunningCallback() {
@Override
public void stepsWillRun(Optional<BuildTarget> buildTarget) {
Preconditions.checkState(buildTarget.isPresent());
LOG.debug("Test steps will run for %s", buildTarget);
params.getBuckEventBus().post(TestRuleEvent.started(buildTarget.get()));
}
@Override
public void stepsDidRun(Optional<BuildTarget> buildTarget) {
Preconditions.checkState(buildTarget.isPresent());
LOG.debug("Test steps did run for %s", buildTarget);
params.getBuckEventBus().post(TestRuleEvent.finished(buildTarget.get()));
}
};
for (TestRun testRun : parallelTestRuns) {
ListenableFuture<TestResults> testResults =
stepRunner.runStepsAndYieldResult(
testRun.getSteps(),
testRun.getTestResultsCallable(),
Optional.of(testRun.getTest().getBuildTarget()),
service,
testStepRunningCallback);
results.add(
transformTestResults(
params,
testResults,
grouper,
testRun.getTest(),
testTargets,
printTestResults,
lastReportedTestSequenceNumber,
totalNumberOfTests));
}
ListenableFuture<List<TestResults>> parallelTestStepsFuture = Futures.allAsList(results);
final List<TestResults> completedResults = Lists.newArrayList();
final ListeningExecutorService directExecutorService = MoreExecutors.newDirectExecutorService();
ListenableFuture<Void> uberFuture =
stepRunner.addCallback(
parallelTestStepsFuture,
new FutureCallback<List<TestResults>>() {
@Override
public void onSuccess(List<TestResults> parallelTestResults) {
LOG.debug("Parallel tests completed, running separate tests...");
completedResults.addAll(parallelTestResults);
List<ListenableFuture<TestResults>> separateResultsList = Lists.newArrayList();
for (TestRun testRun : separateTestRuns) {
separateResultsList.add(
transformTestResults(
params,
stepRunner.runStepsAndYieldResult(
testRun.getSteps(),
testRun.getTestResultsCallable(),
Optional.of(testRun.getTest().getBuildTarget()),
directExecutorService,
testStepRunningCallback),
grouper,
testRun.getTest(),
testTargets,
printTestResults,
lastReportedTestSequenceNumber,
totalNumberOfTests));
}
ListenableFuture<List<TestResults>> serialResults =
Futures.allAsList(separateResultsList);
try {
completedResults.addAll(serialResults.get());
} catch (ExecutionException e) {
LOG.error(e, "Error fetching serial test results");
throw new HumanReadableException(e, "Error fetching serial test results");
} catch (InterruptedException e) {
LOG.error(e, "Interrupted fetching serial test results");
try {
serialResults.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw new HumanReadableException(e, "Test cancelled");
}
LOG.debug("Done running serial tests.");
}
@Override
public void onFailure(Throwable e) {
LOG.error(e, "Parallel tests failed, not running serial tests");
throw new HumanReadableException(e, "Parallel tests failed");
}
},
directExecutorService);
try {
// Block until all the tests have finished running.
uberFuture.get();
} catch (ExecutionException e) {
e.printStackTrace(params.getConsole().getStdErr());
return 1;
} catch (InterruptedException e) {
try {
uberFuture.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw e;
}
params.getBuckEventBus().post(TestRunEvent.finished(testTargets, completedResults));
// Write out the results as XML, if requested.
Optional<String> path = options.getPathToXmlTestOutput();
if (path.isPresent()) {
try (Writer writer = Files.newWriter(new File(path.get()), Charsets.UTF_8)) {
writeXmlOutput(completedResults, writer);
}
}
// Generate the code coverage report.
if (options.isCodeCoverageEnabled() && !rulesUnderTest.isEmpty()) {
try {
Optional<DefaultJavaPackageFinder> defaultJavaPackageFinderOptional =
Optional.fromNullable(params.getBuckConfig().createDefaultJavaPackageFinder());
stepRunner.runStepForBuildTarget(
getReportCommand(
rulesUnderTest,
defaultJavaPackageFinderOptional,
params.getRepository().getFilesystem(),
JUnitStep.JACOCO_OUTPUT_DIR,
options.getCoverageReportFormat()),
Optional.<BuildTarget>absent());
} catch (StepFailedException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
}
boolean failures =
Iterables.any(
completedResults,
new Predicate<TestResults>() {
@Override
public boolean apply(TestResults results) {
LOG.debug("Checking result %s for failure", results);
return !results.isSuccess();
}
});
return failures ? TEST_FAILURES_EXIT_CODE : 0;
}