@SuppressWarnings("PMD.EmptyCatchBlock")
public static int runTests(
final CommandRunnerParams params,
Iterable<TestRule> tests,
BuildContext buildContext,
ExecutionContext executionContext,
final TestRunningOptions options,
ListeningExecutorService service,
BuildEngine buildEngine,
final StepRunner stepRunner)
throws IOException, ExecutionException, InterruptedException {
if (options.isUsingOneTimeOutputDirectories()) {
BuckConstant.setOneTimeTestSubdirectory(UUID.randomUUID().toString());
}
ImmutableSet<JavaLibrary> rulesUnderTest;
// If needed, we first run instrumentation on the class files.
if (options.isCodeCoverageEnabled()) {
rulesUnderTest = getRulesUnderTest(tests);
if (!rulesUnderTest.isEmpty()) {
try {
stepRunner.runStepForBuildTarget(
new MakeCleanDirectoryStep(JUnitStep.JACOCO_OUTPUT_DIR),
Optional.<BuildTarget>absent());
} catch (StepFailedException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
}
} else {
rulesUnderTest = ImmutableSet.of();
}
final ImmutableSet<String> testTargets =
FluentIterable.from(tests)
.transform(HasBuildTarget.TO_TARGET)
.transform(Functions.toStringFunction())
.toSet();
final int totalNumberOfTests = Iterables.size(tests);
params
.getBuckEventBus()
.post(
TestRunEvent.started(
options.isRunAllTests(),
options.getTestSelectorList(),
options.shouldExplainTestSelectorList(),
testTargets));
// Start running all of the tests. The result of each java_test() rule is represented as a
// ListenableFuture.
List<ListenableFuture<TestResults>> results = Lists.newArrayList();
// Unless `--verbose 0` is specified, print out test results as they become available.
// Failures with the ListenableFuture should always be printed, as they indicate an error with
// Buck, not the test being run.
Verbosity verbosity = params.getConsole().getVerbosity();
final boolean printTestResults = (verbosity != Verbosity.SILENT);
// For grouping results!
final TestResultsGrouper grouper;
if (options.isIgnoreFailingDependencies()) {
grouper = new TestResultsGrouper(tests);
} else {
grouper = null;
}
TestRuleKeyFileHelper testRuleKeyFileHelper =
new TestRuleKeyFileHelper(executionContext.getProjectFilesystem(), buildEngine);
final AtomicInteger lastReportedTestSequenceNumber = new AtomicInteger();
final List<TestRun> separateTestRuns = Lists.newArrayList();
List<TestRun> parallelTestRuns = Lists.newArrayList();
for (final TestRule test : tests) {
// Determine whether the test needs to be executed.
boolean isTestRunRequired;
isTestRunRequired =
isTestRunRequiredForTest(
test,
buildEngine,
executionContext,
testRuleKeyFileHelper,
options.isResultsCacheEnabled(),
!options.getTestSelectorList().isEmpty());
List<Step> steps;
if (isTestRunRequired) {
params.getBuckEventBus().post(IndividualTestEvent.started(testTargets));
ImmutableList.Builder<Step> stepsBuilder = ImmutableList.builder();
Preconditions.checkState(buildEngine.isRuleBuilt(test.getBuildTarget()));
final Map<String, UUID> testUUIDMap = new HashMap<>();
List<Step> testSteps =
test.runTests(
buildContext,
executionContext,
options.isDryRun(),
options.isShufflingTests(),
options.getTestSelectorList(),
new TestRule.TestReportingCallback() {
@Override
public void testsDidBegin() {
LOG.debug("Tests for rule %s began", test.getBuildTarget());
}
@Override
public void testDidBegin(String testCaseName, String testName) {
LOG.debug(
"Test rule %s test case %s test name %s began",
test.getBuildTarget(), testCaseName, testName);
UUID testUUID = UUID.randomUUID();
// UUID is immutable and thread-safe as of Java 7, so it's
// safe to stash in a map and use later:
//
// http://bugs.java.com/view_bug.do?bug_id=6611830
testUUIDMap.put(testCaseName + ":" + testName, testUUID);
params
.getBuckEventBus()
.post(TestSummaryEvent.started(testUUID, testCaseName, testName));
}
@Override
public void testDidEnd(TestResultSummary testResultSummary) {
LOG.debug(
"Test rule %s test did end: %s", test.getBuildTarget(), testResultSummary);
UUID testUUID =
testUUIDMap.get(
testResultSummary.getTestCaseName()
+ ":"
+ testResultSummary.getTestName());
Preconditions.checkNotNull(testUUID);
params
.getBuckEventBus()
.post(TestSummaryEvent.finished(testUUID, testResultSummary));
}
@Override
public void testsDidEnd(List<TestCaseSummary> testCaseSummaries) {
LOG.debug(
"Test rule %s tests did end: %s", test.getBuildTarget(), testCaseSummaries);
}
});
if (!testSteps.isEmpty()) {
stepsBuilder.addAll(testSteps);
stepsBuilder.add(testRuleKeyFileHelper.createRuleKeyInDirStep(test));
}
steps = stepsBuilder.build();
} else {
steps = ImmutableList.of();
}
TestRun testRun =
TestRun.of(
test,
steps,
getCachingStatusTransformingCallable(
isTestRunRequired,
test.interpretTestResults(
executionContext,
/*isUsingTestSelectors*/ !options.getTestSelectorList().isEmpty(),
/*isDryRun*/ options.isDryRun())));
// Always run the commands, even if the list of commands as empty. There may be zero
// commands because the rule is cached, but its results must still be processed.
if (test.runTestSeparately()) {
LOG.debug("Running test %s in serial", test);
separateTestRuns.add(testRun);
} else {
LOG.debug("Running test %s in parallel", test);
parallelTestRuns.add(testRun);
}
}
final StepRunner.StepRunningCallback testStepRunningCallback =
new StepRunner.StepRunningCallback() {
@Override
public void stepsWillRun(Optional<BuildTarget> buildTarget) {
Preconditions.checkState(buildTarget.isPresent());
LOG.debug("Test steps will run for %s", buildTarget);
params.getBuckEventBus().post(TestRuleEvent.started(buildTarget.get()));
}
@Override
public void stepsDidRun(Optional<BuildTarget> buildTarget) {
Preconditions.checkState(buildTarget.isPresent());
LOG.debug("Test steps did run for %s", buildTarget);
params.getBuckEventBus().post(TestRuleEvent.finished(buildTarget.get()));
}
};
for (TestRun testRun : parallelTestRuns) {
ListenableFuture<TestResults> testResults =
stepRunner.runStepsAndYieldResult(
testRun.getSteps(),
testRun.getTestResultsCallable(),
Optional.of(testRun.getTest().getBuildTarget()),
service,
testStepRunningCallback);
results.add(
transformTestResults(
params,
testResults,
grouper,
testRun.getTest(),
testTargets,
printTestResults,
lastReportedTestSequenceNumber,
totalNumberOfTests));
}
ListenableFuture<List<TestResults>> parallelTestStepsFuture = Futures.allAsList(results);
final List<TestResults> completedResults = Lists.newArrayList();
final ListeningExecutorService directExecutorService = MoreExecutors.newDirectExecutorService();
ListenableFuture<Void> uberFuture =
stepRunner.addCallback(
parallelTestStepsFuture,
new FutureCallback<List<TestResults>>() {
@Override
public void onSuccess(List<TestResults> parallelTestResults) {
LOG.debug("Parallel tests completed, running separate tests...");
completedResults.addAll(parallelTestResults);
List<ListenableFuture<TestResults>> separateResultsList = Lists.newArrayList();
for (TestRun testRun : separateTestRuns) {
separateResultsList.add(
transformTestResults(
params,
stepRunner.runStepsAndYieldResult(
testRun.getSteps(),
testRun.getTestResultsCallable(),
Optional.of(testRun.getTest().getBuildTarget()),
directExecutorService,
testStepRunningCallback),
grouper,
testRun.getTest(),
testTargets,
printTestResults,
lastReportedTestSequenceNumber,
totalNumberOfTests));
}
ListenableFuture<List<TestResults>> serialResults =
Futures.allAsList(separateResultsList);
try {
completedResults.addAll(serialResults.get());
} catch (ExecutionException e) {
LOG.error(e, "Error fetching serial test results");
throw new HumanReadableException(e, "Error fetching serial test results");
} catch (InterruptedException e) {
LOG.error(e, "Interrupted fetching serial test results");
try {
serialResults.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw new HumanReadableException(e, "Test cancelled");
}
LOG.debug("Done running serial tests.");
}
@Override
public void onFailure(Throwable e) {
LOG.error(e, "Parallel tests failed, not running serial tests");
throw new HumanReadableException(e, "Parallel tests failed");
}
},
directExecutorService);
try {
// Block until all the tests have finished running.
uberFuture.get();
} catch (ExecutionException e) {
e.printStackTrace(params.getConsole().getStdErr());
return 1;
} catch (InterruptedException e) {
try {
uberFuture.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw e;
}
params.getBuckEventBus().post(TestRunEvent.finished(testTargets, completedResults));
// Write out the results as XML, if requested.
Optional<String> path = options.getPathToXmlTestOutput();
if (path.isPresent()) {
try (Writer writer = Files.newWriter(new File(path.get()), Charsets.UTF_8)) {
writeXmlOutput(completedResults, writer);
}
}
// Generate the code coverage report.
if (options.isCodeCoverageEnabled() && !rulesUnderTest.isEmpty()) {
try {
Optional<DefaultJavaPackageFinder> defaultJavaPackageFinderOptional =
Optional.fromNullable(params.getBuckConfig().createDefaultJavaPackageFinder());
stepRunner.runStepForBuildTarget(
getReportCommand(
rulesUnderTest,
defaultJavaPackageFinderOptional,
params.getRepository().getFilesystem(),
JUnitStep.JACOCO_OUTPUT_DIR,
options.getCoverageReportFormat()),
Optional.<BuildTarget>absent());
} catch (StepFailedException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
}
boolean failures =
Iterables.any(
completedResults,
new Predicate<TestResults>() {
@Override
public boolean apply(TestResults results) {
LOG.debug("Checking result %s for failure", results);
return !results.isSuccess();
}
});
return failures ? TEST_FAILURES_EXIT_CODE : 0;
}
public class TestRunningTest {
private static ImmutableSortedSet<String> pathsFromRoot;
private static ImmutableSet<String> pathElements;
private static final TestRunningOptions DEFAULT_OPTIONS = TestRunningOptions.builder().build();
private static final Logger LOG = Logger.get(TestRunningTest.class);
@BeforeClass
public static void setUp() {
pathsFromRoot = ImmutableSortedSet.of("java/");
pathElements = ImmutableSet.of("src", "src-gen");
}
/**
* If the source paths specified are all generated files, then our path to source tmp should be
* absent.
*/
@Test
public void testGeneratedSourceFile() {
Path pathToGenFile = GEN_PATH.resolve("GeneratedFile.java");
assertTrue(MorePaths.isGeneratedFile(pathToGenFile));
ImmutableSortedSet<Path> javaSrcs = ImmutableSortedSet.of(pathToGenFile);
JavaLibrary javaLibrary =
new FakeJavaLibrary(
BuildTarget.builder("//foo", "bar").build(),
new SourcePathResolver(new BuildRuleResolver()))
.setJavaSrcs(javaSrcs);
DefaultJavaPackageFinder defaultJavaPackageFinder = createMock(DefaultJavaPackageFinder.class);
Object[] mocks = new Object[] {defaultJavaPackageFinder};
replay(mocks);
ImmutableSet<String> result =
TestRunning.getPathToSourceFolders(
javaLibrary, Optional.of(defaultJavaPackageFinder), new FakeProjectFilesystem());
assertTrue(
"No path should be returned if the library contains only generated files.",
result.isEmpty());
verify(mocks);
}
/**
* If the source paths specified are all for non-generated files then we should return the correct
* source tmp corresponding to a non-generated source path.
*/
@Test
public void testNonGeneratedSourceFile() {
Path pathToNonGenFile = Paths.get("package/src/SourceFile1.java");
assertFalse(MorePaths.isGeneratedFile(pathToNonGenFile));
ImmutableSortedSet<Path> javaSrcs = ImmutableSortedSet.of(pathToNonGenFile);
JavaLibrary javaLibrary =
new FakeJavaLibrary(
BuildTarget.builder("//foo", "bar").build(),
new SourcePathResolver(new BuildRuleResolver()))
.setJavaSrcs(javaSrcs);
DefaultJavaPackageFinder defaultJavaPackageFinder = createMock(DefaultJavaPackageFinder.class);
expect(defaultJavaPackageFinder.getPathsFromRoot()).andReturn(pathsFromRoot);
expect(defaultJavaPackageFinder.getPathElements()).andReturn(pathElements);
replay(defaultJavaPackageFinder);
ImmutableSet<String> result =
TestRunning.getPathToSourceFolders(
javaLibrary, Optional.of(defaultJavaPackageFinder), new FakeProjectFilesystem());
assertEquals(
"All non-generated source files are under one source tmp.",
ImmutableSet.of("./package/src/"),
result);
verify(defaultJavaPackageFinder);
}
/**
* If the source paths specified are from the new unified source tmp then we should return the
* correct source tmp corresponding to the unified source path.
*/
@Test
public void testUnifiedSourceFile() {
Path pathToNonGenFile = Paths.get("java/package/SourceFile1.java");
assertFalse(MorePaths.isGeneratedFile(pathToNonGenFile));
ImmutableSortedSet<Path> javaSrcs = ImmutableSortedSet.of(pathToNonGenFile);
JavaLibrary javaLibrary =
new FakeJavaLibrary(
BuildTarget.builder("//foo", "bar").build(),
new SourcePathResolver(new BuildRuleResolver()))
.setJavaSrcs(javaSrcs);
DefaultJavaPackageFinder defaultJavaPackageFinder = createMock(DefaultJavaPackageFinder.class);
expect(defaultJavaPackageFinder.getPathsFromRoot()).andReturn(pathsFromRoot);
Object[] mocks = new Object[] {defaultJavaPackageFinder};
replay(mocks);
ImmutableSet<String> result =
TestRunning.getPathToSourceFolders(
javaLibrary, Optional.of(defaultJavaPackageFinder), new FakeProjectFilesystem());
assertEquals(
"All non-generated source files are under one source tmp.",
ImmutableSet.of("java/"),
result);
verify(mocks);
}
/**
* If the source paths specified contains one source path to a non-generated file then we should
* return the correct source tmp corresponding to that non-generated source path. Especially when
* the generated file comes first in the ordered set.
*/
@Test
public void testMixedSourceFile() {
Path pathToGenFile = GEN_PATH.resolve("com/facebook/GeneratedFile.java");
Path pathToNonGenFile1 = Paths.get("package/src/SourceFile1.java");
Path pathToNonGenFile2 = Paths.get("package/src-gen/SourceFile2.java");
ImmutableSortedSet<Path> javaSrcs =
ImmutableSortedSet.of(pathToGenFile, pathToNonGenFile1, pathToNonGenFile2);
DefaultJavaPackageFinder defaultJavaPackageFinder = createMock(DefaultJavaPackageFinder.class);
expect(defaultJavaPackageFinder.getPathsFromRoot()).andReturn(pathsFromRoot).times(2);
expect(defaultJavaPackageFinder.getPathElements()).andReturn(pathElements).times(2);
JavaLibrary javaLibrary =
new FakeJavaLibrary(
BuildTarget.builder("//foo", "bar").build(),
new SourcePathResolver(new BuildRuleResolver()))
.setJavaSrcs(javaSrcs);
replay(defaultJavaPackageFinder);
ImmutableSet<String> result =
TestRunning.getPathToSourceFolders(
javaLibrary, Optional.of(defaultJavaPackageFinder), new FakeProjectFilesystem());
assertEquals(
"The non-generated source files are under two different source folders.",
ImmutableSet.of("./package/src-gen/", "./package/src/"),
result);
verify(defaultJavaPackageFinder);
}
/** Tests the --xml flag, ensuring that test result data is correctly formatted. */
@Test
public void testXmlGeneration() throws Exception {
// Set up sample test data.
TestResultSummary result1 =
new TestResultSummary(
/* testCaseName */ "TestCase",
/* testName */ "passTest",
/* type */ ResultType.SUCCESS,
/* time */ 5000,
/* message */ null,
/* stacktrace */ null,
/* stdOut */ null,
/* stdErr */ null);
TestResultSummary result2 =
new TestResultSummary(
/* testCaseName */ "TestCase",
/* testName */ "failWithMsg",
/* type */ ResultType.FAILURE,
/* time */ 7000,
/* message */ "Index out of bounds!",
/* stacktrace */ "Stacktrace",
/* stdOut */ null,
/* stdErr */ null);
TestResultSummary result3 =
new TestResultSummary(
/* testCaseName */ "TestCase",
/* testName */ "failNoMsg",
/* isSuccess */
/* type */ ResultType.SUCCESS,
/* time */ 4000,
/* message */ null,
/* stacktrace */ null,
/* stdOut */ null,
/* stdErr */ null);
List<TestResultSummary> resultList = ImmutableList.of(result1, result2, result3);
TestCaseSummary testCase = new TestCaseSummary("TestCase", resultList);
List<TestCaseSummary> testCases = ImmutableList.of(testCase);
TestResults testResults = new TestResults(testCases);
List<TestResults> testResultsList = ImmutableList.of(testResults);
// Call the XML generation method with our test data.
StringWriter writer = new StringWriter();
TestRunning.writeXmlOutput(testResultsList, writer);
ByteArrayInputStream resultStream = new ByteArrayInputStream(writer.toString().getBytes());
// Convert the raw XML data into a DOM object, which we will check.
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder docBuilder = dbf.newDocumentBuilder();
Document doc = docBuilder.parse(resultStream);
// Check for exactly one <tests> tag.
NodeList testsList = doc.getElementsByTagName("tests");
assertEquals(testsList.getLength(), 1);
// Check for exactly one <test> tag.
Element testsEl = (Element) testsList.item(0);
NodeList testList = testsEl.getElementsByTagName("test");
assertEquals(testList.getLength(), 1);
// Check for exactly three <testresult> tags.
// There should be two failures and one success.
Element testEl = (Element) testList.item(0);
NodeList resultsList = testEl.getElementsByTagName("testresult");
assertEquals(resultsList.getLength(), 3);
// Verify the text elements of the first <testresult> tag.
Element passResultEl = (Element) resultsList.item(0);
assertEquals(passResultEl.getAttribute("name"), "passTest");
assertEquals(passResultEl.getAttribute("time"), "5000");
checkXmlTextContents(passResultEl, "message", "");
checkXmlTextContents(passResultEl, "stacktrace", "");
// Verify the text elements of the second <testresult> tag.
assertEquals(testEl.getAttribute("name"), "TestCase");
Element failResultEl1 = (Element) resultsList.item(1);
assertEquals(failResultEl1.getAttribute("name"), "failWithMsg");
assertEquals(failResultEl1.getAttribute("time"), "7000");
checkXmlTextContents(failResultEl1, "message", "Index out of bounds!");
checkXmlTextContents(failResultEl1, "stacktrace", "Stacktrace");
// Verify the text elements of the third <testresult> tag.
Element failResultEl2 = (Element) resultsList.item(2);
assertEquals(failResultEl2.getAttribute("name"), "failNoMsg");
assertEquals(failResultEl2.getAttribute("time"), "4000");
checkXmlTextContents(failResultEl2, "message", "");
checkXmlTextContents(failResultEl2, "stacktrace", "");
}
/** Helper method for testXMLGeneration(). Used to verify the message and stacktrace fields */
private void checkXmlTextContents(
Element testResult, String attributeName, String expectedValue) {
// Check for exactly one text element.
NodeList fieldMatchList = testResult.getElementsByTagName(attributeName);
assertEquals(fieldMatchList.getLength(), 1);
Element fieldEl = (Element) fieldMatchList.item(0);
// Check that the value within the text element is as expected.
Node firstChild = fieldEl.getFirstChild();
String expectedStr = Strings.nullToEmpty(expectedValue);
assertTrue(
((firstChild == null) && (expectedStr.equals("")))
|| ((firstChild != null) && expectedStr.equals(firstChild.getNodeValue())));
}
@Test
public void testIsTestRunRequiredForTestInDebugMode()
throws IOException, ExecutionException, InterruptedException {
ExecutionContext executionContext = createMock(ExecutionContext.class);
expect(executionContext.isDebugEnabled()).andReturn(true);
replay(executionContext);
assertTrue(
"In debug mode, test should always run regardless of any cached results since "
+ "the user is expecting to hook up a debugger.",
TestRunning.isTestRunRequiredForTest(
createMock(TestRule.class),
createMock(CachingBuildEngine.class),
executionContext,
createMock(TestRuleKeyFileHelper.class),
true,
false));
verify(executionContext);
}
@Test
public void testIsTestRunRequiredForTestBuiltFromCacheIfHasTestResultFiles()
throws IOException, ExecutionException, InterruptedException {
ExecutionContext executionContext = createMock(ExecutionContext.class);
expect(executionContext.isDebugEnabled()).andReturn(false);
FakeTestRule testRule =
new FakeTestRule(
ImmutableSet.<Label>of(Label.of("windows")),
BuildTargetFactory.newInstance("//:lulz"),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSortedSet.<BuildRule>of());
CachingBuildEngine cachingBuildEngine = createMock(CachingBuildEngine.class);
BuildResult result = new BuildResult(testRule, FETCHED_FROM_CACHE, CacheResult.hit("dir"));
expect(cachingBuildEngine.getBuildRuleResult(BuildTargetFactory.newInstance("//:lulz")))
.andReturn(result);
replay(executionContext, cachingBuildEngine);
assertTrue(
"A cache hit updates the build artifact but not the test results. "
+ "Therefore, the test should be re-run to ensure the test results are up to date.",
TestRunning.isTestRunRequiredForTest(
testRule,
cachingBuildEngine,
executionContext,
createMock(TestRuleKeyFileHelper.class),
/* results cache enabled */ true,
/* running with test selectors */ false));
verify(executionContext, cachingBuildEngine);
}
@Test
public void testIsTestRunRequiredForTestBuiltLocally()
throws IOException, ExecutionException, InterruptedException {
ExecutionContext executionContext = createMock(ExecutionContext.class);
expect(executionContext.isDebugEnabled()).andReturn(false);
FakeTestRule testRule =
new FakeTestRule(
ImmutableSet.<Label>of(Label.of("windows")),
BuildTargetFactory.newInstance("//:lulz"),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSortedSet.<BuildRule>of());
CachingBuildEngine cachingBuildEngine = createMock(CachingBuildEngine.class);
BuildResult result = new BuildResult(testRule, BUILT_LOCALLY, CacheResult.skip());
expect(cachingBuildEngine.getBuildRuleResult(BuildTargetFactory.newInstance("//:lulz")))
.andReturn(result);
replay(executionContext, cachingBuildEngine);
assertTrue(
"A test built locally should always run regardless of any cached result. ",
TestRunning.isTestRunRequiredForTest(
testRule,
cachingBuildEngine,
executionContext,
createMock(TestRuleKeyFileHelper.class),
/* results cache enabled */ true,
/* running with test selectors */ false));
verify(executionContext, cachingBuildEngine);
}
@Test
public void testIsTestRunRequiredIfRuleKeyNotPresent()
throws IOException, ExecutionException, InterruptedException {
ExecutionContext executionContext = createMock(ExecutionContext.class);
expect(executionContext.isDebugEnabled()).andReturn(false);
FakeTestRule testRule =
new FakeTestRule(
ImmutableSet.<Label>of(Label.of("windows")),
BuildTargetFactory.newInstance("//:lulz"),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSortedSet.<BuildRule>of()) {
@Override
public boolean hasTestResultFiles(ExecutionContext context) {
return true;
}
};
TestRuleKeyFileHelper testRuleKeyFileHelper = createNiceMock(TestRuleKeyFileHelper.class);
expect(testRuleKeyFileHelper.isRuleKeyInDir(testRule)).andReturn(false);
CachingBuildEngine cachingBuildEngine = createMock(CachingBuildEngine.class);
BuildResult result = new BuildResult(testRule, MATCHING_RULE_KEY, CacheResult.skip());
expect(cachingBuildEngine.getBuildRuleResult(BuildTargetFactory.newInstance("//:lulz")))
.andReturn(result);
replay(executionContext, cachingBuildEngine, testRuleKeyFileHelper);
assertTrue(
"A cached build should run the tests if the test output directory\'s rule key is not "
+ "present or does not matche the rule key for the test.",
TestRunning.isTestRunRequiredForTest(
testRule,
cachingBuildEngine,
executionContext,
testRuleKeyFileHelper,
/* results cache enabled */ true,
/* running with test selectors */ false));
verify(executionContext, cachingBuildEngine, testRuleKeyFileHelper);
}
@Test
public void whenAllTestsAreSeparateTestsRunInOrder() throws Exception {
CommandRunnerParams commandRunnerParams = CommandRunnerParamsForTesting.builder().build();
AtomicInteger atomicExecutionOrder = new AtomicInteger(0);
ExecutionOrderAwareFakeStep separateTestStep1 =
new ExecutionOrderAwareFakeStep("teststep1", "teststep1", 0, atomicExecutionOrder);
final TestResults fakeTestResults =
new TestResults(
ImmutableList.of(
new TestCaseSummary(
"TestCase",
ImmutableList.of(
new TestResultSummary(
"TestCaseResult",
"passTest",
ResultType.SUCCESS,
5000,
null,
null,
null,
null)))));
BuildTarget separateTest1Target = BuildTargetFactory.newInstance("//:test1");
FakeTestRule separateTest1 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest1Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("separateTestStep1OutputDir")),
true, // runTestSeparately
ImmutableList.<Step>of(separateTestStep1),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
ExecutionOrderAwareFakeStep separateTestStep2 =
new ExecutionOrderAwareFakeStep("teststep2", "teststep2", 0, atomicExecutionOrder);
BuildTarget separateTest2Target = BuildTargetFactory.newInstance("//:test2");
FakeTestRule separateTest2 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest2Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("separateTestStep2OutputDir")),
true, // runTestSeparately
ImmutableList.<Step>of(separateTestStep2),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
ExecutionOrderAwareFakeStep separateTestStep3 =
new ExecutionOrderAwareFakeStep("teststep3", "teststep3", 0, atomicExecutionOrder);
BuildTarget separateTest3Target = BuildTargetFactory.newInstance("//:test3");
FakeTestRule separateTest3 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest3Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("separateTestStep3OutputDir")),
true, // runTestSeparately
ImmutableList.<Step>of(separateTestStep3),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
// We explicitly use an actual thread pool here; the logic should ensure the
// separate tests are run in the correct order.
ListeningExecutorService service =
MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
FakeBuildEngine fakeBuildEngine =
new FakeBuildEngine(
ImmutableMap.of(
separateTest1Target,
new BuildResult(separateTest1, BUILT_LOCALLY, CacheResult.skip()),
separateTest2Target,
new BuildResult(separateTest2, BUILT_LOCALLY, CacheResult.skip()),
separateTest3Target,
new BuildResult(separateTest3, BUILT_LOCALLY, CacheResult.skip())),
ImmutableMap.of(
separateTest1Target, new RuleKey("00"),
separateTest2Target, new RuleKey("00"),
separateTest3Target, new RuleKey("00")));
ExecutionContext fakeExecutionContext =
TestExecutionContext.newBuilder().setProjectFilesystem(new FakeProjectFilesystem()).build();
DefaultStepRunner stepRunner = new DefaultStepRunner(fakeExecutionContext);
int ret =
TestRunning.runTests(
commandRunnerParams,
ImmutableList.<TestRule>of(separateTest1, separateTest2, separateTest3),
FakeBuildContext.NOOP_CONTEXT,
fakeExecutionContext,
DEFAULT_OPTIONS,
service,
fakeBuildEngine,
stepRunner);
assertThat(ret, equalTo(0));
assertThat(separateTestStep1.getExecutionBeginOrder(), equalTo(Optional.of(0)));
assertThat(separateTestStep1.getExecutionEndOrder(), equalTo(Optional.of(1)));
assertThat(separateTestStep2.getExecutionBeginOrder(), equalTo(Optional.of(2)));
assertThat(separateTestStep2.getExecutionEndOrder(), equalTo(Optional.of(3)));
assertThat(separateTestStep3.getExecutionBeginOrder(), equalTo(Optional.of(4)));
assertThat(separateTestStep3.getExecutionEndOrder(), equalTo(Optional.of(5)));
}
@Test
public void whenSomeTestsAreSeparateThenSeparateTestsRunAtEnd() throws Exception {
CommandRunnerParams commandRunnerParams = CommandRunnerParamsForTesting.builder().build();
AtomicInteger atomicExecutionOrder = new AtomicInteger(0);
ExecutionOrderAwareFakeStep separateTestStep1 =
new ExecutionOrderAwareFakeStep("teststep1", "teststep1", 0, atomicExecutionOrder);
final TestResults fakeTestResults =
new TestResults(
ImmutableList.of(
new TestCaseSummary(
"TestCase",
ImmutableList.of(
new TestResultSummary(
"TestCaseResult",
"passTest",
ResultType.SUCCESS,
5000,
null,
null,
null,
null)))));
BuildTarget separateTest1Target = BuildTargetFactory.newInstance("//:test1");
FakeTestRule separateTest1 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest1Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("separateTestStep1OutputDir")),
true, // runTestSeparately
ImmutableList.<Step>of(separateTestStep1),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
ExecutionOrderAwareFakeStep separateTestStep2 =
new ExecutionOrderAwareFakeStep("teststep2", "teststep2", 0, atomicExecutionOrder);
BuildTarget separateTest2Target = BuildTargetFactory.newInstance("//:test2");
FakeTestRule separateTest2 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest2Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("separateTestStep2OutputDir")),
true, // runTestSeparately
ImmutableList.<Step>of(separateTestStep2),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
ExecutionOrderAwareFakeStep separateTestStep3 =
new ExecutionOrderAwareFakeStep("teststep3", "teststep3", 0, atomicExecutionOrder);
BuildTarget separateTest3Target = BuildTargetFactory.newInstance("//:test3");
FakeTestRule separateTest3 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(separateTest3Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("separateTestStep3OutputDir")),
true, // runTestSeparately
ImmutableList.<Step>of(separateTestStep3),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
ExecutionOrderAwareFakeStep parallelTestStep1 =
new ExecutionOrderAwareFakeStep(
"parallelteststep1", "parallelteststep1", 0, atomicExecutionOrder);
BuildTarget parallelTest1Target = BuildTargetFactory.newInstance("//:paralleltest1");
FakeTestRule parallelTest1 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(parallelTest1Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("parallelTestStep1OutputDir")),
false, // runTestSeparately
ImmutableList.<Step>of(parallelTestStep1),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
ExecutionOrderAwareFakeStep parallelTestStep2 =
new ExecutionOrderAwareFakeStep(
"parallelteststep2", "parallelteststep2", 0, atomicExecutionOrder);
BuildTarget parallelTest2Target = BuildTargetFactory.newInstance("//:paralleltest2");
FakeTestRule parallelTest2 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(parallelTest2Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("parallelTestStep2OutputDir")),
false, // runTestSeparately
ImmutableList.<Step>of(parallelTestStep2),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
ExecutionOrderAwareFakeStep parallelTestStep3 =
new ExecutionOrderAwareFakeStep(
"parallelteststep3", "parallelteststep3", 0, atomicExecutionOrder);
BuildTarget parallelTest3Target = BuildTargetFactory.newInstance("//:paralleltest3");
FakeTestRule parallelTest3 =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(parallelTest3Target),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("parallelTestStep3OutputDir")),
false, // runTestSeparately
ImmutableList.<Step>of(parallelTestStep3),
new Callable<TestResults>() {
@Override
public TestResults call() {
return fakeTestResults;
}
});
// We explicitly use an actual thread pool here; the logic should ensure the
// separate tests are run in the correct order.
ListeningExecutorService service =
MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
FakeBuildEngine fakeBuildEngine =
new FakeBuildEngine(
ImmutableMap.<BuildTarget, BuildResult>builder()
.put(
separateTest1Target,
new BuildResult(separateTest1, BUILT_LOCALLY, CacheResult.skip()))
.put(
separateTest2Target,
new BuildResult(separateTest2, BUILT_LOCALLY, CacheResult.skip()))
.put(
separateTest3Target,
new BuildResult(separateTest3, BUILT_LOCALLY, CacheResult.skip()))
.put(
parallelTest1Target,
new BuildResult(parallelTest1, BUILT_LOCALLY, CacheResult.skip()))
.put(
parallelTest2Target,
new BuildResult(parallelTest2, BUILT_LOCALLY, CacheResult.skip()))
.put(
parallelTest3Target,
new BuildResult(parallelTest3, BUILT_LOCALLY, CacheResult.skip()))
.build(),
ImmutableMap.<BuildTarget, RuleKey>builder()
.put(separateTest1Target, new RuleKey("00"))
.put(separateTest2Target, new RuleKey("00"))
.put(separateTest3Target, new RuleKey("00"))
.put(parallelTest1Target, new RuleKey("00"))
.put(parallelTest2Target, new RuleKey("00"))
.put(parallelTest3Target, new RuleKey("00"))
.build());
ExecutionContext fakeExecutionContext =
TestExecutionContext.newBuilder().setProjectFilesystem(new FakeProjectFilesystem()).build();
DefaultStepRunner stepRunner = new DefaultStepRunner(fakeExecutionContext);
int ret =
TestRunning.runTests(
commandRunnerParams,
ImmutableList.<TestRule>of(
separateTest1,
parallelTest1,
separateTest2,
parallelTest2,
separateTest3,
parallelTest3),
FakeBuildContext.NOOP_CONTEXT,
fakeExecutionContext,
DEFAULT_OPTIONS,
service,
fakeBuildEngine,
stepRunner);
assertThat(ret, equalTo(0));
// The tests not marked as separate could run in any order -- but they must run
// before the separate test steps.
ImmutableSet<Optional<Integer>> expectedParallelStepExecutionOrderSet =
ImmutableSet.<Optional<Integer>>builder()
.add(Optional.of(0))
.add(Optional.of(1))
.add(Optional.of(2))
.add(Optional.of(3))
.add(Optional.of(4))
.add(Optional.of(5))
.build();
ImmutableSet<Optional<Integer>> actualParallelStepExecutionOrderSet =
ImmutableSet.<Optional<Integer>>builder()
.add(parallelTestStep1.getExecutionBeginOrder())
.add(parallelTestStep1.getExecutionEndOrder())
.add(parallelTestStep2.getExecutionBeginOrder())
.add(parallelTestStep2.getExecutionEndOrder())
.add(parallelTestStep3.getExecutionBeginOrder())
.add(parallelTestStep3.getExecutionEndOrder())
.build();
LOG.debug(
"Expected parallel execution order: %s Actual parallel execution order: %s",
expectedParallelStepExecutionOrderSet, actualParallelStepExecutionOrderSet);
// We allow the parallel steps to begin and end in any order (note the thread
// pool of size 3 above), so we use a set.
assertThat(actualParallelStepExecutionOrderSet, equalTo(expectedParallelStepExecutionOrderSet));
// The separate test steps must begin and end in a specific order, so we use a list.
ImmutableList<Optional<Integer>> expectedSeparateStepExecutionOrderList =
ImmutableList.<Optional<Integer>>builder()
.add(Optional.of(6))
.add(Optional.of(7))
.add(Optional.of(8))
.add(Optional.of(9))
.add(Optional.of(10))
.add(Optional.of(11))
.build();
ImmutableList<Optional<Integer>> actualSeparateStepExecutionOrderList =
ImmutableList.<Optional<Integer>>builder()
.add(separateTestStep1.getExecutionBeginOrder())
.add(separateTestStep1.getExecutionEndOrder())
.add(separateTestStep2.getExecutionBeginOrder())
.add(separateTestStep2.getExecutionEndOrder())
.add(separateTestStep3.getExecutionBeginOrder())
.add(separateTestStep3.getExecutionEndOrder())
.build();
LOG.debug(
"Expected separate execution order: %s Actual separate execution order: %s",
expectedSeparateStepExecutionOrderList, actualSeparateStepExecutionOrderList);
assertThat(
actualSeparateStepExecutionOrderList, equalTo(expectedSeparateStepExecutionOrderList));
}
@Test
public void whenSeparateTestFailsThenBuildFails() throws Exception {
CommandRunnerParams commandRunnerParams = CommandRunnerParamsForTesting.builder().build();
final TestResults failingTestResults =
new TestResults(
ImmutableList.of(
new TestCaseSummary(
"TestCase",
ImmutableList.of(
new TestResultSummary(
"TestCaseResult",
"failTest",
ResultType.FAILURE,
5000,
null,
null,
null,
null)))));
BuildTarget failingTestTarget = BuildTargetFactory.newInstance("//:failingtest");
FakeTestRule failingTest =
new FakeTestRule(
BuildRuleParamsFactory.createTrivialBuildRuleParams(failingTestTarget),
new SourcePathResolver(new BuildRuleResolver()),
ImmutableSet.<Label>of(),
Optional.of(Paths.get("failingTestStep1OutputDir")),
true, // runTestSeparately
ImmutableList.<Step>of(),
new Callable<TestResults>() {
@Override
public TestResults call() {
return failingTestResults;
}
});
ListeningExecutorService service =
MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(3));
FakeBuildEngine fakeBuildEngine =
new FakeBuildEngine(
ImmutableMap.of(
failingTestTarget, new BuildResult(failingTest, BUILT_LOCALLY, CacheResult.skip())),
ImmutableMap.of(failingTestTarget, new RuleKey("00")));
ExecutionContext fakeExecutionContext =
TestExecutionContext.newBuilder().setProjectFilesystem(new FakeProjectFilesystem()).build();
DefaultStepRunner stepRunner = new DefaultStepRunner(fakeExecutionContext);
int ret =
TestRunning.runTests(
commandRunnerParams,
ImmutableList.<TestRule>of(failingTest),
FakeBuildContext.NOOP_CONTEXT,
fakeExecutionContext,
DEFAULT_OPTIONS,
service,
fakeBuildEngine,
stepRunner);
assertThat(ret, equalTo(TestRunning.TEST_FAILURES_EXIT_CODE));
}
}
