public SavePoint() {
MockFixture mockFixture = TestRun.mockFixture();
previousTransformedClasses = mockFixture.getTransformedClasses();
previousRedefinedClasses = mockFixture.getRedefinedClasses();
previousCaptureTransformerCount = mockFixture.getCaptureTransformerCount();
previousMockedClasses = mockFixture.getMockedClasses();
previousMockClasses = TestRun.getMockClasses().new SavePoint();
}
public static void rollbackForTestClass() {
SavePoint savePoint = TestRun.getSavePointForTestClass();
if (savePoint != null) {
savePoint.rollback();
TestRun.setSavePointForTestClass(null);
}
}
public synchronized void rollback() {
if (rollbackActions != null) {
for (Runnable action : rollbackActions) {
action.run();
}
}
MockFixture mockFixture = TestRun.mockFixture();
mockFixture.restoreTransformedClasses(previousTransformedClasses);
mockFixture.restoreRedefinedClasses(previousRedefinedClasses);
TestRun.getMockClasses().getRegularMocks().removeInstances(previousMockInstancesCount);
}
private boolean isMatchingInstance(
@Nonnull Object invokedInstance, @Nonnull Expectation expectation) {
ExpectedInvocation invocation = expectation.invocation;
assert invocation.instance != null;
if (isEquivalentInstance(invocation.instance, invokedInstance)) {
return true;
}
if (TestRun.getExecutingTest().isInjectableMock(invokedInstance)) {
return false;
}
if (dynamicMockInstancesToMatch != null) {
if (containsReference(dynamicMockInstancesToMatch, invokedInstance)) {
return false;
}
Class<?> invokedClass = invocation.instance.getClass();
for (Object dynamicMock : dynamicMockInstancesToMatch) {
if (dynamicMock.getClass() == invokedClass) {
return false;
}
}
}
return !invocation.matchInstance && expectation.recordPhase != null;
}
boolean isEquivalentInstance(
@Nonnull Object invocationInstance, @Nonnull Object invokedInstance) {
return invocationInstance == invokedInstance
|| invocationInstance == replacementMap.get(invokedInstance)
|| invocationInstance == instanceMap.get(invokedInstance)
|| invokedInstance == instanceMap.get(invocationInstance)
|| TestRun.getExecutingTest()
.isInvokedInstanceEquivalentToCapturedInstance(invocationInstance, invokedInstance);
}
private boolean isSameMockedClass(@Nullable Object mock1, @Nullable Object mock2) {
if (mock1 == mock2) {
return true;
}
if (mock1 != null && mock2 != null) {
Class<?> mockedClass1 = mock1.getClass();
Class<?> mockedClass2 = mock2.getClass();
if (mockedClass1 == mockedClass2
|| TestRun.getExecutingTest()
.isInvokedInstanceEquivalentToCapturedInstance(mock1, mock2)) {
return true;
}
return TestRun.mockFixture().areCapturedClasses(mockedClass1, mockedClass2);
}
return false;
}
private void runImpl(TestRun run) {
try {
String result = strategy.runTest(run);
if (result == null) {
run.getCallback().complete();
return;
}
ObjectMapper mapper = new ObjectMapper();
ObjectNode resultObject = (ObjectNode) mapper.readTree(result);
String status = resultObject.get("status").asText();
switch (status) {
case "ok":
if (!run.getExpectedExceptions().isEmpty()) {
run.getCallback().error(new AssertionError("Expected exception was not thrown"));
} else {
run.getCallback().complete();
}
break;
case "exception":
{
String stack = resultObject.get("stack").asText();
String exception =
resultObject.has("exception") ? resultObject.get("exception").asText() : null;
Class<?> exceptionClass;
if (exception != null) {
try {
exceptionClass = Class.forName(exception, false, TestRunner.class.getClassLoader());
} catch (ClassNotFoundException e) {
exceptionClass = null;
}
} else {
exceptionClass = null;
}
if (exceptionClass != null) {
Class<?> caught = exceptionClass;
if (run.getExpectedExceptions().stream().anyMatch(e -> e.isAssignableFrom(caught))) {
run.getCallback().complete();
break;
}
}
run.getCallback().error(new AssertionError(exception + "\n" + stack));
break;
}
}
} catch (Exception e) {
run.getCallback().error(e);
}
}
public synchronized void rollback() {
RECORD_OR_REPLAY_LOCK.lock();
try {
MockFixture mockFixture = TestRun.mockFixture();
mockFixture.removeCaptureTransformers(previousCaptureTransformerCount);
mockFixture.restoreTransformedClasses(previousTransformedClasses);
mockFixture.restoreRedefinedClasses(previousRedefinedClasses);
mockFixture.removeMockedClasses(previousMockedClasses);
previousMockClasses.rollback();
} finally {
RECORD_OR_REPLAY_LOCK.unlock();
}
}
/**
* Excutes the selected test run.
*
* @param testRunName name of the test run to execute
*/
private void executeTestRun(String testRunName) {
log.log(Level.INFO, "Executing Test Run '" + testRunName + "'");
// get the selected test run
TestRun testRun = config.getTestRun(testRunName);
// check if the application under test is available
BaseTestCase.testAUTHTTPConnection(testRun.getBaseURL());
// check if all configured servers for this test run are responding
BaseTestCase.testSeleniumHTTPConnection(config, testRun.getTestRunServerIterator());
// set the Application-under-Test parameters in the JUnit Base Class
BaseTestCase.setBaseURL(testRun.getBaseURL());
String msg = "Application under Test:\n" + "BaseURL=" + testRun.getBaseURL();
System.out.println(msg);
log.log(Level.INFO, msg);
// go through all servers defined in the test run and execute the tests
Iterator<TestRun.TestRunServer> selServerIt = testRun.getTestRunServerIterator();
while (selServerIt.hasNext()) {
// get the next server entry from the test run
TestRun.TestRunServer selServerConfig = selServerIt.next();
// set the speed for this server/browser combination
BaseTestCase.setSpeed(selServerConfig.getSpeed());
// get the test server settings
Map<String, String> testServer = config.getTestServers().get(selServerConfig.getName());
// set the host,port and browser used for the next tests
BaseTestCase.isExternallySetup = true;
BaseTestCase.setSeleniumServerHost(testServer.get("host"));
BaseTestCase.setSeleniumServerPort(testServer.get("port"));
msg = "Platform is: " + System.getProperty("os.name");
System.out.println(msg);
log.log(Level.INFO, msg);
Map<String, Platform> platforms = config.getPlatforms();
for (Platform platformToCheck : platforms.values()) {
if (platformToCheck.isValid()) {
executeForPlatform(selServerConfig, testServer, platformToCheck);
}
}
}
}
boolean isToBeMatchedOnInstance(@Nullable Object mock, @Nonnull String mockNameAndDesc) {
if (mock == null || mockNameAndDesc.charAt(0) == '<') {
return false;
}
if (dynamicMockInstancesToMatch != null
&& containsReference(dynamicMockInstancesToMatch, mock)) {
return true;
}
if (mockedTypesToMatchOnInstances != null) {
Class<?> mockedClass = GeneratedClasses.getMockedClass(mock);
if (containsReference(mockedTypesToMatchOnInstances, mockedClass)) {
return true;
}
} else if (TestRun.getExecutingTest().isInjectableMock(mock)) {
return true;
}
return false;
}
@Override
boolean isToExecuteRealObjectOverride(@Nonnull Object instance) {
return !TestRun.getExecutingTest().isMockedInstance(instance);
}
@Override
boolean isToExecuteRealImplementation(@Nullable Object instance) {
return instance == null || !TestRun.getExecutingTest().isMockedInstance(instance);
}
@Override
boolean isWithRealImplementation(@Nullable Object instance) {
return instance == null || !TestRun.getExecutingTest().isInjectableMock(instance);
}
@Override
boolean isToExecuteRealImplementation(@Nullable Object instance) {
return instance != null && !TestRun.mockFixture().isInstanceOfMockedClass(instance);
}
private void discardStateForCorrespondingMockClassIfAny(@Nonnull Class<?> redefinedClass) {
String mockClassesInternalNames = realClassesToMockClasses.remove(redefinedClass);
TestRun.getMockStates().removeClassState(redefinedClass, mockClassesInternalNames);
}
void registerAssignment() {
int testId = TestRun.getTestId();
testIdsToAssignments.put(testId, Boolean.TRUE);
writeCount++;
}
/**
* Discards any mocks set up for the specified classes that are currently in effect, for all test
* scopes: the current test method (if any), the current test (which starts with the first
* "before" method and continues until the last "after" method), the current test class (which
* includes all code from the first "before class" method to the last "after class" method), and
* the current test suite.
*
* <p>Notice that if one of the given real classes has a mock class applied at the level of the
* test class, calling this method would negate the application of that mock class. JMockit will
* automatically restore classes mocked by a test at the end of its execution, as well as all
* classes mocked for the test class as a whole (through a "before class" method or an
* {@code @UsingMocksAndStubs} annotation) before the first test in the next test class is
* executed.
*
* @param realClasses one or more real classes from production code, which may have mocked methods
*/
public static void tearDownMocks(Class<?>... realClasses) {
Set<Class<?>> classesToRestore = new HashSet<Class<?>>();
Collections.addAll(classesToRestore, realClasses);
TestRun.mockFixture().restoreAndRemoveRedefinedClasses(classesToRestore);
}
void registerRead() {
int testId = TestRun.getTestId();
testIdsToAssignments.put(testId, null);
readCount++;
}
public SavePoint() {
MockFixture mockFixture = TestRun.mockFixture();
previousTransformedClasses = mockFixture.getTransformedClasses();
previousRedefinedClasses = mockFixture.getRedefinedClasses();
previousMockInstancesCount = TestRun.getMockClasses().getRegularMocks().getInstanceCount();
}
public static void registerNewActiveSavePoint() {
TestRun.setSavePointForTestClass(new SavePoint());
}
@SuppressWarnings("PMD.EmptyCatchBlock")
public static int runTests(
final CommandRunnerParams params,
Iterable<TestRule> tests,
BuildContext buildContext,
ExecutionContext executionContext,
final TestRunningOptions options,
ListeningExecutorService service,
BuildEngine buildEngine,
final StepRunner stepRunner)
throws IOException, ExecutionException, InterruptedException {
if (options.isUsingOneTimeOutputDirectories()) {
BuckConstant.setOneTimeTestSubdirectory(UUID.randomUUID().toString());
}
ImmutableSet<JavaLibrary> rulesUnderTest;
// If needed, we first run instrumentation on the class files.
if (options.isCodeCoverageEnabled()) {
rulesUnderTest = getRulesUnderTest(tests);
if (!rulesUnderTest.isEmpty()) {
try {
stepRunner.runStepForBuildTarget(
new MakeCleanDirectoryStep(JUnitStep.JACOCO_OUTPUT_DIR),
Optional.<BuildTarget>absent());
} catch (StepFailedException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
}
} else {
rulesUnderTest = ImmutableSet.of();
}
final ImmutableSet<String> testTargets =
FluentIterable.from(tests)
.transform(HasBuildTarget.TO_TARGET)
.transform(Functions.toStringFunction())
.toSet();
final int totalNumberOfTests = Iterables.size(tests);
params
.getBuckEventBus()
.post(
TestRunEvent.started(
options.isRunAllTests(),
options.getTestSelectorList(),
options.shouldExplainTestSelectorList(),
testTargets));
// Start running all of the tests. The result of each java_test() rule is represented as a
// ListenableFuture.
List<ListenableFuture<TestResults>> results = Lists.newArrayList();
// Unless `--verbose 0` is specified, print out test results as they become available.
// Failures with the ListenableFuture should always be printed, as they indicate an error with
// Buck, not the test being run.
Verbosity verbosity = params.getConsole().getVerbosity();
final boolean printTestResults = (verbosity != Verbosity.SILENT);
// For grouping results!
final TestResultsGrouper grouper;
if (options.isIgnoreFailingDependencies()) {
grouper = new TestResultsGrouper(tests);
} else {
grouper = null;
}
TestRuleKeyFileHelper testRuleKeyFileHelper =
new TestRuleKeyFileHelper(executionContext.getProjectFilesystem(), buildEngine);
final AtomicInteger lastReportedTestSequenceNumber = new AtomicInteger();
final List<TestRun> separateTestRuns = Lists.newArrayList();
List<TestRun> parallelTestRuns = Lists.newArrayList();
for (final TestRule test : tests) {
// Determine whether the test needs to be executed.
boolean isTestRunRequired;
isTestRunRequired =
isTestRunRequiredForTest(
test,
buildEngine,
executionContext,
testRuleKeyFileHelper,
options.isResultsCacheEnabled(),
!options.getTestSelectorList().isEmpty());
List<Step> steps;
if (isTestRunRequired) {
params.getBuckEventBus().post(IndividualTestEvent.started(testTargets));
ImmutableList.Builder<Step> stepsBuilder = ImmutableList.builder();
Preconditions.checkState(buildEngine.isRuleBuilt(test.getBuildTarget()));
final Map<String, UUID> testUUIDMap = new HashMap<>();
List<Step> testSteps =
test.runTests(
buildContext,
executionContext,
options.isDryRun(),
options.isShufflingTests(),
options.getTestSelectorList(),
new TestRule.TestReportingCallback() {
@Override
public void testsDidBegin() {
LOG.debug("Tests for rule %s began", test.getBuildTarget());
}
@Override
public void testDidBegin(String testCaseName, String testName) {
LOG.debug(
"Test rule %s test case %s test name %s began",
test.getBuildTarget(), testCaseName, testName);
UUID testUUID = UUID.randomUUID();
// UUID is immutable and thread-safe as of Java 7, so it's
// safe to stash in a map and use later:
//
// http://bugs.java.com/view_bug.do?bug_id=6611830
testUUIDMap.put(testCaseName + ":" + testName, testUUID);
params
.getBuckEventBus()
.post(TestSummaryEvent.started(testUUID, testCaseName, testName));
}
@Override
public void testDidEnd(TestResultSummary testResultSummary) {
LOG.debug(
"Test rule %s test did end: %s", test.getBuildTarget(), testResultSummary);
UUID testUUID =
testUUIDMap.get(
testResultSummary.getTestCaseName()
+ ":"
+ testResultSummary.getTestName());
Preconditions.checkNotNull(testUUID);
params
.getBuckEventBus()
.post(TestSummaryEvent.finished(testUUID, testResultSummary));
}
@Override
public void testsDidEnd(List<TestCaseSummary> testCaseSummaries) {
LOG.debug(
"Test rule %s tests did end: %s", test.getBuildTarget(), testCaseSummaries);
}
});
if (!testSteps.isEmpty()) {
stepsBuilder.addAll(testSteps);
stepsBuilder.add(testRuleKeyFileHelper.createRuleKeyInDirStep(test));
}
steps = stepsBuilder.build();
} else {
steps = ImmutableList.of();
}
TestRun testRun =
TestRun.of(
test,
steps,
getCachingStatusTransformingCallable(
isTestRunRequired,
test.interpretTestResults(
executionContext,
/*isUsingTestSelectors*/ !options.getTestSelectorList().isEmpty(),
/*isDryRun*/ options.isDryRun())));
// Always run the commands, even if the list of commands as empty. There may be zero
// commands because the rule is cached, but its results must still be processed.
if (test.runTestSeparately()) {
LOG.debug("Running test %s in serial", test);
separateTestRuns.add(testRun);
} else {
LOG.debug("Running test %s in parallel", test);
parallelTestRuns.add(testRun);
}
}
final StepRunner.StepRunningCallback testStepRunningCallback =
new StepRunner.StepRunningCallback() {
@Override
public void stepsWillRun(Optional<BuildTarget> buildTarget) {
Preconditions.checkState(buildTarget.isPresent());
LOG.debug("Test steps will run for %s", buildTarget);
params.getBuckEventBus().post(TestRuleEvent.started(buildTarget.get()));
}
@Override
public void stepsDidRun(Optional<BuildTarget> buildTarget) {
Preconditions.checkState(buildTarget.isPresent());
LOG.debug("Test steps did run for %s", buildTarget);
params.getBuckEventBus().post(TestRuleEvent.finished(buildTarget.get()));
}
};
for (TestRun testRun : parallelTestRuns) {
ListenableFuture<TestResults> testResults =
stepRunner.runStepsAndYieldResult(
testRun.getSteps(),
testRun.getTestResultsCallable(),
Optional.of(testRun.getTest().getBuildTarget()),
service,
testStepRunningCallback);
results.add(
transformTestResults(
params,
testResults,
grouper,
testRun.getTest(),
testTargets,
printTestResults,
lastReportedTestSequenceNumber,
totalNumberOfTests));
}
ListenableFuture<List<TestResults>> parallelTestStepsFuture = Futures.allAsList(results);
final List<TestResults> completedResults = Lists.newArrayList();
final ListeningExecutorService directExecutorService = MoreExecutors.newDirectExecutorService();
ListenableFuture<Void> uberFuture =
stepRunner.addCallback(
parallelTestStepsFuture,
new FutureCallback<List<TestResults>>() {
@Override
public void onSuccess(List<TestResults> parallelTestResults) {
LOG.debug("Parallel tests completed, running separate tests...");
completedResults.addAll(parallelTestResults);
List<ListenableFuture<TestResults>> separateResultsList = Lists.newArrayList();
for (TestRun testRun : separateTestRuns) {
separateResultsList.add(
transformTestResults(
params,
stepRunner.runStepsAndYieldResult(
testRun.getSteps(),
testRun.getTestResultsCallable(),
Optional.of(testRun.getTest().getBuildTarget()),
directExecutorService,
testStepRunningCallback),
grouper,
testRun.getTest(),
testTargets,
printTestResults,
lastReportedTestSequenceNumber,
totalNumberOfTests));
}
ListenableFuture<List<TestResults>> serialResults =
Futures.allAsList(separateResultsList);
try {
completedResults.addAll(serialResults.get());
} catch (ExecutionException e) {
LOG.error(e, "Error fetching serial test results");
throw new HumanReadableException(e, "Error fetching serial test results");
} catch (InterruptedException e) {
LOG.error(e, "Interrupted fetching serial test results");
try {
serialResults.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw new HumanReadableException(e, "Test cancelled");
}
LOG.debug("Done running serial tests.");
}
@Override
public void onFailure(Throwable e) {
LOG.error(e, "Parallel tests failed, not running serial tests");
throw new HumanReadableException(e, "Parallel tests failed");
}
},
directExecutorService);
try {
// Block until all the tests have finished running.
uberFuture.get();
} catch (ExecutionException e) {
e.printStackTrace(params.getConsole().getStdErr());
return 1;
} catch (InterruptedException e) {
try {
uberFuture.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw e;
}
params.getBuckEventBus().post(TestRunEvent.finished(testTargets, completedResults));
// Write out the results as XML, if requested.
Optional<String> path = options.getPathToXmlTestOutput();
if (path.isPresent()) {
try (Writer writer = Files.newWriter(new File(path.get()), Charsets.UTF_8)) {
writeXmlOutput(completedResults, writer);
}
}
// Generate the code coverage report.
if (options.isCodeCoverageEnabled() && !rulesUnderTest.isEmpty()) {
try {
Optional<DefaultJavaPackageFinder> defaultJavaPackageFinderOptional =
Optional.fromNullable(params.getBuckConfig().createDefaultJavaPackageFinder());
stepRunner.runStepForBuildTarget(
getReportCommand(
rulesUnderTest,
defaultJavaPackageFinderOptional,
params.getRepository().getFilesystem(),
JUnitStep.JACOCO_OUTPUT_DIR,
options.getCoverageReportFormat()),
Optional.<BuildTarget>absent());
} catch (StepFailedException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
}
boolean failures =
Iterables.any(
completedResults,
new Predicate<TestResults>() {
@Override
public boolean apply(TestResults results) {
LOG.debug("Checking result %s for failure", results);
return !results.isSuccess();
}
});
return failures ? TEST_FAILURES_EXIT_CODE : 0;
}
/**
* Discards any mocks currently in effect, for all test scopes: the current test method (if any),
* the current test (which starts with the first "before" method and continues until the last
* "after" method), the current test class (which includes all code from the first "before class"
* method to the last "after class" method), and the current test suite.
*
* <p>Notice that a call to this method will tear down <em>all</em> mock classes that were applied
* through use of the Mockups API that are still in effect, as well as any mock classes or stubs
* applied to the current test class through {@code @UsingMocksAndStubs}. In other words, it would
* effectively prevent mocks to be set up at the test class and test suite levels. So, use it only
* if necessary and if it won't discard mock classes that should remain in effect. Consider using
* {@link #tearDownMocks(Class...)} instead, which lets you restrict the set of real classes to be
* restored.
*
* <p>JMockit will automatically restore classes mocked by a test at the end of its execution, as
* well as all classes mocked for the test class as a whole (through a "before class" method or an
* {@code @UsingMocksAndStubs} annotation) before the first test in the next test class is
* executed.
*
* @see <a
* href="http://code.google.com/p/jmockit/source/browse/trunk/main/test/mockit/MockAnnotationsTest.java#450">
* Example</a>
*/
public static void tearDownMocks() {
TestRun.mockFixture().restoreAndRemoveRedefinedClasses(null);
TestRun.getMockClasses().getRegularMocks().discardInstances();
}
