protected void runFullSuite(final RunNotifier notifier) {
JUnitHelper helper = createJUnitHelper(notifier);
try {
helper.assertSuitePasses(suiteName, suiteFilter, excludeSuiteFilter);
} catch (AssertionFailedError e) {
notifier.fireTestFailure(new Failure(Description.createSuiteDescription(suiteClass), e));
} catch (Exception e) {
notifier.fireTestFailure(new Failure(Description.createSuiteDescription(suiteClass), e));
}
}
@Override
protected void runChild(String test, RunNotifier notifier) {
JUnitHelper helper = createJUnitHelper(notifier);
try {
helper.assertTestPasses(test);
} catch (AssertionFailedError e) {
notifier.fireTestFailure(new Failure(Description.createSuiteDescription(suiteClass), e));
} catch (Exception e) {
notifier.fireTestFailure(new Failure(Description.createSuiteDescription(suiteClass), e));
}
}
@Override
protected void runChild(TestCase testCase, RunNotifier notifier) {
notifier.fireTestStarted(testCase.name);
ByteArrayOutputStream out = new ByteArrayOutputStream();
try {
PolyglotEngine vm =
PolyglotEngine.newBuilder()
.setIn(
new ByteArrayInputStream(repeat(testCase.testInput, repeats).getBytes("UTF-8")))
.setOut(out)
.build();
String script = readAllLines(testCase.path);
PrintWriter printer = new PrintWriter(out);
SLLanguage.run(vm, testCase.path, null, printer, repeats, builtins);
printer.flush();
String actualOutput = new String(out.toByteArray());
Assert.assertEquals(script, repeat(testCase.expectedOutput, repeats), actualOutput);
} catch (Throwable ex) {
notifier.fireTestFailure(
new Failure(
testCase.name, new IllegalStateException("Cannot run " + testCase.sourceName, ex)));
} finally {
notifier.fireTestFinished(testCase.name);
}
}
@Override
protected void runChild(Interaction interaction, RunNotifier notifier) {
final Description description = describeChild(interaction);
notifier.fireTestStarted(description);
if (interaction.providerState().isDefined()) {
final Optional<FrameworkMethod> initializationMethod =
findProvderStateInitializationMethod(interaction.providerState().get());
if (initializationMethod.isPresent()) {
try {
invokeMethod(initializationMethod.get());
testPact(interaction);
notifier.fireTestFinished(description);
return;
} catch (Exception ex) {
notifier.fireTestFailure(new Failure(description, ex));
return;
}
} else {
notifier.fireTestIgnored(description);
return;
}
}
notifier.fireTestIgnored(description);
}
/**
* @param frameworkMethod
* @param notifier
*/
@Override
protected void runChild(FrameworkMethod frameworkMethod, RunNotifier notifier) {
try {
beforeRunChild();
try {
// This will execute the createTest method below, the activeContainer handling relies on
// this.
System.out.println(
"START running test "
+ frameworkMethod.getName()
+ " for thread "
+ Thread.currentThread().toString());
super.runChild(frameworkMethod, notifier);
System.out.println(
"END running test "
+ frameworkMethod.getName()
+ " for thread "
+ Thread.currentThread().toString());
} finally {
afterRunChild();
}
} catch (Throwable e) {
e.printStackTrace();
Description description = describeChild(frameworkMethod);
notifier.fireTestFailure(new Failure(description, e));
notifier.fireTestFinished(description);
}
}
protected void notifyOfTestSystemException(String testSystemName, Throwable cause) {
if (cause != null) {
Exception e =
new Exception("Exception while executing tests using: " + testSystemName, cause);
notifier.fireTestFailure(new Failure(Description.createSuiteDescription(mainClass), e));
}
}
/** {@inheritDoc} */
private void notifyTestFailed(final Description junitDescription, final Throwable cause) {
if (junitRunNotifier != null && junitDescription != null) {
log.debug(junitDescription.getDisplayName() + " notify running TestFailed");
log.debug(junitDescription.getDisplayName() + " notify TestFailed");
junitRunNotifier.fireTestFailure(new Failure(junitDescription, cause));
}
}
@Override
public void fireTestFailure(Failure failure) {
log.debug("Test failed: " + failure);
testStartAlreadyFired = false;
testFailed = true;
lastFailure = failure;
retryAwareRunNotifier.fireTestFailure(failure);
}
private void runMethodInvoker(
RunNotifier notifier,
Description description,
Statement methodInvoker,
Description methodWithParams) {
try {
methodInvoker.evaluate();
} catch (Throwable e) {
notifier.fireTestFailure(new Failure(methodWithParams, e));
}
}
@Override
public void close() {
if (completedTests != totalNumberOfTests) {
String msg =
String.format(
"Not all tests executed. Completed %s of %s tests.",
completedTests, totalNumberOfTests);
Exception e = new Exception(msg);
notifier.fireTestFailure(new Failure(Description.createSuiteDescription(mainClass), e));
}
}
/**
* Executes the passed in test case. Instrumentation is added according to the name of the file as
* explained in {@link #createTests(Class)}. Note that this code is not generalizable.
*/
@Override
protected void runChild(InstrumentTestCase testCase, RunNotifier notifier) {
// TODO Current tests are hard-coded, automate this eventually
notifier.fireTestStarted(testCase.name);
ByteArrayOutputStream out = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(out);
PolyglotEngine vm = null;
try {
// We use the name of the file to determine what visitor to attach to it.
if (testCase.baseName.endsWith(ASSIGNMENT_VALUE_SUFFIX)) {
// Set up the execution context for Simple and register our two listeners
vm =
PolyglotEngine.newBuilder()
.setIn(new ByteArrayInputStream(testCase.testInput.getBytes("UTF-8")))
.setOut(out)
.build();
final String src = readAllLines(testCase.path);
vm.eval(Source.fromText(src, testCase.path.toString()).withMimeType("application/x-sl"));
PolyglotEngine.Value main = vm.findGlobalSymbol("main");
main.execute();
} else {
notifier.fireTestFailure(
new Failure(
testCase.name, new UnsupportedOperationException("No instrumentation found.")));
}
ps.flush();
String actualOutput = new String(out.toByteArray());
Assert.assertEquals(testCase.expectedOutput, actualOutput);
} catch (Throwable ex) {
notifier.fireTestFailure(new Failure(testCase.name, ex));
} finally {
if (vm != null) {
vm.dispose();
}
notifier.fireTestFinished(testCase.name);
}
}
@Override
public void testComplete(WikiTestPage test, TestSummary testSummary) {
increaseCompletedTests();
if (firstFailure != null) {
notifier.fireTestFailure(new Failure(descriptionFor(test), firstFailure));
} else if (test.isTestPage()) {
if (testSummary.getExceptions() > 0) {
notifier.fireTestFailure(
new Failure(
descriptionFor(test),
new Exception("Exception occurred on page " + test.getFullPath())));
} else if (testSummary.getWrong() > 0) {
notifier.fireTestFailure(
new Failure(
descriptionFor(test),
new AssertionError("Test failures occurred on page " + test.getFullPath())));
} else {
notifier.fireTestFinished(descriptionFor(test));
}
}
}
@Override
protected void runChild(FrameworkMethod method, RunNotifier notifier) {
Description description = describeChild(method);
if (method.getAnnotation(Ignore.class) != null) {
notifier.fireTestIgnored(description);
return;
}
String methodName = method.getName();
String allowedMethodName = methodNames.get(description);
if (methodName != null && !methodName.equals(allowedMethodName)) {
// notifier.fireTestIgnored(description);
return;
}
try {
notifier.fireTestStarted(description);
HttpURLConnection connection = getUrl(methodName, "POST");
handleError(connection);
String enc = connection.getContentEncoding();
if (enc == null) {
enc = "ISO-8859-1";
}
InputStream is = connection.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(is, Charset.forName(enc)));
String line = null;
while ((line = reader.readLine()) != null) {
if (line.startsWith("E")) {
System.err.println(line.substring(1).trim());
} else if (line.startsWith("O")) {
System.out.println(line.substring(1).trim());
} else if (line.startsWith("RSUCCESS")) {
break;
} else if (line.startsWith("RERROR")) {
StringBuilder error = new StringBuilder(line.substring(6));
while ((line = reader.readLine()) != null) {
error.append(line).append("\n");
}
throw new AssertionFailedError(error.toString());
} else {
log.error("Protocol error in response: {}", line);
}
}
is.close();
connection.disconnect();
} catch (Throwable e) {
e.printStackTrace();
notifier.fireTestFailure(new Failure(description, e));
} finally {
notifier.fireTestFinished(description);
}
}
@Override
public void evaluate() throws Throwable {
Error error = error();
notifier.addListener(listener);
try {
log.debug(displayName(description) + " STARTED");
for (int retry = 1; retry <= retryCount(); retry++) {
gotFailure = false;
onEnter(retry);
try {
log.debug(displayName(description) + " retry " + retry);
base.evaluate();
} catch (AssumptionViolatedException cause) {
Throwable t = new Throwable("On retry " + retry).initCause(cause);
error.addSuppressed(t);
notifier.fireTestAssumptionFailed(new Failure(description, t));
} catch (Throwable cause) {
// Repeat annotation is on method (else the Throwable is not raised up to here)
Throwable t = new Throwable("On retry " + retry).initCause(cause);
error.addSuppressed(t);
if (returnOnFailure()) {
notifier.fireTestFailure(new Failure(description, t));
} else {
gotFailure = true;
log.debug(displayName(description) + " FAILURE SWALLOW");
log.trace(t, t);
}
} finally {
onLeave();
}
if (gotFailure && returnOnFailure()) {
log.debug(displayName(description) + " returnOnFailure");
return;
}
if (!gotFailure && returnOnSuccess()) {
log.debug(displayName(description) + " returnOnSuccess");
return;
}
}
} finally {
log.debug(displayName(description) + " FINISHED");
notifier.removeListener(listener);
}
log.trace("throw " + error);
throw error;
}
@Override
public void run(RunNotifier notifier) {
filteredTests.foreachDoEffect(
p -> {
Description desc = p._3();
notifier.fireTestStarted(desc);
CheckResult result = checkProperty(p._1(), p._2());
try {
CheckResult.summaryEx.showS(result);
} catch (Throwable t) {
notifier.fireTestFailure(new Failure(desc, t));
}
notifier.fireTestFinished(desc);
});
}
@Override
public void fireTestFailure(Failure failure) {
console.error("Failure: ", failure.getDescription().getDisplayName());
Throwable exception = failure.getException();
if (exception == null) {
console.error(failure.getMessage());
} else {
StackTraceFilterer filterer = new DefaultStackTraceFilterer();
filterer.setCutOffPackage("org.junit");
filterer.filter(exception, true);
StringWriter sw = new StringWriter();
PrintWriter ps = new PrintWriter(sw);
exception.printStackTrace(ps);
console.error("", sw.toString());
}
super.fireTestFailure(failure);
}
private void runFailure(Description failDesc, Throwable cause, RunNotifier notifier) {
notifier.fireTestStarted(failDesc);
notifier.fireTestFailure(new Failure(failDesc, cause));
notifier.fireTestFinished(failDesc);
}
// Implement junit.framework.TestListener
public void addError(Test test, Throwable t) {
Failure failure = new Failure(asDescription(test), t);
fNotifier.fireTestFailure(failure);
}
public void failed(String step, Throwable e) {
currentStep = getStepDescription(step);
notifier.fireTestStarted(currentStep);
notifier.fireTestFailure(new Failure(currentStep, e));
finishedDescriptions.add(currentStep);
}
@Override
public void evaluate() throws Throwable {
ProcessBuilder builder = new ProcessBuilder();
builder.command(buildCommand());
builder.redirectErrorStream(true);
builder.directory(testCase.getWorkingDirectory());
System.out.println("Working directory: " + SystemProperty.INSTANCE.getProperty("user.dir"));
System.out.println("Command: " + builder.command());
Description parentDescription = runner.getDescription();
final Process process = builder.start();
try {
Scanner scanner = testCase.startTest(process);
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
if (line.startsWith(JUNIT_OUTPUT_DELIMITER)) {
System.out.println(line);
String[] tokens = line.split(JUNIT_OUTPUT_DELIMITER_REGEXP);
String event = tokens[1];
if ("testRunStarted".equals(event)) {
notifier.fireTestRunStarted(parentDescription);
} else if ("testRunFinished".equals(event)) {
notifier.fireTestRunFinished(result);
} else if ("testStarted".equals(event)) {
Description ofTest = findDescription(parentDescription, tokens[2]);
notifier.fireTestStarted(ofTest);
} else if ("testFinished".equals(event)) {
Description ofTest = findDescription(parentDescription, tokens[2]);
notifier.fireTestFinished(ofTest);
} else if ("testFailure".equals(event)) {
List<String> stackTrace =
tokens.length > 5
? Arrays.asList(tokens).subList(5, tokens.length - 1)
: Collections.<String>emptyList();
Failure failure =
toFailure(parentDescription, tokens[2], tokens[3], tokens[4], stackTrace);
notifier.fireTestFailure(failure);
} else if ("testAssumptionFailure".equals(event)) {
List<String> stackTrace =
tokens.length > 5
? Arrays.asList(tokens).subList(5, tokens.length - 1)
: Collections.<String>emptyList();
Failure failure =
toFailure(parentDescription, tokens[2], tokens[3], tokens[4], stackTrace);
notifier.fireTestAssumptionFailed(failure);
} else if ("testIgnored".equals(event)) {
Description ofTest = findDescription(parentDescription, tokens[2]);
notifier.fireTestIgnored(ofTest);
}
} else {
System.out.println(line);
}
}
} catch (Throwable t) {
System.err.println("Failed to execute test");
t.printStackTrace();
throw t;
} finally {
try {
testCase.endTest(process);
} finally {
Timer timer = new Timer();
timer.schedule(
new TimerTask() {
@Override
public void run() {
process.destroy();
}
},
10 * 1000);
try {
process.waitFor();
} catch (InterruptedException e) {
// ignore
}
timer.cancel();
}
}
}
private void runCause(Throwable child, RunNotifier notifier) {
Description description = describeCause(child);
notifier.fireTestStarted(description);
notifier.fireTestFailure(new Failure(description, child));
notifier.fireTestFinished(description);
}