private void updateResultsForAnyExternalFailures(
TestOutcome testOutcome, TestOutcome scenarioOutcome) {
if (rowResultsAreInconsistantWithOverallResult(testOutcome)) {
testOutcome.getDataTable().getRows().get(0).updateResult(testOutcome.getResult());
scenarioOutcome.addFailingExternalStep(
new AssertionError(testOutcome.getTestFailureMessage()));
}
}
@Test
public void a_test_run_with_only_successful_tests_is_successful() {
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 1"));
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 2"));
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 3"));
assertThat(testOutcome.getResult(), is(TestResult.SUCCESS));
}
@Test
public void the_acceptance_test_case_is_successful_if_all_the_tests_are_successful() {
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 1"));
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 2"));
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 3"));
assertThat(testOutcome.getResult(), is(SUCCESS));
assertThat(testOutcome.isSuccess(), is(true));
}
@Test
public void the_acceptance_test_case_is_ignored_if_all_test_cases_are_ignored() {
testOutcome.recordStep(forAnIgnoredTestStepCalled("Step 1"));
testOutcome.recordStep(forAnIgnoredTestStepCalled("Step 2"));
testOutcome.recordStep(forAnIgnoredTestStepCalled("Step 3"));
testOutcome.recordStep(forAnIgnoredTestStepCalled("Step 4"));
assertThat(testOutcome.getResult(), is(IGNORED));
}
@Test
public void the_acceptance_test_case_is_failing_if_multiple_tests_have_failed() {
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 1"));
testOutcome.recordStep(forAFailingTestStepCalled("Step 2", new AssertionError("Oh bother!")));
testOutcome.recordStep(forAFailingTestStepCalled("Step 3", new AssertionError("Oh bother!")));
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 4"));
assertThat(testOutcome.getResult(), is(FAILURE));
}
@Test
public void the_acceptance_test_case_is_a_failure_if_one_test_has_failed() {
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 1"));
testOutcome.recordStep(forAFailingTestStepCalled("Step 2", new AssertionError("Oh bother!")));
testOutcome.recordStep(forASkippedTestStepCalled("Step 3"));
assertThat(testOutcome.getResult(), is(FAILURE));
assertThat(testOutcome.isFailure(), is(true));
}
@Test
public void
the_acceptance_test_case_is_failing_if_there_is_a_failure_even_with_pending_test_cases() {
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 1"));
testOutcome.recordStep(forAPendingTestStepCalled("Step 2"));
testOutcome.recordStep(forAFailingTestStepCalled("Step 3", new AssertionError("Oh bother!")));
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 4"));
assertThat(testOutcome.getResult(), is(FAILURE));
}
@Test
public void
the_acceptance_test_case_is_pending_if_at_least_one_test_is_pending_and_none_have_failed() {
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 1"));
testOutcome.recordStep(forAPendingTestStepCalled("Step 2"));
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 3"));
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 4"));
assertThat(testOutcome.getResult(), is(PENDING));
}
@Test
public void
if_one_test_is_ignored_among_others_it_will_not_affect_the_outcome_for_successful_tests() {
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 1"));
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 2"));
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 3"));
testOutcome.recordStep(forAnIgnoredTestStepCalled("Step 4"));
assertThat(testOutcome.getResult(), is(SUCCESS));
}
private List<TestOutcome> outcomesWithResults(
List<? extends TestOutcome> outcomes, TestResult... possibleResults) {
List<TestOutcome> validOutcomes = Lists.newArrayList();
List<TestResult> possibleResultsList = Arrays.asList(possibleResults);
for (TestOutcome outcome : outcomes) {
if (possibleResultsList.contains(outcome.getResult())) {
validOutcomes.add(outcome);
}
}
return validOutcomes;
}
@Test
public void
if_one_test_is_ignored_among_others_it_will_not_affect_the_outcome_for_failing_tests() {
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 1"));
testOutcome.recordStep(forAPendingTestStepCalled("Step 2"));
testOutcome.recordStep(forAFailingTestStepCalled("Step 3", new AssertionError("Oh bother!")));
testOutcome.recordStep(forAnIgnoredTestStepCalled("Step 4"));
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 5"));
assertThat(testOutcome.getResult(), is(FAILURE));
}
@Test
public void a_test_run_with_a_nested_group_containing_a_failure_is_a_failure() {
testOutcome.startGroup("A group");
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 1"));
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 2"));
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 3"));
testOutcome.startGroup("Another group");
testOutcome.recordStep(forASuccessfulTestStepCalled("Step 4"));
testOutcome.recordStep(forAFailingTestStepCalled("Step 5", new AssertionError("Oh bother!")));
testOutcome.endGroup();
testOutcome.endGroup();
assertThat(testOutcome.getResult(), is(TestResult.FAILURE));
}
private void recordTestOutcomeAsSteps(TestOutcome testOutcome, TestOutcome scenarioOutcome) {
TestStep nestedStep =
TestStep.forStepCalled(testOutcome.getTitle()).withResult(testOutcome.getResult());
List<TestStep> testSteps = testOutcome.getTestSteps();
if (testOutcome.getTestFailureCause() != null) {
nestedStep.failedWith(testOutcome.getTestFailureCause().toException());
}
if (!testSteps.isEmpty()) {
for (TestStep nextStep : testSteps) {
nextStep.setDescription(
normalizeTestStepDescription(
nextStep.getDescription(), scenarioOutcome.getTestSteps().size() + 1));
nestedStep.addChildStep(nextStep);
}
}
scenarioOutcome.recordStep(nestedStep);
}
private void recordTestOutcomeAsSteps(TestOutcome testOutcome, TestOutcome scenarioOutcome) {
final String name = alternativeMethodName(testOutcome);
TestStep nestedStep = TestStep.forStepCalled(name).withResult(testOutcome.getResult());
List<TestStep> testSteps = testOutcome.getTestSteps();
if (testOutcome.getTestFailureCause() != null) {
nestedStep.failedWith(testOutcome.getTestFailureCause().toException());
}
if (!testSteps.isEmpty()) {
for (TestStep nextStep : testSteps) {
nextStep.setDescription(
normalizeTestStepDescription(
nextStep.getDescription(), scenarioOutcome.getTestSteps().size() + 1));
nestedStep.addChildStep(nextStep);
nestedStep.setDuration(nextStep.getDuration() + nestedStep.getDuration());
}
}
if (nestedStep.getDuration() == 0) {
nestedStep.setDuration(testOutcome.getDuration());
}
scenarioOutcome.recordStep(nestedStep);
}
public TestResult convert(final TestOutcome step) {
return step.getResult();
}
@Test
public void the_acceptance_test_case_is_considered_pending_if_has_no_steps() {
assertThat(testOutcome.getResult(), is(PENDING));
}
private boolean rowResultsAreInconsistantWithOverallResult(TestOutcome testOutcome) {
TestResult overallRowResult = overallResultFrom(testOutcome.getDataTable().getRows());
return (testOutcome.isError() || testOutcome.isFailure())
&& (!testOutcome.getDataTable().getRows().isEmpty())
&& (testOutcome.getResult() != overallRowResult);
}
@Test
public void a_test_case_with_no_steps_should_be_considered_pending() {
assertThat(testOutcome.getResult(), is(PENDING));
assertThat(testOutcome.isSuccess(), is(false));
assertThat(testOutcome.isPending(), is(true));
}