@Test
public void
a_test_outcome_should_not_inject_issue__links_from_the_Issue_annotation_if_not_configured() {
TestOutcome outcome = TestOutcome.forTest("should_do_this", SomeTestScenario.class);
assertThat(outcome.getFormattedIssues(), is("(#ISSUE-123)"));
}
@Test
public void a_test_outcome_should_know_what_issues_there_are_in_the_title() {
TestOutcome outcome =
TestOutcome.forTest("should_do_this", SomeAnnotatedTestScenarioWithAnIssue.class);
assertThat(outcome.getIssues(), hasItem("#ISSUE-123"));
}
@Test
public void
a_test_outcome_should_also_inject_issue_links_from_the_Issue_annotation_when_only_at_the_class_level() {
TestOutcome outcome = TestOutcome.forTest("should_do_something_else", SomeTestScenario.class);
assertThat(outcome.getFormattedIssues(), is("(#ISSUE-123)"));
}
@Test
public void should_be_able_to_add_extra_issues() {
TestOutcome outcome = TestOutcome.forTest("should_do_this", SomeTestScenario.class);
List<String> extraIssues = Arrays.asList("#ISSUE-456", "#ISSUE-789");
outcome.addIssues(extraIssues);
assertThat(outcome.getIssues(), hasItems("#ISSUE-123", "#ISSUE-456", "#ISSUE-789"));
}
@Test
public void a_test_outcome_should_record_multiple_issues_at_class_level() {
TestOutcome outcome =
TestOutcome.forTest("should_do_something_else", SomeOtherTestScenario.class);
assertThat(outcome.getFormattedIssues(), is("(#ISSUE-123, #ISSUE-456)"));
}
@Test
public void should_record_minimal_nested_test_steps_as_nested_structures() throws Exception {
TestOutcome testOutcome =
TestOutcome.forTest("a_nested_test_case", SomeNestedTestScenario.class);
String expectedReport =
"<acceptance-test-run title='A nested test case' name='a_nested_test_case' steps='1' successful='1' failures='0' skipped='0' ignored='0' pending='0' result='SUCCESS'>\n"
+ " <user-story id='net.thucydides.core.reports.integration.WhenGeneratingAnXMLReport.AUserStory' name='A user story' />\n"
+ " <test-group name='Group 1' result='SUCCESS'>\n"
+ " <test-group name='Group 1.1' result='SUCCESS'>\n"
+ " <test-group name='Group 1.1.1' result='SUCCESS'>\n"
+ " <test-step result='SUCCESS'>\n"
+ " <description>step 1</description>\n"
+ " </test-step>\n"
+ " </test-group>\n"
+ " </test-group>\n"
+ " </test-group>\n"
+ "</acceptance-test-run>";
testOutcome.startGroup("Group 1");
testOutcome.startGroup("Group 1.1");
testOutcome.startGroup("Group 1.1.1");
testOutcome.recordStep(TestStepFactory.successfulTestStepCalled("step 1"));
testOutcome.endGroup();
testOutcome.endGroup();
testOutcome.endGroup();
File xmlReport = reporter.generateReportFor(testOutcome);
String generatedReportText = getStringFrom(xmlReport);
assertThat(generatedReportText, isSimilarTo(expectedReport));
}
@Test
public void should_include_the_name_of_any_screenshots_where_present() throws Exception {
TestOutcome testOutcome = TestOutcome.forTest("a_simple_test_case", SomeTestScenario.class);
String expectedReport =
"<acceptance-test-run title='A simple test case' name='a_simple_test_case' steps='2' successful='1' failures='1' skipped='0' ignored='0' pending='0' result='FAILURE'>\n"
+ " <user-story id='net.thucydides.core.reports.integration.WhenGeneratingAnXMLReport.AUserStory' name='A user story' />\n"
+ " <test-step result='SUCCESS'>\n"
+ " <screenshots>\n"
+ " <screenshot image='step_1.png' source='step_1.html'/>\n"
+ " </screenshots>\n"
+ " <description>step 1</description>\n"
+ " </test-step>\n"
+ " <test-step result='FAILURE'>\n"
+ " <description>step 2</description>\n"
+ " </test-step>\n"
+ "</acceptance-test-run>";
File screenshot = temporaryDirectory.newFile("step_1.png");
File source = temporaryDirectory.newFile("step_1.html");
TestStep step1 = TestStepFactory.successfulTestStepCalled("step 1");
step1.addScreenshot(new ScreenshotAndHtmlSource(screenshot, source));
testOutcome.recordStep(step1);
testOutcome.recordStep(TestStepFactory.failingTestStepCalled("step 2"));
File xmlReport = reporter.generateReportFor(testOutcome);
String generatedReportText = getStringFrom(xmlReport);
assertThat(generatedReportText, isSimilarTo(expectedReport));
}
private TestOutcome createScenarioOutcome(TestOutcome parameterizedOutcome) {
TestOutcome testOutcome =
TestOutcome.forTest(
normalizeMethodName(parameterizedOutcome), parameterizedOutcome.getTestCase());
return testOutcome;
}
@Test
public void should_find_all_the_issues_in_a_test() {
TestOutcome outcome =
TestOutcome.forTest("should_do_this", SomeAnnotatedTestScenarioWithManyIssues.class);
assertThat(
outcome.getIssues(), hasItems("#ISSUE-123", "#ISSUE-456", "#ISSUE-100", "#ISSUE-200"));
}
@Test
public void a_test_outcome_can_be_associated_with_an_issue() {
TestOutcome outcome = TestOutcome.forTest("should_do_this", SomeAnnotatedTestScenario.class);
outcome.isRelatedToIssue("#ISSUE-999");
outcome.isRelatedToIssue("#ISSUE-000");
assertThat(outcome.getIssues(), hasItems("#ISSUE-000", "#ISSUE-999"));
}
@Test
public void should_generate_an_XML_report_in_the_target_directory() throws Exception {
TestOutcome testOutcome = TestOutcome.forTest("a_simple_test_case", SomeTestScenario.class);
File xmlReport = reporter.generateReportFor(testOutcome);
assertThat(xmlReport.getPath(), startsWith(outputDirectory.getPath()));
}
@Test
public void should_have_a_meaningful_filename() throws Exception {
TestOutcome testOutcome = TestOutcome.forTest("a_simple_test_case", SomeTestScenario.class);
TestStep step1 = TestStepFactory.successfulTestStepCalled("step 1");
File screenshot = temporaryDirectory.newFile("step_1.png");
File source = temporaryDirectory.newFile("step_1.html");
step1.addScreenshot(new ScreenshotAndHtmlSource(screenshot, source));
testOutcome.recordStep(step1);
File xmlReport = reporter.generateReportFor(testOutcome);
assertThat(xmlReport.getName(), is("a_user_story_a_simple_test_case.xml"));
}
@Test
public void should_count_the_total_number_of_steps_with_each_outcome_in_acceptance_test_run()
throws Exception {
TestOutcome testOutcome = TestOutcome.forTest("a_simple_test_case", SomeTestScenario.class);
String expectedReport =
"<acceptance-test-run title='A simple test case' name='a_simple_test_case' steps='9' successful='2' failures='3' skipped='1' ignored='2' pending='1' result='FAILURE'>\n"
+ " <user-story id='net.thucydides.core.reports.integration.WhenGeneratingAnXMLReport.AUserStory' name='A user story' />\n"
+ " <test-step result='SUCCESS'>\n"
+ " <description>step 1</description>\n"
+ " </test-step>\n"
+ " <test-step result='IGNORED'>\n"
+ " <description>step 2</description>\n"
+ " </test-step>\n"
+ " <test-step result='IGNORED'>\n"
+ " <description>step 3</description>\n"
+ " </test-step>\n"
+ " <test-step result='SUCCESS'>\n"
+ " <description>step 4</description>\n"
+ " </test-step>\n"
+ " <test-step result='FAILURE'>\n"
+ " <description>step 5</description>\n"
+ " </test-step>\n"
+ " <test-step result='FAILURE'>\n"
+ " <description>step 6</description>\n"
+ " </test-step>\n"
+ " <test-step result='FAILURE'>\n"
+ " <description>step 7</description>\n"
+ " </test-step>\n"
+ " <test-step result='SKIPPED'>\n"
+ " <description>step 8</description>\n"
+ " </test-step>\n"
+ " <test-step result='PENDING'>\n"
+ " <description>step 9</description>\n"
+ " </test-step>\n"
+ "</acceptance-test-run>";
testOutcome.recordStep(TestStepFactory.successfulTestStepCalled("step 1"));
testOutcome.recordStep(TestStepFactory.ignoredTestStepCalled("step 2"));
testOutcome.recordStep(TestStepFactory.ignoredTestStepCalled("step 3"));
testOutcome.recordStep(TestStepFactory.successfulTestStepCalled("step 4"));
testOutcome.recordStep(TestStepFactory.failingTestStepCalled("step 5"));
testOutcome.recordStep(TestStepFactory.failingTestStepCalled("step 6"));
testOutcome.recordStep(TestStepFactory.failingTestStepCalled("step 7"));
testOutcome.recordStep(TestStepFactory.skippedTestStepCalled("step 8"));
testOutcome.recordStep(TestStepFactory.pendingTestStepCalled("step 9"));
File xmlReport = reporter.generateReportFor(testOutcome);
String generatedReportText = getStringFrom(xmlReport);
assertThat(generatedReportText, isSimilarTo(expectedReport));
}
@Test
public void a_test_outcome_should_inject_issue_links_from_the_Issue_annotation_if_requested() {
MockEnvironmentVariables environmentVariables = new MockEnvironmentVariables();
environmentVariables.setProperty("jira.url", "http://my.jira");
IssueTracking issueTracking = new SystemPropertiesIssueTracking(environmentVariables);
TestOutcome outcome =
TestOutcome.forTest("should_do_this", SomeTestScenario.class)
.usingIssueTracking(issueTracking);
assertThat(
outcome.getFormattedIssues(),
is("(<a href=\"http://my.jira/browse/ISSUE-123\">#ISSUE-123</a>)"));
}
@Test
public void should_have_a_meaningful_filename() throws Exception {
TestOutcome testOutcome = TestOutcome.forTest("should_do_this", SomeTestScenario.class);
TestStep step1 = TestStepFactory.successfulTestStepCalled("step 1");
File screenshot = temporaryDirectory.newFile("google_page_1.png");
File screenshotSource = temporaryDirectory.newFile("google_page_1.html");
step1.addScreenshot(new ScreenshotAndHtmlSource(screenshot, screenshotSource));
testOutcome.recordStep(step1);
File xmlReport = reporter.generateReportFor(testOutcome, allTestOutcomes);
assertThat(
xmlReport.getName(), is(Digest.ofTextValue("a_user_story_should_do_this") + ".html"));
}
@Test
public void should_include_error_message_for_failing_test() throws Exception {
TestOutcome testOutcome = TestOutcome.forTest("a_simple_test_case", SomeTestScenario.class);
TestStep step = TestStepFactory.failingTestStepCalled("step 1");
step.failedWith(new IllegalArgumentException("Oh nose!"));
testOutcome.recordStep(step);
File xmlReport = reporter.generateReportFor(testOutcome);
String generatedReportText = getStringFrom(xmlReport);
assertThat(generatedReportText, containsString("<error>Oh nose!</error>"));
}
@Test
public void spaces_in_the_qualifier_are_converted_to_underscores_for_the_report_name()
throws Exception {
TestOutcome testOutcome = TestOutcome.forTest("should_do_this", SomeTestScenario.class);
TestStep step1 = TestStepFactory.successfulTestStepCalled("step 1");
File screenshot = temporaryDirectory.newFile("google_page_1.png");
File screenshotSource = temporaryDirectory.newFile("google_page_1.html");
step1.addScreenshot(new ScreenshotAndHtmlSource(screenshot, screenshotSource));
testOutcome.recordStep(step1);
reporter.setQualifier("a b c");
File xmlReport = reporter.generateReportFor(testOutcome, allTestOutcomes);
assertThat(
xmlReport.getName(), is(Digest.ofTextValue("a_user_story_should_do_this_a_b_c") + ".html"));
}
@Test
public void should_generate_an_XML_report_for_an_acceptance_test_run() throws Exception {
TestOutcome testOutcome = TestOutcome.forTest("should_do_this", SomeTestScenario.class);
String expectedReport =
"<acceptance-test-run title='Should do this' name='should_do_this' steps='1' successful='1' failures='0' skipped='0' ignored='0' pending='0' result='SUCCESS'>\n"
+ " <user-story id='net.thucydides.core.reports.integration.WhenGeneratingAnXMLReport.AUserStory' name='A user story' />\n"
+ " <test-step result='SUCCESS'>\n"
+ " <description>step 1</description>\n"
+ " </test-step>\n"
+ "</acceptance-test-run>";
testOutcome.recordStep(TestStepFactory.successfulTestStepCalled("step 1"));
File xmlReport = reporter.generateReportFor(testOutcome);
String generatedReportText = getStringFrom(xmlReport);
assertThat(generatedReportText, isSimilarTo(expectedReport));
}
@Test
public void the_screenshots_report_should_contain_the_overall_test_description()
throws Exception {
TestOutcome testOutcome = TestOutcome.forTest("should_do_this", SomeTestScenario.class);
recordStepWithScreenshot(testOutcome, "Search cats on Google", "google_page_1.png");
recordStepWithScreenshot(testOutcome, "View the results", "google_page_2.png");
recordStepWithScreenshot(testOutcome, "Display a resulting page", "google_page_3.png");
reporter.generateReportFor(testOutcome, allTestOutcomes);
File screenshotReport =
new File(
outputDirectory,
Digest.ofTextValue("a_user_story_should_do_this") + "_screenshots.html");
String reportContents = FileUtils.readFileToString(screenshotReport);
assertThat(reportContents, containsString("Should do this"));
}
@Test
public void screenshots_should_have_a_separate_html_report() throws Exception {
TestOutcome testOutcome = TestOutcome.forTest("should_do_this", SomeTestScenario.class);
TestStep step1 = TestStepFactory.successfulTestStepCalled("step 1");
File screenshot = temporaryDirectory.newFile("google_page_1.png");
File screenshotSource = temporaryDirectory.newFile("google_page_1.html");
step1.addScreenshot(new ScreenshotAndHtmlSource(screenshot, screenshotSource));
testOutcome.recordStep(step1);
reporter.generateReportFor(testOutcome, allTestOutcomes);
File screenshotReport =
new File(
outputDirectory,
Digest.ofTextValue("a_user_story_should_do_this") + "_screenshots.html");
assertThat(screenshotReport.exists(), is(true));
}
@Test
public void spaces_in_the_qualifer_should_be_converted_to_underscores_in_the_test_run_name()
throws Exception {
TestOutcome testOutcome = TestOutcome.forTest("a_simple_test_case", SomeTestScenario.class);
TestStep step1 = TestStepFactory.successfulTestStepCalled("step 1");
File screenshot = temporaryDirectory.newFile("step_1.png");
File source = temporaryDirectory.newFile("step_1.html");
step1.addScreenshot(new ScreenshotAndHtmlSource(screenshot, source));
testOutcome.recordStep(step1);
reporter.setQualifier("a b c");
File xmlReport = reporter.generateReportFor(testOutcome);
String generatedReportText = getStringFrom(xmlReport);
assertThat(generatedReportText, containsString("name=\"a_simple_test_case_a_b_c\""));
}
@Test
public void screenshot_html_should_mention_the_step_name() throws Exception {
TestOutcome testOutcome = TestOutcome.forTest("should_do_this", SomeTestScenario.class);
TestStep step1 = TestStepFactory.successfulTestStepCalled("step 1");
File screenshot = temporaryDirectory.newFile("google_page_1.png");
File screenshotSource = temporaryDirectory.newFile("google_page_1.html");
step1.addScreenshot(new ScreenshotAndHtmlSource(screenshot, screenshotSource));
testOutcome.recordStep(step1);
reporter.generateReportFor(testOutcome, allTestOutcomes);
File screenshotReport =
new File(
outputDirectory,
Digest.ofTextValue("a_user_story_should_do_this") + "_screenshots.html");
String reportContents = FileUtils.readFileToString(screenshotReport);
assertThat(reportContents, containsString("step 1"));
}
@Test
public void
should_generate_a_qualified_XML_report_with_formatted_parameters_if_the_qualifier_is_specified()
throws Exception {
TestOutcome testOutcome = TestOutcome.forTest("a_simple_test_case", SomeTestScenario.class);
String expectedReport =
"<acceptance-test-run title='A simple test case [a/b]' name='a_simple_test_case_a_b' steps='1' successful='1' failures='0' skipped='0' ignored='0' pending='0' result='SUCCESS'>\n"
+ " <user-story id='net.thucydides.core.reports.integration.WhenGeneratingAnXMLReport.AUserStory' name='A user story' />\n"
+ " <test-step result='SUCCESS'>\n"
+ " <description>step 1</description>\n"
+ " </test-step>\n"
+ "</acceptance-test-run>";
testOutcome.recordStep(TestStepFactory.successfulTestStepCalled("step 1"));
reporter.setQualifier("a_b");
File xmlReport = reporter.generateReportFor(testOutcome);
String generatedReportText = getStringFrom(xmlReport);
assertThat(generatedReportText, isSimilarTo(expectedReport));
}
@Test
public void a_test_outcome_should_record_multiple_issues() {
TestOutcome outcome = TestOutcome.forTest("should_do_this", SomeOtherTestScenario.class);
assertThat(outcome.getFormattedIssues(), is("(#ISSUE-123, #ISSUE-456, #ISSUE-789)"));
}
/** Case for JUnit integration, where a test case is present. */
@Test
public void a_test_outcome_title_should_be_based_on_the_tested_method_name_if_defined() {
TestOutcome outcome = TestOutcome.forTest("should_do_this", SomeTestScenario.class);
assertThat(outcome.getTitle(), is("Should do this"));
}
@Test
public void a_test_outcome_title_can_be_overriden_using_the_Title_annotation() {
TestOutcome outcome = TestOutcome.forTest("should_do_this", SomeAnnotatedTestScenario.class);
assertThat(outcome.getTitle(), is("Really should do this!"));
}
@Test
public void should_be_able_to_obtain_a_link_to_the_saucelabs_video() {
testOutcome = TestOutcome.forTest("should_do_this", SomeTestScenario.class);
testOutcome.setSessionId("1234");
assertThat(testOutcome.getVideoLink(), is("http://saucelabs.com/jobs/1234"));
}
@Test
public void a_test_outcome_title_can_be_overriden_manually_even_with_an_annotation() {
TestOutcome outcome = TestOutcome.forTest("should_do_this", SomeAnnotatedTestScenario.class);
outcome.setTitle("My custom title");
assertThat(outcome.getTitle(), is("My custom title"));
}
@Test
public void a_test_outcome_should_record_issue_numbers_from_the_Issue_annotation() {
TestOutcome outcome = TestOutcome.forTest("should_do_this", SomeTestScenario.class);
assertThat(outcome.getIssues(), hasItem("#ISSUE-123"));
}
@Test
public void a_test_outcome_can_have_no_issues() {
TestOutcome outcome = TestOutcome.forTest("should_do_this", SomeAnnotatedTestScenario.class);
assertThat(outcome.getIssues().size(), is(0));
}