/** * Prints out the results of the given tests, and returns true if they all passed. Posts any * targets which weren't already completed by the listener to the EventBus. Reports all targets on * the console via the given notifier. Run at the end of the build, run only once. * * @param testTargets The list of targets being run * @param listener An aggregating listener with intermediate results * @param notifier A console notifier to echo results to. * @return true if all the tests passed, else false */ public boolean differentialAnalyzeAndReport( Collection<ConfiguredTarget> testTargets, AggregatingTestListener listener, TestResultNotifier notifier) { Preconditions.checkNotNull(testTargets); Preconditions.checkNotNull(listener); Preconditions.checkNotNull(notifier); // The natural ordering of the summaries defines their output order. Set<TestSummary> summaries = Sets.newTreeSet(); int totalRun = 0; // Number of targets running at least one non-cached test. int passCount = 0; for (ConfiguredTarget testTarget : testTargets) { TestSummary summary = aggregateAndReportSummary(testTarget, listener).build(); summaries.add(summary); // Finished aggregating; build the final console output. if (summary.actionRan()) { totalRun++; } if (TestResult.isBlazeTestStatusPassed(summary.getStatus())) { passCount++; } } Preconditions.checkState(summaries.size() == testTargets.size()); notifier.notify(summaries, totalRun); return passCount == testTargets.size(); }
@Test public void testSummarizeResultsForAllTypes() { report.addResult(ReportUtil.getListResult()); TestSummary summary = report.summarizeResults(""); assertEquals(3, summary.getSteps()); assertEquals(1, summary.getFailures()); assertEquals(0, summary.getErrors()); assertEquals("test", summary.getScriptName()); }
public TestSummary summarizeResults(String logFileNameBase) { TestSummary summary = new TestSummary(); boolean fail = false; summary.setScriptName(scriptName); summary.setSteps(listResult.size()); summary.setLogFileName(logFileNameBase); if (listResult.size() == 0) fail = true; else { for (Iterator<TestResult> iter = listResult.iterator(); iter.hasNext(); ) { TestResult result = (TestResult) iter.next(); if (ResultType.FAILURE.equals(result.type)) { summary.incrementFailures(); fail = true; } else if (ResultType.ERROR.equals(result.type)) { summary.incrementErrors(); fail = true; } else if (ResultType.SUCCESS.equals(result.type)) { summary.incrementSuccesses(); } } } summary.setTimeTaken(getTimeTaken()); summary.setFail(fail); return summary; }
private TestSummary.Builder markIncomplete(TestSummary.Builder summaryBuilder) { // TODO(bazel-team): (2010) Make NotRunTestResult support both tests failed to built and // tests with no status and post it here. TestSummary summary = summaryBuilder.peek(); BlazeTestStatus status = summary.getStatus(); if (status != BlazeTestStatus.NO_STATUS) { status = aggregateStatus(status, BlazeTestStatus.INCOMPLETE); } return summaryBuilder.setStatus(status); }
public void testFinalCounts() throws Exception { FitProtocol.writeData(result1.toString(), output); Counts counts = new Counts(1, 2, 3, 4); FitProtocol.writeCounts(counts, output); responder.processResults(input); TestSummary summary = formatter.finalSummary; assertEquals(counts.right, summary.getRight()); assertEquals(counts.wrong, summary.getWrong()); assertEquals(counts.ignores, summary.getIgnores()); assertEquals(counts.exceptions, summary.getExceptions()); }
public void acceptResultsLast(TestSummary testSummary) throws Exception { if (response.isXmlFormat()) { addTestResultsToXmlDocument(testSummary, page.getName()); } else { assertionCounts.tally(testSummary); } }
@Override public void testComplete(WikiTestPage test, TestSummary testSummary) { increaseCompletedTests(); if (firstFailure != null) { notifier.fireTestFailure(new Failure(descriptionFor(test), firstFailure)); } else if (test.isTestPage()) { if (testSummary.getExceptions() > 0) { notifier.fireTestFailure( new Failure( descriptionFor(test), new Exception("Exception occurred on page " + test.getFullPath()))); } else if (testSummary.getWrong() > 0) { notifier.fireTestFailure( new Failure( descriptionFor(test), new AssertionError("Test failures occurred on page " + test.getFullPath()))); } else { notifier.fireTestFinished(descriptionFor(test)); } } }
/** * Incrementally updates a TestSummary given an existing summary and a new TestResult. Only call * on built targets. * * @param summaryBuilder Existing unbuilt test summary associated with a target. * @param result New test result to aggregate into the summary. * @return The updated TestSummary. */ public TestSummary.Builder incrementalAnalyze( TestSummary.Builder summaryBuilder, TestResult result) { // Cache retrieval should have been performed already. Preconditions.checkNotNull(result); Preconditions.checkNotNull(summaryBuilder); TestSummary existingSummary = Preconditions.checkNotNull(summaryBuilder.peek()); TransitiveInfoCollection target = existingSummary.getTarget(); Preconditions.checkNotNull(target, "The existing TestSummary must be associated with a target"); BlazeTestStatus status = existingSummary.getStatus(); int numCached = existingSummary.numCached(); int numLocalActionCached = existingSummary.numLocalActionCached(); // If a test was neither cached locally nor remotely we say action was taken. if (!(result.isCached() || result.getData().getRemotelyCached())) { summaryBuilder.setActionRan(true); } else { numCached++; } if (result.isCached()) { numLocalActionCached++; } PathFragment coverageData = result.getCoverageData(); if (coverageData != null) { summaryBuilder.addCoverageFiles( Collections.singletonList(execRoot.getRelative(coverageData))); } if (!executionOptions.runsPerTestDetectsFlakes) { status = aggregateStatus(status, result.getData().getStatus()); } else { int shardNumber = result.getShardNum(); int runsPerTestForLabel = target.getProvider(TestProvider.class).getTestParams().getRuns(); List<BlazeTestStatus> singleShardStatuses = summaryBuilder.addShardStatus(shardNumber, result.getData().getStatus()); if (singleShardStatuses.size() == runsPerTestForLabel) { BlazeTestStatus shardStatus = BlazeTestStatus.NO_STATUS; int passes = 0; for (BlazeTestStatus runStatusForShard : singleShardStatuses) { shardStatus = aggregateStatus(shardStatus, runStatusForShard); if (TestResult.isBlazeTestStatusPassed(shardStatus)) { passes++; } } // Under the RunsPerTestDetectsFlakes option, return flaky if 1 <= p < n shards pass. // If all results pass or fail, aggregate the passing/failing shardStatus. if (passes == 0 || passes == runsPerTestForLabel) { status = aggregateStatus(status, shardStatus); } else { status = aggregateStatus(status, BlazeTestStatus.FLAKY); } } } List<Path> passed = new ArrayList<>(); if (result.getData().hasPassedLog()) { passed.add( result .getTestAction() .getTestLog() .getPath() .getRelative(result.getData().getPassedLog())); } List<Path> failed = new ArrayList<>(); for (String path : result.getData().getFailedLogsList()) { failed.add(result.getTestAction().getTestLog().getPath().getRelative(path)); } summaryBuilder .addTestTimes(result.getData().getTestTimesList()) .addPassedLogs(passed) .addFailedLogs(failed) .addWarnings(result.getData().getWarningList()) .collectFailedTests(result.getData().getTestCase()) .setRanRemotely(result.getData().getIsRemoteStrategy()); List<String> warnings = new ArrayList<>(); if (status == BlazeTestStatus.PASSED && shouldEmitTestSizeWarningInSummary( summaryOptions.testVerboseTimeoutWarnings, warnings, result.getData().getTestProcessTimesList(), target)) { summaryBuilder.setWasUnreportedWrongSize(true); } return summaryBuilder .setStatus(status) .setNumCached(numCached) .setNumLocalActionCached(numLocalActionCached) .addWarnings(warnings); }
@Test public void testSummarizeResultsForEmptyList() { TestSummary summary = report.summarizeResults(""); assertEquals(0, summary.getSteps()); }