private Set<String> identifyTestsWithDifferentResults( TestExecResults expected_results, TestExecResults exec_results) { Set<String> diffTests = new LinkedHashSet<String>(); // check the number of JVM launches Utils.checkTrue( expected_results.getExecutionRecords().size() == 1, "Number of JVM launches: " + expected_results.getExecutionRecords().size()); Utils.checkTrue( exec_results.getExecutionRecords().size() == 1, "Number of JVM launches: " + exec_results.getExecutionRecords().size()); // get the test results TestExecResult default_results = expected_results.getExecutionRecords().get(0); TestExecResult actual_results = exec_results.getExecutionRecords().get(0); // check the number of the tests Utils.checkTrue( default_results.getAllTests().size() == actual_results.getAllTests().size(), "Size not equal: " + default_results.getAllTests().size() + " vs. " + actual_results.getAllTests().size()); // then compare the test execution outcome for (String test : default_results.getAllTests()) { OneTestExecResult defaultTestResult = default_results.getResult(test); OneTestExecResult actualTestResult = actual_results.getResult(test); // compare the results if (!sameResults(defaultTestResult, actualTestResult)) { diffTests.add(test); if (verbose) { Log.logln(" Diff Test: " + test); Log.logln(" default result: " + defaultTestResult.result); Log.logln(" shuffled result: " + actualTestResult.result); } } } return diffTests; }
public Set<String> findDependentTests() { long startTime = System.currentTimeMillis(); // use linked hash set to keep the original order Set<String> depTests = new LinkedHashSet<String>(); // first execute all tests in the default order FixedOrderRunner fixedRunner = new FixedOrderRunner(this.defaultTestList); TestExecResults expected_results = fixedRunner.run(); Utils.checkTrue( expected_results.getExecutionRecords().size() == 1, "The size is: " + expected_results.getExecutionRecords().size()); TestExecResult rs = expected_results.getExecutionRecords().get(0); if (verbose) { Log.logln("The default execution results: "); for (String test : rs.getAllTests()) { Log.logln(" Test: " + test + ": " + rs.getResult(test).result); } } // then start to shuffle the test suite, and executed the shuffled suite // and then observe the results. List<String> shuffledTests = new ArrayList<String>(); shuffledTests.addAll(this.defaultTestList); for (int i = 0; i < this.trialNum; i++) { System.out.println("The: " + i + "-th trail..."); Log.logln("The: " + i + "-th trail..."); // do shuffling and execute the tests Collections.shuffle(shuffledTests, rnd); if (verbose) { System.out.println(" shuffled tests: " + shuffledTests); } FixedOrderRunner runner = new FixedOrderRunner(shuffledTests); TestExecResults exec_results = runner.run(); // find new dependent tests Set<String> diffTests = this.identifyTestsWithDifferentResults(expected_results, exec_results); depTests.addAll(diffTests); System.out.println(" Num of dependent tests: " + depTests.size()); if (i == 9 || i == 99 || i == 999) { Log.logln("Time cost: " + (System.currentTimeMillis() - startTime) / 1000 + " second"); Log.logln("Number of dependent tests: " + depTests.size()); for (String dt : depTests) { Log.logln(" " + dt); } } } return depTests; }