public void excludeTest(String testName, String reason) { TestInfo testInfo = findTestInfo(testName); if (testInfo == null) return; testInfo.isExcluded(true); testInfo.reasonToExclude(reason); }
public void addFailure(Test test, AssertionFailedError error) { message = error.getMessage(); failCount++; status = FAIL; TestInfo testInfo = testMap.get(test); testInfo.addFailure(error); message = Messages.getInstance().getString("UI.USER_TEST_SUITE_FAILED"); // $NON-NLS-1$ }
public void addError(Test test, Throwable error) { message = error.getMessage(); errorCount++; status = FAIL; TestInfo testInfo = testMap.get(test); testInfo.addError(error); message = Messages.getInstance().getString("UI.USER_TEST_SUITE_FAILED"); // $NON-NLS-1$ }
public void includeTest(String testName) { TestInfo testInfo = findTestInfo(testName); if (testInfo == null) { testInfo = new TestInfo(this, testName); m_testInfoList.add(testInfo); } testInfo.isExcluded(false); }
@Override public TestInfo clone() { TestInfo result = new TestInfo(testName); result.id = id; result.passes = new ArrayList<List<COV>>(passes); result.failures = new ArrayList<List<COV>>(failures); result.failuresValid = new ArrayList<List<COV>>(failuresValid); return result; }
public void endTest(Test test) { if (status != FAIL) { passCount++; } status = NOT_RUNNING; TestInfo testInfo = testMap.get(test); testInfo.endTest(); message = ""; // $NON-NLS-1$ }
@Override public void setUp() throws Exception { mTestInstance = new BluetoothStressTest() { @Override void reportMetrics( ITestInvocationListener l, TestInfo test, Map<String, String> metrics) { mReportedTestInfo = test; mReportedMetrics = metrics; } }; mScanInfo = new TestInfo(); mScanInfo.mTestName = "scan"; mScanInfo.mTestMethod = "testScan"; mScanInfo.mIterCount = 1; mScanInfo.mPerfMetrics.add("startScan"); mScanInfo.mPerfMetrics.add("stopScan"); }
public String toString(String prefix) { if (prefix == null) prefix = ""; StringBuilder result = new StringBuilder(); result.append(prefix + "Directory: " + m_path + Utils.EOL); result.append( prefix + " | isExcluded = " + m_isExcluded + " [reason: " + StringUtils.nullOrString(m_reasonToExclude) + "]" + Utils.EOL); for (TestInfo testInfo : m_testInfoList) { result.append(testInfo.toString(prefix + " | ") + Utils.EOL); } return StringUtils.trimEnd(result.toString(), "\r\n"); }
public Node getStatus(Element parent) { Element node = parent.addElement("suite"); // $NON-NLS-1$ node.addAttribute("class", className); // $NON-NLS-1$ node.addAttribute("name", name); // $NON-NLS-1$ node.addAttribute("test-count", Integer.toString(testCount)); // $NON-NLS-1$ node.addAttribute("run-count", Integer.toString(runCount)); // $NON-NLS-1$ node.addAttribute("pass-count", Integer.toString(passCount)); // $NON-NLS-1$ node.addAttribute("fail-count", Integer.toString(errorCount + failCount)); // $NON-NLS-1$ Element messageNode = node.addElement("message"); // $NON-NLS-1$ Element tests = node.addElement("tests"); // $NON-NLS-1$ int currentPassCount = 0; int currentRunCount = 0; int currentFailCount = 0; for (int idx = 0; idx < testList.size(); idx++) { TestInfo test = testList.get(idx); test.getStatus(tests); if (test.lastResult == PASS) { currentPassCount++; } if (test.lastResult == FAIL) { currentFailCount++; } if (test.lastResult != test.NOT_RUN) { currentRunCount++; } } if (currentRunCount == 0) { message = Messages.getInstance().getString("UI.USER_TEST_SUITE_NOT_RUN"); // $NON-NLS-1$ } else if (currentRunCount == currentPassCount && currentRunCount < testCount) { message = Messages.getInstance().getString("UI.USER_TEST_SUITE_SOME_PASSED"); // $NON-NLS-1$ } else if (currentRunCount == currentPassCount && currentRunCount == testCount) { message = Messages.getInstance().getString("UI.USER_TEST_SUITE_ALL_PASSED"); // $NON-NLS-1$ } else if (currentFailCount > 0) { message = Messages.getInstance().getString("UI.USER_TEST_SUITE_SOME_FAILURES"); // $NON-NLS-1$ } messageNode.setText((message == null) ? "" : message); // $NON-NLS-1$ return node; }
/** * Report run metrics by creating an empty test run to stick them in * * <p>Exposed for unit testing */ void reportMetrics(ITestInvocationListener listener, TestInfo test, Map<String, String> metrics) { // Create an empty testRun to report the parsed runMetrics CLog.d("About to report metrics to %s: %s", test.getTestMetricsName(), metrics); listener.testRunStarted(test.getTestMetricsName(), 0); listener.testRunEnded(0, metrics); }
private void setupTests() { if (mTestCases != null) { // assume already set up return; } // Allocate enough space for all of the TestInfo instances below mTestCases = new ArrayList<TestInfo>(12); TestInfo t = new TestInfo(); t.mTestName = "discoverable"; t.mTestMethod = "testDiscoverable"; t.mIterKey = "discoverable_iterations"; t.mIterCount = mDiscoverableIterations; t.mPerfMetrics.add("discoverable"); t.mPerfMetrics.add("undiscoverable"); mTestCases.add(t); t = new TestInfo(); t.mTestName = "enable"; t.mTestMethod = "testEnable"; t.mIterKey = "enable_iterations"; t.mIterCount = mEnableIterations; t.mPerfMetrics.add("enable"); t.mPerfMetrics.add("disable"); mTestCases.add(t); t = new TestInfo(); t.mTestName = "scan"; t.mTestMethod = "testScan"; t.mIterKey = "scan_iterations"; t.mIterCount = mScanIterations; t.mPerfMetrics.add("startScan"); t.mPerfMetrics.add("stopScan"); mTestCases.add(t); t = new TestInfo(); t.mTestName = "enable_pan"; t.mTestMethod = "testEnablePan"; t.mIterKey = "enable_pan_iterations"; t.mIterCount = mEnablePanIterations; t.mPerfMetrics.add("enablePan"); t.mPerfMetrics.add("disablePan"); mTestCases.add(t); // Remote device tests t = new TestInfo(); t.mTestName = "pair"; t.mTestMethod = "testPair"; t.mIterKey = "pair_iterations"; t.mIterCount = mPairIterations; t.mPerfMetrics.add("pair"); t.mPerfMetrics.add("unpair"); t.mRemoteAddress = mDeviceAddress; t.mPairPin = mDevicePairPin; t.mPairPasskey = mDevicePairPasskey; t.mInstructions = String.format( "Start the testAcceptPair instrumentation on the remote " + "Android device with the following command:\n\n%s\n\nHit Enter when done.", String.format( INSTRUCTIONS_INSTRUMENT_CMD, mDeviceSerial, mLocalAddress, "pair_iterations", mPairIterations, mTestClassName, "testAcceptPair", mTestPackageName, TEST_RUNNER_NAME)); mTestCases.add(t); t = new TestInfo(); t.mTestName = "accept_pair"; t.mTestMethod = "testAcceptPair"; t.mIterKey = "pair_iterations"; t.mIterCount = mAcceptPairIterations; t.mPerfMetrics.add("acceptPair"); t.mPerfMetrics.add("unpair"); t.mRemoteAddress = mDeviceAddress; t.mPairPin = mDevicePairPin; t.mPairPasskey = mDevicePairPasskey; t.mInstructions = String.format( "Start the testPair instrumentation on the remote " + "Android device with the following command:\n\n%s\n\nHit Enter when done.", String.format( INSTRUCTIONS_INSTRUMENT_CMD, mDeviceSerial, mLocalAddress, "pair_iterations", mAcceptPairIterations, mTestClassName, "testPair", mTestPackageName, TEST_RUNNER_NAME)); mTestCases.add(t); t = new TestInfo(); t.mTestName = "connect_headset"; t.mTestMethod = "testConnectHeadset"; t.mIterKey = "connect_headset_iterations"; t.mIterCount = mConnectHeadsetIterations; t.mPerfMetrics.add("connectHeadset"); t.mPerfMetrics.add("disconnectHeadset"); t.mRemoteAddress = mHeadsetAddress; t.mPairPin = mHeadsetPairPin; t.mPairPasskey = mHeadsetPairPasskey; t.mInstructions = "Put the remote headset device in pairing mode. Hit Enter when done."; mTestCases.add(t); t = new TestInfo(); t.mTestName = "connect_a2dp"; t.mTestMethod = "testConnectA2dp"; t.mIterKey = "connect_a2dp_iterations"; t.mIterCount = mConnectA2dpIterations; t.mPerfMetrics.add("connectA2dp"); t.mPerfMetrics.add("disconnectA2dp"); t.mRemoteAddress = mA2dpAddress; t.mPairPin = mA2dpPairPin; t.mPairPasskey = mA2dpPairPasskey; t.mInstructions = "Put the remote A2DP device in pairing mode. Hit Enter when done."; mTestCases.add(t); t = new TestInfo(); t.mTestName = "connect_input"; t.mTestMethod = "testConnectInput"; t.mIterKey = "connect_input_iterations"; t.mIterCount = mConnectInputIterations; t.mPerfMetrics.add("connectInput"); t.mPerfMetrics.add("disconnectInput"); t.mRemoteAddress = mInputAddress; t.mPairPin = mInputPairPin; t.mPairPasskey = mInputPairPasskey; t.mInstructions = "Put the remote input device in pairing mode. Hit Enter when done."; mTestCases.add(t); t = new TestInfo(); t.mTestName = "connect_pan"; t.mTestMethod = "testConnectPan"; t.mIterKey = "connect_pan_iterations"; t.mIterCount = mConnectPanIterations; t.mPerfMetrics.add("connectPan"); t.mPerfMetrics.add("disconnectPan"); t.mRemoteAddress = mDeviceAddress; t.mPairPin = mDevicePairPin; t.mPairPasskey = mDevicePairPasskey; t.mInstructions = String.format( "Start the testIncomingPanConnection instrumentation on " + "the remote Android device with the following command:\n\n%s\n\nHit Enter when " + "done.", String.format( INSTRUCTIONS_INSTRUMENT_CMD, mDeviceSerial, mLocalAddress, "connect_pan_iterations", mConnectPanIterations, mTestClassName, "testIncomingPanConnection", mTestPackageName, TEST_RUNNER_NAME)); mTestCases.add(t); t = new TestInfo(); t.mTestName = "incoming_pan_connection"; t.mTestMethod = "testIncomingPanConnection"; t.mIterKey = "connect_pan_iterations"; t.mIterCount = mIncomingPanConnectionIterations; t.mPerfMetrics.add("incomingPanConnection"); t.mPerfMetrics.add("incomingPanDisconnection"); t.mRemoteAddress = mDeviceAddress; t.mPairPin = mDevicePairPin; t.mPairPasskey = mDevicePairPasskey; t.mInstructions = ("Start the testConnectPan instrumentation on the remote Android " + "device. Hit Enter when done."); t.mInstructions = String.format( "Start the testConnectPan instrumentation on the remote " + "Android device with the following command:\n\n%s\n\nHit Enter when done.", String.format( INSTRUCTIONS_INSTRUMENT_CMD, mDeviceSerial, mLocalAddress, "connect_pan_iterations", mIncomingPanConnectionIterations, mTestClassName, "testConnectPan", mTestPackageName, TEST_RUNNER_NAME)); mTestCases.add(t); t = new TestInfo(); t.mTestName = "start_stop_sco"; t.mTestMethod = "testStartStopSco"; t.mIterKey = "start_stop_sco_iterations"; t.mIterCount = mStartStopScoIterations; t.mPerfMetrics.add("startSco"); t.mPerfMetrics.add("stopSco"); t.mRemoteAddress = mHeadsetAddress; t.mPairPin = mHeadsetPairPin; t.mPairPasskey = mHeadsetPairPasskey; t.mInstructions = "Put the remote headset device in pairing mode. Hit Enter when done."; mTestCases.add(t); }
private TestInfo findTestInfo(String testName) { for (TestInfo info : m_testInfoList) { if (info.name().equalsIgnoreCase(testName)) return info; } return null; }
private void summarize(StringBuffer sb) throws ClassNotFoundException, NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, InstantiationException { // Map<Double,Integer> ratioDistribution = new // TreeMap<Double,Integer>(); // Map<Integer,Integer> runDistribution = new // TreeMap<Integer,Integer>(); // Map<String, Integer> testsFailInValidConfs = new TreeMap<String, // Integer>(); // Map<String, Integer> testsFailValidConfs = new TreeMap<String, // Integer>(); // Set<String> failingTestsValidConfs = new TreeSet<String>(); Map<Integer, Integer> numConfigurationFailInNTests = new HashMap<Integer, Integer>(); Set<String> removeme = new TreeSet<String>(); for (Entry e : counts) { removeme.addAll(e.failingTestNames); removeme.addAll(e.passingTestNames); } for (Entry entry : counts) { // int numPassingRuns = entry.numPassing; // int numTotalRuns = entry.numExecs; // if (numTotalRuns == 0) { // continue; // } // double frac = ((double) numPassingRuns) / numTotalRuns; // double fraction = Math.floor(frac * 100) / 100; // // // distributions // calculateRatioDistribution(ratioDistribution, fraction); // calculateRunDistribution(runDistribution, numTotalRuns); // failing tests for (String s : entry.failingTestNames) { TestInfo ti = tInfo.get(s); if (ti == null) { ti = new TestInfo(s); tInfo.put(s, ti); } ti.failures.add(entry.conf); if (entry.isValid) { ti.failuresValid.add(entry.conf); } } for (String s : entry.passingTestNames) { TestInfo ti = tInfo.get(s); if (ti == null) { ti = new TestInfo(s); tInfo.put(s, ti); } ti.passes.add(entry.conf); } if (entry.numExecs - entry.numPassing > 0 && entry.isValid) { int size = entry.failingTestNames.size(); Integer k = numConfigurationFailInNTests.get(size); if (k == null) k = 0; numConfigurationFailInNTests.put(size, ++k); // failingConfs.append(String.format("%s\n %d: %s\n", // entry.conf, size, // entry.failingTestNames)); // failingTestsValidConfs.addAll(entry.failingTestNames); // for (String testName : entry.failingTestNames) { // Integer i = testsFailValidConfs.get(testName); // if (i == null) i = 0; // testsFailValidConfs.put(testName, ++i); // } } // // if (entry.numExecs - entry.numPassing > 0 && !entry.isValid) { // for (String testName : entry.failingTestNames) { // Integer i = testsFailInValidConfs.get(testName); // if (i == null) i = 0; // testsFailInValidConfs.put(testName, ++i); // } // } } /** shows the distribution of configurations */ sb.append("==== summary\n"); for (Integer i : numConfigurationFailInNTests.keySet()) { Integer j = numConfigurationFailInNTests.get(i); sb.append(String.format("%s configurations fail in %s tests\n", j, i)); } sb.append("==== latex summary of configs \n"); sb.append("\\# Confs. & \\# Tests\\\\ \n"); sb.append("\\hline\n"); for (Integer i : numConfigurationFailInNTests.keySet()) { Integer j = numConfigurationFailInNTests.get(i); sb.append(String.format("%s & %s\\\\ \n", j, i)); sb.append("\\hline\n"); } /** shows the list of failing, passing, and failing (valid) configurations */ Comparator<TestInfo> comp = new Comparator<LogReader.TestInfo>() { @Override public int compare(TestInfo o1, TestInfo o2) { int result; if (o1.failuresValid.size() != 0 && o2.failuresValid.size() == 0) { result = 1; } else if (o1.failuresValid.size() == 0 && o2.failuresValid.size() != 0) { result = -1; } else { int sizeOne = o1.failures.size(); int sizeTwo = o2.failures.size(); int diff = sizeOne - sizeTwo; if (diff > 0) { result = 1; // o1 greater than o2 } else if (diff < 0) { result = -1; } else { sizeOne = o1.failuresValid.size(); sizeTwo = o2.failuresValid.size(); diff = sizeOne - sizeTwo; if (diff > 0) { result = 1; } else if (diff < 0) { result = -1; } else { result = 0; } } } return result; } }; List<TestInfo> l = new ArrayList<TestInfo>(tInfo.values()); Collections.sort(l, comp); int id = 1; sb.append("\\emph{Ti} & \\emph{F} & \\emph{P} & \\emph{Fv}\\\\\n"); sb.append("\\hline\n"); sb.append("\\multicolumn{4}{|c|}{$\\dots$}\\\\\n"); sb.append("\\hline\n\\hline\n"); boolean block1 = true; boolean block2 = true; for (TestInfo ti : l) { if (block1 && ti.failuresValid.size() != 0) { block1 = false; } if (block2 && ti.failures.size() != 0) { sb.append("\\hline\n\\hline\n"); block2 = false; } if (!block1 || !block2) { sb.append( String.format( "%s & %s & %s & %s \\\\ \n", id, ti.failures.size(), ti.passes.size(), ti.failuresValid.size())); } ti.setTestId(id); id++; } /** shows the ranking of tests */ Comparator<TestInfo> ranker = new Comparator<LogReader.TestInfo>() { @Override public int compare(TestInfo o1, TestInfo o2) { int numFailuresOne = o1.failures.size(); int numFailuresTwo = o2.failures.size(); double ratioOne = o1.ratioPF(); double ratioTwo = o2.ratioPF(); int result; if (ratioOne > ratioTwo) { result = 1; } else if (ratioTwo > ratioOne) { result = -1; } else { // tie-breaker if (numFailuresOne > numFailuresTwo) { result = 1; } else if (numFailuresOne > numFailuresTwo) { result = -1; } else { result = 0; } } return result; } }; List<TestInfo> rankedList = new ArrayList<TestInfo>(tInfo.values()); Collections.sort(rankedList, Collections.reverseOrder(ranker)); int rank = 1; for (TestInfo ti : rankedList) { double ratio = ti.ratioPF(); if (ratio > 0) { if (ti.failuresValid.size() > 0) { sb.append("\\rowcolor{gray}\n"); } sb.append(String.format("%d & %d & %.2f\\\\ \n", rank++, ti.id, ratio)); } } Set<String> ReachableFailingConfs = new TreeSet<String>(); Set<String> ReachableFailingValidConfs = new TreeSet<String>(); /** * shows the list of tests with ratio > 0 and valid failing configurations * */ sb.append("---"); sb.append("ranked test list (valid configurations highlighted)\n"); for (TestInfo ti : rankedList) { // finding the reachable failing distinct confs for (List<COV> failConf : ti.failures) { ReachableFailingConfs.add(failConf.toString()); } // finding the reachable failing valid distinct confs for (List<COV> failConf : ti.failuresValid) { ReachableFailingValidConfs.add(failConf.toString()); } // double ratio = ti.ratioPF(); // if (ratio > 0) { // if (ti.failuresValid.size() > 0) { // sb.append(String.format("%d : %s\n", ti.id, ti.testName)); // for (List<COV> c : ti.failuresValid) { // sb.append(String.format(" %s\n", c)); // } // } // } } sb.append( String.format( "----->Number of reachable failing confs: %d\n", ReachableFailingConfs.size())); sb.append( String.format( "----->Number of reachable failing valid confs: %d\n", ReachableFailingValidConfs.size())); /** * shows stats about the tests * */ sb.append("---"); sb.append(">>>>>stats about the tests<<<<<\n"); sb.append("Test | #Total | #F | #P | #Fv |\n"); for (TestInfo ti : rankedList) { sb.append( // "TC"+ ti.id + "(" + ti.testName + ") | " + ti.testName + "|" + (ti.failures.size() + ti.passes.size()) + " | " + ti.failures.size() + " | " + ti.passes.size() + " |\n"); // ti.passes.size() + " | " + // ti.failuresValid.size() + " |\n"); // if (!ti.failures.isEmpty()) { // for (List<COV> c : ti.failures) { // sb.append(String.format(" %s\n", c)); // } // } // sb.append(String.format("%d : %s\n", ti.id, ti.testName)); // sb.append(String.format("%s : %d\n", "#TotalConsfs", // (ti.failures.size()+ti.passes.size()))); // sb.append(String.format("%s : %d\n", "#F", ti.failures.size())); // sb.append(String.format("%s : %d\n", "#P", ti.passes.size())); // sb.append(String.format("%s : %d\n", "#Fv", // ti.failuresValid.size())); // for (List<COV> c : ti.failuresValid) { // sb.append(String.format(" %s\n", c)); // } } }
public void startTest(Test test) { runCount++; TestInfo testInfo = testMap.get(test); testInfo.startTest(); message = Messages.getInstance().getString("UI.USER_TEST_SUITE_RUNNING"); // $NON-NLS-1$ }
public void runTest(String suiteClass, String method) { TestInfo test = getTest(suiteClass, method); test.run(); }