public void excludeTest(String testName, String reason) {
TestInfo testInfo = findTestInfo(testName);
if (testInfo == null) return;
testInfo.isExcluded(true);
testInfo.reasonToExclude(reason);
}
public void addFailure(Test test, AssertionFailedError error) {
message = error.getMessage();
failCount++;
status = FAIL;
TestInfo testInfo = testMap.get(test);
testInfo.addFailure(error);
message = Messages.getInstance().getString("UI.USER_TEST_SUITE_FAILED"); // $NON-NLS-1$
}
public void addError(Test test, Throwable error) {
message = error.getMessage();
errorCount++;
status = FAIL;
TestInfo testInfo = testMap.get(test);
testInfo.addError(error);
message = Messages.getInstance().getString("UI.USER_TEST_SUITE_FAILED"); // $NON-NLS-1$
}
public void includeTest(String testName) {
TestInfo testInfo = findTestInfo(testName);
if (testInfo == null) {
testInfo = new TestInfo(this, testName);
m_testInfoList.add(testInfo);
}
testInfo.isExcluded(false);
}
@Override
public TestInfo clone() {
TestInfo result = new TestInfo(testName);
result.id = id;
result.passes = new ArrayList<List<COV>>(passes);
result.failures = new ArrayList<List<COV>>(failures);
result.failuresValid = new ArrayList<List<COV>>(failuresValid);
return result;
}
public void endTest(Test test) {
if (status != FAIL) {
passCount++;
}
status = NOT_RUNNING;
TestInfo testInfo = testMap.get(test);
testInfo.endTest();
message = ""; // $NON-NLS-1$
}
@Override
public void setUp() throws Exception {
mTestInstance =
new BluetoothStressTest() {
@Override
void reportMetrics(
ITestInvocationListener l, TestInfo test, Map<String, String> metrics) {
mReportedTestInfo = test;
mReportedMetrics = metrics;
}
};
mScanInfo = new TestInfo();
mScanInfo.mTestName = "scan";
mScanInfo.mTestMethod = "testScan";
mScanInfo.mIterCount = 1;
mScanInfo.mPerfMetrics.add("startScan");
mScanInfo.mPerfMetrics.add("stopScan");
}
public String toString(String prefix) {
if (prefix == null) prefix = "";
StringBuilder result = new StringBuilder();
result.append(prefix + "Directory: " + m_path + Utils.EOL);
result.append(
prefix
+ " | isExcluded = "
+ m_isExcluded
+ " [reason: "
+ StringUtils.nullOrString(m_reasonToExclude)
+ "]"
+ Utils.EOL);
for (TestInfo testInfo : m_testInfoList) {
result.append(testInfo.toString(prefix + " | ") + Utils.EOL);
}
return StringUtils.trimEnd(result.toString(), "\r\n");
}
public Node getStatus(Element parent) {
Element node = parent.addElement("suite"); // $NON-NLS-1$
node.addAttribute("class", className); // $NON-NLS-1$
node.addAttribute("name", name); // $NON-NLS-1$
node.addAttribute("test-count", Integer.toString(testCount)); // $NON-NLS-1$
node.addAttribute("run-count", Integer.toString(runCount)); // $NON-NLS-1$
node.addAttribute("pass-count", Integer.toString(passCount)); // $NON-NLS-1$
node.addAttribute("fail-count", Integer.toString(errorCount + failCount)); // $NON-NLS-1$
Element messageNode = node.addElement("message"); // $NON-NLS-1$
Element tests = node.addElement("tests"); // $NON-NLS-1$
int currentPassCount = 0;
int currentRunCount = 0;
int currentFailCount = 0;
for (int idx = 0; idx < testList.size(); idx++) {
TestInfo test = testList.get(idx);
test.getStatus(tests);
if (test.lastResult == PASS) {
currentPassCount++;
}
if (test.lastResult == FAIL) {
currentFailCount++;
}
if (test.lastResult != test.NOT_RUN) {
currentRunCount++;
}
}
if (currentRunCount == 0) {
message = Messages.getInstance().getString("UI.USER_TEST_SUITE_NOT_RUN"); // $NON-NLS-1$
} else if (currentRunCount == currentPassCount && currentRunCount < testCount) {
message = Messages.getInstance().getString("UI.USER_TEST_SUITE_SOME_PASSED"); // $NON-NLS-1$
} else if (currentRunCount == currentPassCount && currentRunCount == testCount) {
message = Messages.getInstance().getString("UI.USER_TEST_SUITE_ALL_PASSED"); // $NON-NLS-1$
} else if (currentFailCount > 0) {
message =
Messages.getInstance().getString("UI.USER_TEST_SUITE_SOME_FAILURES"); // $NON-NLS-1$
}
messageNode.setText((message == null) ? "" : message); // $NON-NLS-1$
return node;
}
/**
* Report run metrics by creating an empty test run to stick them in
*
* <p>Exposed for unit testing
*/
void reportMetrics(ITestInvocationListener listener, TestInfo test, Map<String, String> metrics) {
// Create an empty testRun to report the parsed runMetrics
CLog.d("About to report metrics to %s: %s", test.getTestMetricsName(), metrics);
listener.testRunStarted(test.getTestMetricsName(), 0);
listener.testRunEnded(0, metrics);
}
private void setupTests() {
if (mTestCases != null) {
// assume already set up
return;
}
// Allocate enough space for all of the TestInfo instances below
mTestCases = new ArrayList<TestInfo>(12);
TestInfo t = new TestInfo();
t.mTestName = "discoverable";
t.mTestMethod = "testDiscoverable";
t.mIterKey = "discoverable_iterations";
t.mIterCount = mDiscoverableIterations;
t.mPerfMetrics.add("discoverable");
t.mPerfMetrics.add("undiscoverable");
mTestCases.add(t);
t = new TestInfo();
t.mTestName = "enable";
t.mTestMethod = "testEnable";
t.mIterKey = "enable_iterations";
t.mIterCount = mEnableIterations;
t.mPerfMetrics.add("enable");
t.mPerfMetrics.add("disable");
mTestCases.add(t);
t = new TestInfo();
t.mTestName = "scan";
t.mTestMethod = "testScan";
t.mIterKey = "scan_iterations";
t.mIterCount = mScanIterations;
t.mPerfMetrics.add("startScan");
t.mPerfMetrics.add("stopScan");
mTestCases.add(t);
t = new TestInfo();
t.mTestName = "enable_pan";
t.mTestMethod = "testEnablePan";
t.mIterKey = "enable_pan_iterations";
t.mIterCount = mEnablePanIterations;
t.mPerfMetrics.add("enablePan");
t.mPerfMetrics.add("disablePan");
mTestCases.add(t);
// Remote device tests
t = new TestInfo();
t.mTestName = "pair";
t.mTestMethod = "testPair";
t.mIterKey = "pair_iterations";
t.mIterCount = mPairIterations;
t.mPerfMetrics.add("pair");
t.mPerfMetrics.add("unpair");
t.mRemoteAddress = mDeviceAddress;
t.mPairPin = mDevicePairPin;
t.mPairPasskey = mDevicePairPasskey;
t.mInstructions =
String.format(
"Start the testAcceptPair instrumentation on the remote "
+ "Android device with the following command:\n\n%s\n\nHit Enter when done.",
String.format(
INSTRUCTIONS_INSTRUMENT_CMD,
mDeviceSerial,
mLocalAddress,
"pair_iterations",
mPairIterations,
mTestClassName,
"testAcceptPair",
mTestPackageName,
TEST_RUNNER_NAME));
mTestCases.add(t);
t = new TestInfo();
t.mTestName = "accept_pair";
t.mTestMethod = "testAcceptPair";
t.mIterKey = "pair_iterations";
t.mIterCount = mAcceptPairIterations;
t.mPerfMetrics.add("acceptPair");
t.mPerfMetrics.add("unpair");
t.mRemoteAddress = mDeviceAddress;
t.mPairPin = mDevicePairPin;
t.mPairPasskey = mDevicePairPasskey;
t.mInstructions =
String.format(
"Start the testPair instrumentation on the remote "
+ "Android device with the following command:\n\n%s\n\nHit Enter when done.",
String.format(
INSTRUCTIONS_INSTRUMENT_CMD,
mDeviceSerial,
mLocalAddress,
"pair_iterations",
mAcceptPairIterations,
mTestClassName,
"testPair",
mTestPackageName,
TEST_RUNNER_NAME));
mTestCases.add(t);
t = new TestInfo();
t.mTestName = "connect_headset";
t.mTestMethod = "testConnectHeadset";
t.mIterKey = "connect_headset_iterations";
t.mIterCount = mConnectHeadsetIterations;
t.mPerfMetrics.add("connectHeadset");
t.mPerfMetrics.add("disconnectHeadset");
t.mRemoteAddress = mHeadsetAddress;
t.mPairPin = mHeadsetPairPin;
t.mPairPasskey = mHeadsetPairPasskey;
t.mInstructions = "Put the remote headset device in pairing mode. Hit Enter when done.";
mTestCases.add(t);
t = new TestInfo();
t.mTestName = "connect_a2dp";
t.mTestMethod = "testConnectA2dp";
t.mIterKey = "connect_a2dp_iterations";
t.mIterCount = mConnectA2dpIterations;
t.mPerfMetrics.add("connectA2dp");
t.mPerfMetrics.add("disconnectA2dp");
t.mRemoteAddress = mA2dpAddress;
t.mPairPin = mA2dpPairPin;
t.mPairPasskey = mA2dpPairPasskey;
t.mInstructions = "Put the remote A2DP device in pairing mode. Hit Enter when done.";
mTestCases.add(t);
t = new TestInfo();
t.mTestName = "connect_input";
t.mTestMethod = "testConnectInput";
t.mIterKey = "connect_input_iterations";
t.mIterCount = mConnectInputIterations;
t.mPerfMetrics.add("connectInput");
t.mPerfMetrics.add("disconnectInput");
t.mRemoteAddress = mInputAddress;
t.mPairPin = mInputPairPin;
t.mPairPasskey = mInputPairPasskey;
t.mInstructions = "Put the remote input device in pairing mode. Hit Enter when done.";
mTestCases.add(t);
t = new TestInfo();
t.mTestName = "connect_pan";
t.mTestMethod = "testConnectPan";
t.mIterKey = "connect_pan_iterations";
t.mIterCount = mConnectPanIterations;
t.mPerfMetrics.add("connectPan");
t.mPerfMetrics.add("disconnectPan");
t.mRemoteAddress = mDeviceAddress;
t.mPairPin = mDevicePairPin;
t.mPairPasskey = mDevicePairPasskey;
t.mInstructions =
String.format(
"Start the testIncomingPanConnection instrumentation on "
+ "the remote Android device with the following command:\n\n%s\n\nHit Enter when "
+ "done.",
String.format(
INSTRUCTIONS_INSTRUMENT_CMD,
mDeviceSerial,
mLocalAddress,
"connect_pan_iterations",
mConnectPanIterations,
mTestClassName,
"testIncomingPanConnection",
mTestPackageName,
TEST_RUNNER_NAME));
mTestCases.add(t);
t = new TestInfo();
t.mTestName = "incoming_pan_connection";
t.mTestMethod = "testIncomingPanConnection";
t.mIterKey = "connect_pan_iterations";
t.mIterCount = mIncomingPanConnectionIterations;
t.mPerfMetrics.add("incomingPanConnection");
t.mPerfMetrics.add("incomingPanDisconnection");
t.mRemoteAddress = mDeviceAddress;
t.mPairPin = mDevicePairPin;
t.mPairPasskey = mDevicePairPasskey;
t.mInstructions =
("Start the testConnectPan instrumentation on the remote Android "
+ "device. Hit Enter when done.");
t.mInstructions =
String.format(
"Start the testConnectPan instrumentation on the remote "
+ "Android device with the following command:\n\n%s\n\nHit Enter when done.",
String.format(
INSTRUCTIONS_INSTRUMENT_CMD,
mDeviceSerial,
mLocalAddress,
"connect_pan_iterations",
mIncomingPanConnectionIterations,
mTestClassName,
"testConnectPan",
mTestPackageName,
TEST_RUNNER_NAME));
mTestCases.add(t);
t = new TestInfo();
t.mTestName = "start_stop_sco";
t.mTestMethod = "testStartStopSco";
t.mIterKey = "start_stop_sco_iterations";
t.mIterCount = mStartStopScoIterations;
t.mPerfMetrics.add("startSco");
t.mPerfMetrics.add("stopSco");
t.mRemoteAddress = mHeadsetAddress;
t.mPairPin = mHeadsetPairPin;
t.mPairPasskey = mHeadsetPairPasskey;
t.mInstructions = "Put the remote headset device in pairing mode. Hit Enter when done.";
mTestCases.add(t);
}
private TestInfo findTestInfo(String testName) {
for (TestInfo info : m_testInfoList) {
if (info.name().equalsIgnoreCase(testName)) return info;
}
return null;
}
private void summarize(StringBuffer sb)
throws ClassNotFoundException, NoSuchMethodException, SecurityException,
IllegalAccessException, IllegalArgumentException, InvocationTargetException,
InstantiationException {
// Map<Double,Integer> ratioDistribution = new
// TreeMap<Double,Integer>();
// Map<Integer,Integer> runDistribution = new
// TreeMap<Integer,Integer>();
// Map<String, Integer> testsFailInValidConfs = new TreeMap<String,
// Integer>();
// Map<String, Integer> testsFailValidConfs = new TreeMap<String,
// Integer>();
// Set<String> failingTestsValidConfs = new TreeSet<String>();
Map<Integer, Integer> numConfigurationFailInNTests = new HashMap<Integer, Integer>();
Set<String> removeme = new TreeSet<String>();
for (Entry e : counts) {
removeme.addAll(e.failingTestNames);
removeme.addAll(e.passingTestNames);
}
for (Entry entry : counts) {
// int numPassingRuns = entry.numPassing;
// int numTotalRuns = entry.numExecs;
// if (numTotalRuns == 0) {
// continue;
// }
// double frac = ((double) numPassingRuns) / numTotalRuns;
// double fraction = Math.floor(frac * 100) / 100;
//
// // distributions
// calculateRatioDistribution(ratioDistribution, fraction);
// calculateRunDistribution(runDistribution, numTotalRuns);
// failing tests
for (String s : entry.failingTestNames) {
TestInfo ti = tInfo.get(s);
if (ti == null) {
ti = new TestInfo(s);
tInfo.put(s, ti);
}
ti.failures.add(entry.conf);
if (entry.isValid) {
ti.failuresValid.add(entry.conf);
}
}
for (String s : entry.passingTestNames) {
TestInfo ti = tInfo.get(s);
if (ti == null) {
ti = new TestInfo(s);
tInfo.put(s, ti);
}
ti.passes.add(entry.conf);
}
if (entry.numExecs - entry.numPassing > 0 && entry.isValid) {
int size = entry.failingTestNames.size();
Integer k = numConfigurationFailInNTests.get(size);
if (k == null) k = 0;
numConfigurationFailInNTests.put(size, ++k);
// failingConfs.append(String.format("%s\n %d: %s\n",
// entry.conf, size,
// entry.failingTestNames));
// failingTestsValidConfs.addAll(entry.failingTestNames);
// for (String testName : entry.failingTestNames) {
// Integer i = testsFailValidConfs.get(testName);
// if (i == null) i = 0;
// testsFailValidConfs.put(testName, ++i);
// }
}
//
// if (entry.numExecs - entry.numPassing > 0 && !entry.isValid) {
// for (String testName : entry.failingTestNames) {
// Integer i = testsFailInValidConfs.get(testName);
// if (i == null) i = 0;
// testsFailInValidConfs.put(testName, ++i);
// }
// }
}
/** shows the distribution of configurations */
sb.append("==== summary\n");
for (Integer i : numConfigurationFailInNTests.keySet()) {
Integer j = numConfigurationFailInNTests.get(i);
sb.append(String.format("%s configurations fail in %s tests\n", j, i));
}
sb.append("==== latex summary of configs \n");
sb.append("\\# Confs. & \\# Tests\\\\ \n");
sb.append("\\hline\n");
for (Integer i : numConfigurationFailInNTests.keySet()) {
Integer j = numConfigurationFailInNTests.get(i);
sb.append(String.format("%s & %s\\\\ \n", j, i));
sb.append("\\hline\n");
}
/** shows the list of failing, passing, and failing (valid) configurations */
Comparator<TestInfo> comp =
new Comparator<LogReader.TestInfo>() {
@Override
public int compare(TestInfo o1, TestInfo o2) {
int result;
if (o1.failuresValid.size() != 0 && o2.failuresValid.size() == 0) {
result = 1;
} else if (o1.failuresValid.size() == 0 && o2.failuresValid.size() != 0) {
result = -1;
} else {
int sizeOne = o1.failures.size();
int sizeTwo = o2.failures.size();
int diff = sizeOne - sizeTwo;
if (diff > 0) {
result = 1; // o1 greater than o2
} else if (diff < 0) {
result = -1;
} else {
sizeOne = o1.failuresValid.size();
sizeTwo = o2.failuresValid.size();
diff = sizeOne - sizeTwo;
if (diff > 0) {
result = 1;
} else if (diff < 0) {
result = -1;
} else {
result = 0;
}
}
}
return result;
}
};
List<TestInfo> l = new ArrayList<TestInfo>(tInfo.values());
Collections.sort(l, comp);
int id = 1;
sb.append("\\emph{Ti} & \\emph{F} & \\emph{P} & \\emph{Fv}\\\\\n");
sb.append("\\hline\n");
sb.append("\\multicolumn{4}{|c|}{$\\dots$}\\\\\n");
sb.append("\\hline\n\\hline\n");
boolean block1 = true;
boolean block2 = true;
for (TestInfo ti : l) {
if (block1 && ti.failuresValid.size() != 0) {
block1 = false;
}
if (block2 && ti.failures.size() != 0) {
sb.append("\\hline\n\\hline\n");
block2 = false;
}
if (!block1 || !block2) {
sb.append(
String.format(
"%s & %s & %s & %s \\\\ \n",
id, ti.failures.size(), ti.passes.size(), ti.failuresValid.size()));
}
ti.setTestId(id);
id++;
}
/** shows the ranking of tests */
Comparator<TestInfo> ranker =
new Comparator<LogReader.TestInfo>() {
@Override
public int compare(TestInfo o1, TestInfo o2) {
int numFailuresOne = o1.failures.size();
int numFailuresTwo = o2.failures.size();
double ratioOne = o1.ratioPF();
double ratioTwo = o2.ratioPF();
int result;
if (ratioOne > ratioTwo) {
result = 1;
} else if (ratioTwo > ratioOne) {
result = -1;
} else { // tie-breaker
if (numFailuresOne > numFailuresTwo) {
result = 1;
} else if (numFailuresOne > numFailuresTwo) {
result = -1;
} else {
result = 0;
}
}
return result;
}
};
List<TestInfo> rankedList = new ArrayList<TestInfo>(tInfo.values());
Collections.sort(rankedList, Collections.reverseOrder(ranker));
int rank = 1;
for (TestInfo ti : rankedList) {
double ratio = ti.ratioPF();
if (ratio > 0) {
if (ti.failuresValid.size() > 0) {
sb.append("\\rowcolor{gray}\n");
}
sb.append(String.format("%d & %d & %.2f\\\\ \n", rank++, ti.id, ratio));
}
}
Set<String> ReachableFailingConfs = new TreeSet<String>();
Set<String> ReachableFailingValidConfs = new TreeSet<String>();
/** * shows the list of tests with ratio > 0 and valid failing configurations * */
sb.append("---");
sb.append("ranked test list (valid configurations highlighted)\n");
for (TestInfo ti : rankedList) {
// finding the reachable failing distinct confs
for (List<COV> failConf : ti.failures) {
ReachableFailingConfs.add(failConf.toString());
}
// finding the reachable failing valid distinct confs
for (List<COV> failConf : ti.failuresValid) {
ReachableFailingValidConfs.add(failConf.toString());
}
// double ratio = ti.ratioPF();
// if (ratio > 0) {
// if (ti.failuresValid.size() > 0) {
// sb.append(String.format("%d : %s\n", ti.id, ti.testName));
// for (List<COV> c : ti.failuresValid) {
// sb.append(String.format(" %s\n", c));
// }
// }
// }
}
sb.append(
String.format(
"----->Number of reachable failing confs: %d\n", ReachableFailingConfs.size()));
sb.append(
String.format(
"----->Number of reachable failing valid confs: %d\n",
ReachableFailingValidConfs.size()));
/** * shows stats about the tests * */
sb.append("---");
sb.append(">>>>>stats about the tests<<<<<\n");
sb.append("Test | #Total | #F | #P | #Fv |\n");
for (TestInfo ti : rankedList) {
sb.append(
// "TC"+ ti.id + "(" + ti.testName + ") | " +
ti.testName
+ "|"
+ (ti.failures.size() + ti.passes.size())
+ " | "
+ ti.failures.size()
+ " | "
+ ti.passes.size()
+ " |\n");
// ti.passes.size() + " | " +
// ti.failuresValid.size() + " |\n");
// if (!ti.failures.isEmpty()) {
// for (List<COV> c : ti.failures) {
// sb.append(String.format(" %s\n", c));
// }
// }
// sb.append(String.format("%d : %s\n", ti.id, ti.testName));
// sb.append(String.format("%s : %d\n", "#TotalConsfs",
// (ti.failures.size()+ti.passes.size())));
// sb.append(String.format("%s : %d\n", "#F", ti.failures.size()));
// sb.append(String.format("%s : %d\n", "#P", ti.passes.size()));
// sb.append(String.format("%s : %d\n", "#Fv",
// ti.failuresValid.size()));
// for (List<COV> c : ti.failuresValid) {
// sb.append(String.format(" %s\n", c));
// }
}
}
public void startTest(Test test) {
runCount++;
TestInfo testInfo = testMap.get(test);
testInfo.startTest();
message = Messages.getInstance().getString("UI.USER_TEST_SUITE_RUNNING"); // $NON-NLS-1$
}
public void runTest(String suiteClass, String method) {
TestInfo test = getTest(suiteClass, method);
test.run();
}
