/** assume the stack has already been populated by the previous test */
public void testCursoredStreamPolicy() {
ValueReadQuery sizeQuery = new ValueReadQuery("select count(*) from ORD");
DataReadQuery query = this.buildNewQuery();
query.useCursoredStream(5, 5, sizeQuery);
CursoredStream stream = (CursoredStream) getSession().executeQuery(query);
// if we get here, we must not have generated a ClassCastException
int count = 0;
while (stream.hasMoreElements()) {
count++;
Record row = (Record) stream.nextElement();
if (row.get("CUSTNAME") == null) {
throw new TestErrorException("missing data");
}
}
if (count != stack.size()) {
throw new TestErrorException(
"stream does not match stack - " + "expected: " + stack.size() + " actual: " + count);
}
}
/**
* Load the last test result from the test result database. Find only the results run on the same
* machine and database. Compare the current test run result with the previous results do
* determine if the test passes or fails.
*/
public static void verify(PerformanceRegressionTest test) {
// Ensures all tests pass to reset baseline,
// Required when tests or environment change to be slower to avoid failures.
if (reset) {
throw new TestWarningException("Reseting baseline.");
}
Session session = LoadBuildSystem.getSystem().getSession();
// Query all previous successful test results for this test on the same machine and database.
// Find only the baseline version, or last version run different than current version.
// If you need to compare results against the current version you must change the TopLink
// version string.
ReadAllQuery query = new ReadAllQuery(TestResult.class);
ExpressionBuilder result = new ExpressionBuilder();
query.setSelectionCriteria(
result
.get("name")
.equal(test.getName())
.and(
result
.get("loadBuildSummary")
.get("machine")
.equal(LoadBuildSystem.getSummary().getMachine()))
.and(
result
.get("loadBuildSummary")
.get("loginChoice")
.equal(LoadBuildSystem.getSummary().getLoginChoice())));
// Allow comparing to a set version through a system property.
String currentVersion = LoadBuildSystem.getSummary().getToplinkVersion();
String baselineVersion = null;
if (System.getProperties().containsKey("toplink.loadbuild.baseline-version")) {
baselineVersion = System.getProperties().getProperty("toplink.loadbuild.baseline-version");
// System properties cannot store spaces so need to replace them from \b.
baselineVersion = baselineVersion.replace('_', ' ');
((PerformanceComparisonTestResult) ((TestCase) test).getTestResult()).baselineVersion =
baselineVersion;
} else {
// Compare against the last successful version.
ReportQuery reportQuery = new ReportQuery(TestResult.class, query.getExpressionBuilder());
reportQuery.useDistinct();
reportQuery.returnSingleValue();
reportQuery.addAttribute("version", result.get("loadBuildSummary").get("toplinkVersion"));
reportQuery.setSelectionCriteria(
query
.getSelectionCriteria()
.and(
(result.get("outcome").equal(TestResult.PASSED))
.or(result.get("outcome").equal(TestResult.WARNING)))
.and(result.get("loadBuildSummary").get("toplinkVersion").notEqual(currentVersion)));
reportQuery.addOrdering(result.get("loadBuildSummary").get("timestamp").descending());
baselineVersion = (String) session.executeQuery(reportQuery);
}
query.setSelectionCriteria(
query
.getSelectionCriteria()
.and(result.get("loadBuildSummary").get("toplinkVersion").equal(baselineVersion)));
query.addOrdering(result.get("loadBuildSummary").get("timestamp").descending());
query.setMaxRows(10);
query.useCursoredStream(1, 1);
CursoredStream stream = (CursoredStream) session.executeQuery(query);
if (!stream.hasMoreElements()) {
throw new TestWarningException("No previous test result to compare performance with.");
}
TestResult lastResult = (TestResult) stream.nextElement();
double lastCount = lastResult.getTestTime();
PerformanceComparisonTestResult testResult =
(PerformanceComparisonTestResult) ((TestCase) test).getTestResult();
testResult.getBaselineVersionResults().add(new Double(lastCount));
// Average last 5 runs.
int numberOfRuns = 0;
while (stream.hasMoreElements() && (numberOfRuns < 4)) {
TestResult nextResult = (TestResult) stream.nextElement();
testResult.getBaselineVersionResults().add(new Double(nextResult.getTestTime()));
numberOfRuns++;
}
stream.close();
double baselineAverage =
PerformanceComparisonTestResult.averageResults(testResult.getBaselineVersionResults());
double currentCount = ((TestCase) test).getTestResult().getTestTime();
testResult.baselineVersion = lastResult.getLoadBuildSummary().getToplinkVersion();
testResult.percentageDifferenceLastRun =
PerformanceComparisonTestResult.percentageDifference(currentCount, lastCount);
// Query the current version last 5 runs for averaging.
query = new ReadAllQuery(TestResult.class);
result = new ExpressionBuilder();
query.setSelectionCriteria(
result
.get("name")
.equal(((TestCase) test).getName())
.and(
result
.get("loadBuildSummary")
.get("machine")
.equal(LoadBuildSystem.getSummary().getMachine()))
.and(
result
.get("loadBuildSummary")
.get("loginChoice")
.equal(LoadBuildSystem.getSummary().getLoginChoice()))
.and(result.get("loadBuildSummary").get("toplinkVersion").equal(currentVersion)));
query.addOrdering(result.get("loadBuildSummary").get("timestamp").descending());
query.useCursoredStream(1, 1);
stream = (CursoredStream) session.executeQuery(query);
// Average last 5 runs.
testResult.getCurrentVersionResults().add(new Double(currentCount));
numberOfRuns = 0;
while (stream.hasMoreElements() && (numberOfRuns < 4)) {
TestResult nextResult = (TestResult) stream.nextElement();
testResult.getCurrentVersionResults().add(new Double(nextResult.getTestTime()));
numberOfRuns++;
}
stream.close();
double currentAverage =
PerformanceComparisonTestResult.averageResults(testResult.getCurrentVersionResults());
testResult.percentageDifferenceAverage =
PerformanceComparisonTestResult.percentageDifference(currentAverage, baselineAverage);
testResult.baselineStandardDeviation =
PerformanceComparisonTestResult.standardDeviationResults(
testResult.getBaselineVersionResults());
testResult.currentStandardDeviation =
PerformanceComparisonTestResult.standardDeviationResults(
testResult.getCurrentVersionResults());
if (testResult.percentageDifferenceAverage < test.getAllowableDecrease()) {
throw new TestErrorException(
"Test is "
+ ((long) testResult.percentageDifferenceAverage)
+ "% slower than last successful execution.");
}
}
