private boolean checkForDeterministicAndRerunTestQueriesIfNeeded() {
// check if the control query is deterministic
for (int i = 0; i < 3; i++) {
QueryResult results = executeQueryControl();
if (results.getState() != State.SUCCESS) {
return false;
}
if (!resultsMatch(controlResult, results, precision)) {
deterministic = false;
return false;
}
}
// Re-run the test query to confirm that the results don't match, in case there was caching on
// the test tier,
// but require that it matches 3 times in a row to rule out a non-deterministic correctness bug.
for (int i = 0; i < 3; i++) {
testResult = executeQueryTest();
if (testResult.getState() != State.SUCCESS) {
return false;
}
if (!resultsMatch(controlResult, testResult, precision)) {
return false;
}
}
// test result agrees with control result 3 times in a row although the first test result didn't
// agree
return true;
}
private boolean validate() {
controlResult = executeQueryControl();
// query has too many rows. Consider blacklisting.
if (controlResult.getState() == State.TOO_MANY_ROWS) {
testResult = new QueryResult(State.INVALID, null, null, ImmutableList.<List<Object>>of());
return false;
}
// query failed in the control
if (controlResult.getState() != State.SUCCESS) {
testResult = new QueryResult(State.INVALID, null, null, ImmutableList.<List<Object>>of());
return true;
}
testResult = executeQueryTest();
if (controlResult.getState() != State.SUCCESS || testResult.getState() != State.SUCCESS) {
return false;
}
if (!checkCorrectness) {
return true;
}
return resultsMatch(controlResult, testResult, precision)
|| checkForDeterministicAndRerunTestQueriesIfNeeded();
}
private static boolean resultsMatch(
QueryResult controlResult, QueryResult testResult, int precision) {
SortedMultiset<List<Object>> control =
ImmutableSortedMultiset.copyOf(rowComparator(precision), controlResult.getResults());
SortedMultiset<List<Object>> test =
ImmutableSortedMultiset.copyOf(rowComparator(precision), testResult.getResults());
try {
return control.equals(test);
} catch (TypesDoNotMatchException e) {
return false;
}
}
private QueryResult executeQueryControl() {
Query query = queryPair.getControl();
QueryResult queryResult =
new QueryResult(State.INVALID, null, null, ImmutableList.<List<Object>>of());
try {
// startup
queryResult =
setup(
query,
controlPreQueryResults,
controlPrequery ->
executeQuery(
controlGateway,
controlUsername,
controlPassword,
queryPair.getControl(),
controlPrequery,
controlTimeout,
sessionProperties));
// if startup is successful -> execute query
if (queryResult.getState() == State.SUCCESS) {
queryResult =
executeQuery(
controlGateway,
controlUsername,
controlPassword,
queryPair.getControl(),
query.getQuery(),
controlTimeout,
sessionProperties);
}
} finally {
// teardown no matter what
QueryResult tearDownResult =
tearDown(
query,
controlPostQueryResults,
controlPostquery ->
executeQuery(
controlGateway,
controlUsername,
controlPassword,
queryPair.getControl(),
controlPostquery,
controlTimeout,
sessionProperties));
// if teardown is not successful the query fails
queryResult = tearDownResult.getState() == State.SUCCESS ? queryResult : tearDownResult;
}
return queryResult;
}
private QueryResult setup(
Query query, List<QueryResult> preQueryResults, Function<String, QueryResult> executor) {
preQueryResults.clear();
for (String prequeryString : query.getPreQueries()) {
QueryResult queryResult = executor.apply(prequeryString);
preQueryResults.add(queryResult);
if (queryResult.getState() != State.SUCCESS) {
return new QueryResult(
State.FAILED_TO_SETUP,
queryResult.getException(),
queryResult.getDuration(),
ImmutableList.<List<Object>>of());
}
}
return new QueryResult(State.SUCCESS, null, null, ImmutableList.of());
}
public String getResultsComparison(int precision) {
List<List<Object>> controlResults = controlResult.getResults();
List<List<Object>> testResults = testResult.getResults();
if (valid() || (controlResults == null) || (testResults == null)) {
return "";
}
Multiset<List<Object>> control =
ImmutableSortedMultiset.copyOf(rowComparator(precision), controlResults);
Multiset<List<Object>> test =
ImmutableSortedMultiset.copyOf(rowComparator(precision), testResults);
try {
Iterable<ChangedRow> diff =
ImmutableSortedMultiset.<ChangedRow>naturalOrder()
.addAll(
Iterables.transform(
Multisets.difference(control, test),
row -> new ChangedRow(Changed.REMOVED, row, precision)))
.addAll(
Iterables.transform(
Multisets.difference(test, control),
row -> new ChangedRow(Changed.ADDED, row, precision)))
.build();
diff = Iterables.limit(diff, 100);
StringBuilder sb = new StringBuilder();
sb.append(format("Control %s rows, Test %s rows%n", control.size(), test.size()));
if (verboseResultsComparison) {
Joiner.on("\n").appendTo(sb, diff);
} else {
sb.append("RESULTS DO NOT MATCH\n");
}
return sb.toString();
} catch (TypesDoNotMatchException e) {
return e.getMessage();
}
}