private static void consolidate(
@Nonnull final TestMatrixArtifact testMatrix,
@Nonnull final Map<String, TestSpecification> requiredTests) {
final Map<String, ConsumableTestDefinition> definedTests = testMatrix.getTests();
// Sets.difference returns a "view" on the original set, which would require concurrent
// modification while iterating (copying the set will prevent this)
final Set<String> toRemove =
ImmutableSet.copyOf(Sets.difference(definedTests.keySet(), requiredTests.keySet()));
for (String testInMatrixNotRequired : toRemove) {
// we don't care about this test
definedTests.remove(testInMatrixNotRequired);
}
// Next, for any required tests that are missing, ensure that
// there is a nonnull test definition in the matrix
final Set<String> missing =
ImmutableSet.copyOf(Sets.difference(requiredTests.keySet(), definedTests.keySet()));
for (String testNotInMatrix : missing) {
definedTests.put(
testNotInMatrix, defaultFor(testNotInMatrix, requiredTests.get(testNotInMatrix)));
}
// Now go through definedTests: for each test, if the test spec
// didn't ask for a payload, then remove any payload that is in
// the test matrix.
for (Entry<String, ConsumableTestDefinition> next : definedTests.entrySet()) {
final String testName = next.getKey();
final ConsumableTestDefinition testDefinition = next.getValue();
final TestSpecification testSpec = requiredTests.get(testName);
if (testSpec.getPayload() == null) {
// No payload was requested...
final List<TestBucket> buckets = testDefinition.getBuckets();
for (final TestBucket bucket : buckets) {
if (bucket.getPayload() != null) {
// ... so stomp the unexpected payloads.
bucket.setPayload(null);
}
}
}
}
}
public static void verifyInternallyConsistentDefinition(
final String testName,
final String matrixSource,
@Nonnull final ConsumableTestDefinition testDefinition)
throws IncompatibleTestMatrixException {
final List<Allocation> allocations = testDefinition.getAllocations();
if (allocations.isEmpty()) {
throw new IncompatibleTestMatrixException("No allocations specified in test " + testName);
}
final List<TestBucket> buckets = testDefinition.getBuckets();
/*
* test the matrix for consistency with itself
*/
final Set<Integer> definedBuckets = Sets.newHashSet();
for (final TestBucket bucket : buckets) {
definedBuckets.add(bucket.getValue());
}
for (final Allocation allocation : allocations) {
final List<Range> ranges = allocation.getRanges();
// ensure that each range refers to a known bucket
double bucketTotal = 0;
for (final Range range : ranges) {
bucketTotal += range.getLength();
// Internally consistent (within matrix itself)
if (!definedBuckets.contains(range.getBucketValue())) {
throw new IncompatibleTestMatrixException(
"Allocation range in "
+ testName
+ " from "
+ matrixSource
+ " refers to unknown bucket value "
+ range.getBucketValue());
}
}
// I hate floating points. TODO: extract a required precision constant/parameter?
if (bucketTotal < 0.9999
|| bucketTotal
> 1.0001) { // compensate for FP imprecision. TODO: determine what these bounds
// really should be by testing stuff
final StringBuilder sb =
new StringBuilder(
testName
+ " range with rule "
+ allocation.getRule()
+ " does not add up to 1 : ")
.append(ranges.get(0).getLength());
for (int i = 1; i < ranges.size(); i++) {
sb.append(" + ").append(ranges.get(i).getLength());
}
sb.append(" = ").append(bucketTotal);
throw new IncompatibleTestMatrixException(sb.toString());
}
}
final Allocation lastAllocation = allocations.get(allocations.size() - 1);
if (!CharMatcher.WHITESPACE.matchesAllOf(Strings.nullToEmpty(lastAllocation.getRule()))) {
throw new IncompatibleTestMatrixException(
"Final allocation for test "
+ testName
+ " from "
+ matrixSource
+ " has non-empty rule: "
+ lastAllocation.getRule());
}
/*
* When defined, within a single test, all test bucket payloads
* should be supplied; they should all have just one type each,
* and they should all be the same type.
*/
Payload nonEmptyPayload = null;
final List<TestBucket> bucketsWithoutPayloads = Lists.newArrayList();
for (final TestBucket bucket : buckets) {
final Payload p = bucket.getPayload();
if (p != null) {
if (p.numFieldsDefined() != 1) {
throw new IncompatibleTestMatrixException(
"Test "
+ testName
+ " from "
+ matrixSource
+ " has a test bucket payload with multiple types: "
+ bucket);
}
if (nonEmptyPayload == null) {
nonEmptyPayload = p;
} else if (!nonEmptyPayload.sameType(p)) {
throw new IncompatibleTestMatrixException(
"Test "
+ testName
+ " from "
+ matrixSource
+ " has test bucket: "
+ bucket
+ " incompatible with type of payload: "
+ nonEmptyPayload);
}
} else {
bucketsWithoutPayloads.add(bucket);
}
}
if ((nonEmptyPayload != null) && (bucketsWithoutPayloads.size() != 0)) {
throw new IncompatibleTestMatrixException(
"Test "
+ testName
+ " from "
+ matrixSource
+ " has some test buckets without payloads: "
+ bucketsWithoutPayloads);
}
}
private static void verifyTest(
String testName,
@Nonnull ConsumableTestDefinition testDefinition,
TestSpecification testSpecification,
@Nonnull Map<Integer, String> knownBuckets,
String matrixSource,
FunctionMapper functionMapper)
throws IncompatibleTestMatrixException {
final List<Allocation> allocations = testDefinition.getAllocations();
verifyInternallyConsistentDefinition(testName, matrixSource, testDefinition);
/*
* test the matrix for adherence to this application's requirements
*/
final Set<Integer> unknownBuckets = Sets.newHashSet();
for (final Allocation allocation : allocations) {
final List<Range> ranges = allocation.getRanges();
// ensure that each range refers to a known bucket
for (final Range range : ranges) {
// Externally consistent (application's requirements)
if (!knownBuckets.containsKey(range.getBucketValue())) {
// If the bucket has a positive allocation, add it to the list of unknownBuckets
if (range.getLength() > 0) {
unknownBuckets.add(range.getBucketValue());
}
}
}
}
if (unknownBuckets.size() > 0) {
throw new IncompatibleTestMatrixException(
"Allocation range in "
+ testName
+ " from "
+ matrixSource
+ " refers to unknown bucket value(s) "
+ unknownBuckets
+ " with length > 0");
}
// TODO(pwp): add some test constants?
final RuleEvaluator ruleEvaluator =
makeRuleEvaluator(RuleEvaluator.EXPRESSION_FACTORY, functionMapper);
PayloadSpecification payloadSpec = testSpecification.getPayload();
if (payloadSpec != null) {
final String specifiedPayloadTypeName =
Preconditions.checkNotNull(payloadSpec.getType(), "Missing payload spec type");
final PayloadType specifiedPayloadType =
PayloadType.payloadTypeForName(specifiedPayloadTypeName);
if (specifiedPayloadType == null) {
// This is probably redundant vs. TestGroupsGenerator.
throw new IncompatibleTestMatrixException(
"For test "
+ testName
+ " from "
+ matrixSource
+ " test specification payload type unknown: "
+ specifiedPayloadTypeName);
}
final String payloadValidatorRule = payloadSpec.getValidator();
final List<TestBucket> buckets = testDefinition.getBuckets();
for (final TestBucket bucket : buckets) {
Payload payload = bucket.getPayload();
if (payload != null) {
if (!specifiedPayloadType.payloadHasThisType(payload)) {
throw new IncompatibleTestMatrixException(
"For test "
+ testName
+ " from "
+ matrixSource
+ " expected payload of type "
+ specifiedPayloadType.payloadTypeName
+ " but matrix has a test bucket payload with wrong type: "
+ bucket);
}
if (payloadValidatorRule != null) {
final boolean payloadIsValid =
evaluatePayloadValidator(ruleEvaluator, payloadValidatorRule, payload);
if (!payloadIsValid) {
throw new IncompatibleTestMatrixException(
"For test "
+ testName
+ " from "
+ matrixSource
+ " payload validation rule "
+ payloadValidatorRule
+ " failed for test bucket: "
+ bucket);
}
}
}
}
}
}
