private static void consolidate(
@Nonnull final TestMatrixArtifact testMatrix,
@Nonnull final Map<String, TestSpecification> requiredTests) {
final Map<String, ConsumableTestDefinition> definedTests = testMatrix.getTests();
// Sets.difference returns a "view" on the original set, which would require concurrent
// modification while iterating (copying the set will prevent this)
final Set<String> toRemove =
ImmutableSet.copyOf(Sets.difference(definedTests.keySet(), requiredTests.keySet()));
for (String testInMatrixNotRequired : toRemove) {
// we don't care about this test
definedTests.remove(testInMatrixNotRequired);
}
// Next, for any required tests that are missing, ensure that
// there is a nonnull test definition in the matrix
final Set<String> missing =
ImmutableSet.copyOf(Sets.difference(requiredTests.keySet(), definedTests.keySet()));
for (String testNotInMatrix : missing) {
definedTests.put(
testNotInMatrix, defaultFor(testNotInMatrix, requiredTests.get(testNotInMatrix)));
}
// Now go through definedTests: for each test, if the test spec
// didn't ask for a payload, then remove any payload that is in
// the test matrix.
for (Entry<String, ConsumableTestDefinition> next : definedTests.entrySet()) {
final String testName = next.getKey();
final ConsumableTestDefinition testDefinition = next.getValue();
final TestSpecification testSpec = requiredTests.get(testName);
if (testSpec.getPayload() == null) {
// No payload was requested...
final List<TestBucket> buckets = testDefinition.getBuckets();
for (final TestBucket bucket : buckets) {
if (bucket.getPayload() != null) {
// ... so stomp the unexpected payloads.
bucket.setPayload(null);
}
}
}
}
}
@Before
public void setupMocks() throws Exception {
expressionFactory = new ExpressionFactoryImpl();
functionMapper = RuleEvaluator.FUNCTION_MAPPER;
testName = "testName";
final List<TestBucket> buckets =
ImmutableList.of(
new TestBucket("inactive", -1, "zoot", null),
new TestBucket("control", 0, "zoot", null),
new TestBucket("test", 1, "zoot", null));
testDefinition = new ConsumableTestDefinition();
testDefinition.setConstants(Collections.<String, Object>emptyMap());
testDefinition.setTestType(TestType.AUTHENTICATED_USER);
// most tests just set the salt to be the same as the test name
testDefinition.setSalt(testName);
testDefinition.setBuckets(buckets);
updateAllocations(RANGES_50_50);
final int effBuckets = buckets.size() - 1;
counts = new int[effBuckets];
hashes = new int[effBuckets];
}
@Test
public void testSimple_50_50() {
testDefinition.setSalt(testName);
final StandardTestChooser chooser = newChooser();
exerciseChooser(chooser);
// uncomment this if you need to recompute these values
// for (int i = 0; i < counts.length; i++) System.err.println(i + ": " + counts[i] + " /
// " + hashes[i]);
// if this ever fails, it means that something is broken about how tests are split
// and you should investigate why!
Assert.assertEquals("bucket0 counts", 4999412, counts[0]);
Assert.assertEquals("bucket1 counts", 5000587, counts[1]);
Assert.assertEquals("bucket0 hash", 1863060514, hashes[0]);
Assert.assertEquals("bucket1 hash", 765061458, hashes[1]);
}
@Test
public void test_50_50_withMagicTestSalt() {
// Now change the spec version and reevaluate
testDefinition.setSalt("&" + testName);
final StandardTestChooser chooser = newChooser();
exerciseChooser(chooser);
// uncomment this if you need to recompute these values
// for (int i = 0; i < counts.length; i++) System.err.println(i + ": " + counts[i] + " /
// " + hashes[i]);
// if this ever fails, it means that something is broken about how tests are split
// and you should investigate why!
Assert.assertEquals("bucket0 counts", COUNTS_BUCKET0_SALT_AMP_TESTNAME, counts[0]);
Assert.assertEquals("bucket1 counts", COUNTS_BUCKET1_SALT_AMP_TESTNAME, counts[1]);
Assert.assertEquals("bucket0 hash", HASH_BUCKET0_SALT_AMP_TESTNAME, hashes[0]);
Assert.assertEquals("bucket1 hash", HASH_BUCKET1_SALT_AMP_TESTNAME, hashes[1]);
}
@Test
public void test50_50_withMagicTestSalt_and_unrelatedTestName() {
final String originalTestName = testName;
testName = "someOtherTestName";
testDefinition.setSalt("&" + originalTestName);
final StandardTestChooser chooser = newChooser();
exerciseChooser(chooser);
// uncomment this if you need to recompute these values
// for (int i = 0; i < counts.length; i++) System.err.println(i + ": " + counts[i] + " /
// " + hashes[i]);
// if this ever fails, it means that something is broken about how tests are split
// and you should investigate why!
// These values should be the same as in the preceding test
Assert.assertEquals("bucket0 counts", COUNTS_BUCKET0_SALT_AMP_TESTNAME, counts[0]);
Assert.assertEquals("bucket1 counts", COUNTS_BUCKET1_SALT_AMP_TESTNAME, counts[1]);
Assert.assertEquals("bucket0 hash", HASH_BUCKET0_SALT_AMP_TESTNAME, hashes[0]);
Assert.assertEquals("bucket1 hash", HASH_BUCKET1_SALT_AMP_TESTNAME, hashes[1]);
}
@Test
public void testExceptionsDealtWith() {
final String testName = "test";
final ConsumableTestDefinition testDefinition = new ConsumableTestDefinition();
testDefinition.setConstants(Collections.<String, Object>emptyMap());
testDefinition.setRule("${lang == 'en'}");
testDefinition.setTestType(TestType.ANONYMOUS_USER);
// most tests just set the salt to be the same as the test name
testDefinition.setSalt(testName);
testDefinition.setBuckets(Collections.<TestBucket>emptyList());
final RuleEvaluator ruleEvaluator = EasyMock.createMock(RuleEvaluator.class);
EasyMock.expect(
ruleEvaluator.evaluateBooleanRule(
EasyMock.<String>anyObject(), EasyMock.<Map<String, Object>>anyObject()))
// throw an unexpected type of runtime exception
.andThrow(new RuntimeException() {})
// Must be evaluated, or this was not a valid test
.once();
EasyMock.replay(ruleEvaluator);
final TestRangeSelector selector =
new TestRangeSelector(ruleEvaluator, testName, testDefinition);
// Ensure no exceptions thrown.
final TestBucket bucket =
new StandardTestChooser(selector)
.choose("identifier", Collections.<String, Object>emptyMap());
assertEquals("Expected no bucket to be found ", null, bucket);
EasyMock.verify(ruleEvaluator);
}
private void updateAllocations(final ImmutableList<Range> ranges) {
final List<Allocation> allocations = Lists.newArrayList();
allocations.add(new Allocation("${}", ranges));
testDefinition.setAllocations(allocations);
}
public static void verifyInternallyConsistentDefinition(
final String testName,
final String matrixSource,
@Nonnull final ConsumableTestDefinition testDefinition)
throws IncompatibleTestMatrixException {
final List<Allocation> allocations = testDefinition.getAllocations();
if (allocations.isEmpty()) {
throw new IncompatibleTestMatrixException("No allocations specified in test " + testName);
}
final List<TestBucket> buckets = testDefinition.getBuckets();
/*
* test the matrix for consistency with itself
*/
final Set<Integer> definedBuckets = Sets.newHashSet();
for (final TestBucket bucket : buckets) {
definedBuckets.add(bucket.getValue());
}
for (final Allocation allocation : allocations) {
final List<Range> ranges = allocation.getRanges();
// ensure that each range refers to a known bucket
double bucketTotal = 0;
for (final Range range : ranges) {
bucketTotal += range.getLength();
// Internally consistent (within matrix itself)
if (!definedBuckets.contains(range.getBucketValue())) {
throw new IncompatibleTestMatrixException(
"Allocation range in "
+ testName
+ " from "
+ matrixSource
+ " refers to unknown bucket value "
+ range.getBucketValue());
}
}
// I hate floating points. TODO: extract a required precision constant/parameter?
if (bucketTotal < 0.9999
|| bucketTotal
> 1.0001) { // compensate for FP imprecision. TODO: determine what these bounds
// really should be by testing stuff
final StringBuilder sb =
new StringBuilder(
testName
+ " range with rule "
+ allocation.getRule()
+ " does not add up to 1 : ")
.append(ranges.get(0).getLength());
for (int i = 1; i < ranges.size(); i++) {
sb.append(" + ").append(ranges.get(i).getLength());
}
sb.append(" = ").append(bucketTotal);
throw new IncompatibleTestMatrixException(sb.toString());
}
}
final Allocation lastAllocation = allocations.get(allocations.size() - 1);
if (!CharMatcher.WHITESPACE.matchesAllOf(Strings.nullToEmpty(lastAllocation.getRule()))) {
throw new IncompatibleTestMatrixException(
"Final allocation for test "
+ testName
+ " from "
+ matrixSource
+ " has non-empty rule: "
+ lastAllocation.getRule());
}
/*
* When defined, within a single test, all test bucket payloads
* should be supplied; they should all have just one type each,
* and they should all be the same type.
*/
Payload nonEmptyPayload = null;
final List<TestBucket> bucketsWithoutPayloads = Lists.newArrayList();
for (final TestBucket bucket : buckets) {
final Payload p = bucket.getPayload();
if (p != null) {
if (p.numFieldsDefined() != 1) {
throw new IncompatibleTestMatrixException(
"Test "
+ testName
+ " from "
+ matrixSource
+ " has a test bucket payload with multiple types: "
+ bucket);
}
if (nonEmptyPayload == null) {
nonEmptyPayload = p;
} else if (!nonEmptyPayload.sameType(p)) {
throw new IncompatibleTestMatrixException(
"Test "
+ testName
+ " from "
+ matrixSource
+ " has test bucket: "
+ bucket
+ " incompatible with type of payload: "
+ nonEmptyPayload);
}
} else {
bucketsWithoutPayloads.add(bucket);
}
}
if ((nonEmptyPayload != null) && (bucketsWithoutPayloads.size() != 0)) {
throw new IncompatibleTestMatrixException(
"Test "
+ testName
+ " from "
+ matrixSource
+ " has some test buckets without payloads: "
+ bucketsWithoutPayloads);
}
}
private static void verifyTest(
String testName,
@Nonnull ConsumableTestDefinition testDefinition,
TestSpecification testSpecification,
@Nonnull Map<Integer, String> knownBuckets,
String matrixSource,
FunctionMapper functionMapper)
throws IncompatibleTestMatrixException {
final List<Allocation> allocations = testDefinition.getAllocations();
verifyInternallyConsistentDefinition(testName, matrixSource, testDefinition);
/*
* test the matrix for adherence to this application's requirements
*/
final Set<Integer> unknownBuckets = Sets.newHashSet();
for (final Allocation allocation : allocations) {
final List<Range> ranges = allocation.getRanges();
// ensure that each range refers to a known bucket
for (final Range range : ranges) {
// Externally consistent (application's requirements)
if (!knownBuckets.containsKey(range.getBucketValue())) {
// If the bucket has a positive allocation, add it to the list of unknownBuckets
if (range.getLength() > 0) {
unknownBuckets.add(range.getBucketValue());
}
}
}
}
if (unknownBuckets.size() > 0) {
throw new IncompatibleTestMatrixException(
"Allocation range in "
+ testName
+ " from "
+ matrixSource
+ " refers to unknown bucket value(s) "
+ unknownBuckets
+ " with length > 0");
}
// TODO(pwp): add some test constants?
final RuleEvaluator ruleEvaluator =
makeRuleEvaluator(RuleEvaluator.EXPRESSION_FACTORY, functionMapper);
PayloadSpecification payloadSpec = testSpecification.getPayload();
if (payloadSpec != null) {
final String specifiedPayloadTypeName =
Preconditions.checkNotNull(payloadSpec.getType(), "Missing payload spec type");
final PayloadType specifiedPayloadType =
PayloadType.payloadTypeForName(specifiedPayloadTypeName);
if (specifiedPayloadType == null) {
// This is probably redundant vs. TestGroupsGenerator.
throw new IncompatibleTestMatrixException(
"For test "
+ testName
+ " from "
+ matrixSource
+ " test specification payload type unknown: "
+ specifiedPayloadTypeName);
}
final String payloadValidatorRule = payloadSpec.getValidator();
final List<TestBucket> buckets = testDefinition.getBuckets();
for (final TestBucket bucket : buckets) {
Payload payload = bucket.getPayload();
if (payload != null) {
if (!specifiedPayloadType.payloadHasThisType(payload)) {
throw new IncompatibleTestMatrixException(
"For test "
+ testName
+ " from "
+ matrixSource
+ " expected payload of type "
+ specifiedPayloadType.payloadTypeName
+ " but matrix has a test bucket payload with wrong type: "
+ bucket);
}
if (payloadValidatorRule != null) {
final boolean payloadIsValid =
evaluatePayloadValidator(ruleEvaluator, payloadValidatorRule, payload);
if (!payloadIsValid) {
throw new IncompatibleTestMatrixException(
"For test "
+ testName
+ " from "
+ matrixSource
+ " payload validation rule "
+ payloadValidatorRule
+ " failed for test bucket: "
+ bucket);
}
}
}
}
}
}