private static void consolidate( @Nonnull final TestMatrixArtifact testMatrix, @Nonnull final Map<String, TestSpecification> requiredTests) { final Map<String, ConsumableTestDefinition> definedTests = testMatrix.getTests(); // Sets.difference returns a "view" on the original set, which would require concurrent // modification while iterating (copying the set will prevent this) final Set<String> toRemove = ImmutableSet.copyOf(Sets.difference(definedTests.keySet(), requiredTests.keySet())); for (String testInMatrixNotRequired : toRemove) { // we don't care about this test definedTests.remove(testInMatrixNotRequired); } // Next, for any required tests that are missing, ensure that // there is a nonnull test definition in the matrix final Set<String> missing = ImmutableSet.copyOf(Sets.difference(requiredTests.keySet(), definedTests.keySet())); for (String testNotInMatrix : missing) { definedTests.put( testNotInMatrix, defaultFor(testNotInMatrix, requiredTests.get(testNotInMatrix))); } // Now go through definedTests: for each test, if the test spec // didn't ask for a payload, then remove any payload that is in // the test matrix. for (Entry<String, ConsumableTestDefinition> next : definedTests.entrySet()) { final String testName = next.getKey(); final ConsumableTestDefinition testDefinition = next.getValue(); final TestSpecification testSpec = requiredTests.get(testName); if (testSpec.getPayload() == null) { // No payload was requested... final List<TestBucket> buckets = testDefinition.getBuckets(); for (final TestBucket bucket : buckets) { if (bucket.getPayload() != null) { // ... so stomp the unexpected payloads. bucket.setPayload(null); } } } } }
@Before public void setupMocks() throws Exception { expressionFactory = new ExpressionFactoryImpl(); functionMapper = RuleEvaluator.FUNCTION_MAPPER; testName = "testName"; final List<TestBucket> buckets = ImmutableList.of( new TestBucket("inactive", -1, "zoot", null), new TestBucket("control", 0, "zoot", null), new TestBucket("test", 1, "zoot", null)); testDefinition = new ConsumableTestDefinition(); testDefinition.setConstants(Collections.<String, Object>emptyMap()); testDefinition.setTestType(TestType.AUTHENTICATED_USER); // most tests just set the salt to be the same as the test name testDefinition.setSalt(testName); testDefinition.setBuckets(buckets); updateAllocations(RANGES_50_50); final int effBuckets = buckets.size() - 1; counts = new int[effBuckets]; hashes = new int[effBuckets]; }
@Test public void testSimple_50_50() { testDefinition.setSalt(testName); final StandardTestChooser chooser = newChooser(); exerciseChooser(chooser); // uncomment this if you need to recompute these values // for (int i = 0; i < counts.length; i++) System.err.println(i + ": " + counts[i] + " / // " + hashes[i]); // if this ever fails, it means that something is broken about how tests are split // and you should investigate why! Assert.assertEquals("bucket0 counts", 4999412, counts[0]); Assert.assertEquals("bucket1 counts", 5000587, counts[1]); Assert.assertEquals("bucket0 hash", 1863060514, hashes[0]); Assert.assertEquals("bucket1 hash", 765061458, hashes[1]); }
@Test public void test_50_50_withMagicTestSalt() { // Now change the spec version and reevaluate testDefinition.setSalt("&" + testName); final StandardTestChooser chooser = newChooser(); exerciseChooser(chooser); // uncomment this if you need to recompute these values // for (int i = 0; i < counts.length; i++) System.err.println(i + ": " + counts[i] + " / // " + hashes[i]); // if this ever fails, it means that something is broken about how tests are split // and you should investigate why! Assert.assertEquals("bucket0 counts", COUNTS_BUCKET0_SALT_AMP_TESTNAME, counts[0]); Assert.assertEquals("bucket1 counts", COUNTS_BUCKET1_SALT_AMP_TESTNAME, counts[1]); Assert.assertEquals("bucket0 hash", HASH_BUCKET0_SALT_AMP_TESTNAME, hashes[0]); Assert.assertEquals("bucket1 hash", HASH_BUCKET1_SALT_AMP_TESTNAME, hashes[1]); }
@Test public void test50_50_withMagicTestSalt_and_unrelatedTestName() { final String originalTestName = testName; testName = "someOtherTestName"; testDefinition.setSalt("&" + originalTestName); final StandardTestChooser chooser = newChooser(); exerciseChooser(chooser); // uncomment this if you need to recompute these values // for (int i = 0; i < counts.length; i++) System.err.println(i + ": " + counts[i] + " / // " + hashes[i]); // if this ever fails, it means that something is broken about how tests are split // and you should investigate why! // These values should be the same as in the preceding test Assert.assertEquals("bucket0 counts", COUNTS_BUCKET0_SALT_AMP_TESTNAME, counts[0]); Assert.assertEquals("bucket1 counts", COUNTS_BUCKET1_SALT_AMP_TESTNAME, counts[1]); Assert.assertEquals("bucket0 hash", HASH_BUCKET0_SALT_AMP_TESTNAME, hashes[0]); Assert.assertEquals("bucket1 hash", HASH_BUCKET1_SALT_AMP_TESTNAME, hashes[1]); }
@Test public void testExceptionsDealtWith() { final String testName = "test"; final ConsumableTestDefinition testDefinition = new ConsumableTestDefinition(); testDefinition.setConstants(Collections.<String, Object>emptyMap()); testDefinition.setRule("${lang == 'en'}"); testDefinition.setTestType(TestType.ANONYMOUS_USER); // most tests just set the salt to be the same as the test name testDefinition.setSalt(testName); testDefinition.setBuckets(Collections.<TestBucket>emptyList()); final RuleEvaluator ruleEvaluator = EasyMock.createMock(RuleEvaluator.class); EasyMock.expect( ruleEvaluator.evaluateBooleanRule( EasyMock.<String>anyObject(), EasyMock.<Map<String, Object>>anyObject())) // throw an unexpected type of runtime exception .andThrow(new RuntimeException() {}) // Must be evaluated, or this was not a valid test .once(); EasyMock.replay(ruleEvaluator); final TestRangeSelector selector = new TestRangeSelector(ruleEvaluator, testName, testDefinition); // Ensure no exceptions thrown. final TestBucket bucket = new StandardTestChooser(selector) .choose("identifier", Collections.<String, Object>emptyMap()); assertEquals("Expected no bucket to be found ", null, bucket); EasyMock.verify(ruleEvaluator); }
private void updateAllocations(final ImmutableList<Range> ranges) { final List<Allocation> allocations = Lists.newArrayList(); allocations.add(new Allocation("${}", ranges)); testDefinition.setAllocations(allocations); }
public static void verifyInternallyConsistentDefinition( final String testName, final String matrixSource, @Nonnull final ConsumableTestDefinition testDefinition) throws IncompatibleTestMatrixException { final List<Allocation> allocations = testDefinition.getAllocations(); if (allocations.isEmpty()) { throw new IncompatibleTestMatrixException("No allocations specified in test " + testName); } final List<TestBucket> buckets = testDefinition.getBuckets(); /* * test the matrix for consistency with itself */ final Set<Integer> definedBuckets = Sets.newHashSet(); for (final TestBucket bucket : buckets) { definedBuckets.add(bucket.getValue()); } for (final Allocation allocation : allocations) { final List<Range> ranges = allocation.getRanges(); // ensure that each range refers to a known bucket double bucketTotal = 0; for (final Range range : ranges) { bucketTotal += range.getLength(); // Internally consistent (within matrix itself) if (!definedBuckets.contains(range.getBucketValue())) { throw new IncompatibleTestMatrixException( "Allocation range in " + testName + " from " + matrixSource + " refers to unknown bucket value " + range.getBucketValue()); } } // I hate floating points. TODO: extract a required precision constant/parameter? if (bucketTotal < 0.9999 || bucketTotal > 1.0001) { // compensate for FP imprecision. TODO: determine what these bounds // really should be by testing stuff final StringBuilder sb = new StringBuilder( testName + " range with rule " + allocation.getRule() + " does not add up to 1 : ") .append(ranges.get(0).getLength()); for (int i = 1; i < ranges.size(); i++) { sb.append(" + ").append(ranges.get(i).getLength()); } sb.append(" = ").append(bucketTotal); throw new IncompatibleTestMatrixException(sb.toString()); } } final Allocation lastAllocation = allocations.get(allocations.size() - 1); if (!CharMatcher.WHITESPACE.matchesAllOf(Strings.nullToEmpty(lastAllocation.getRule()))) { throw new IncompatibleTestMatrixException( "Final allocation for test " + testName + " from " + matrixSource + " has non-empty rule: " + lastAllocation.getRule()); } /* * When defined, within a single test, all test bucket payloads * should be supplied; they should all have just one type each, * and they should all be the same type. */ Payload nonEmptyPayload = null; final List<TestBucket> bucketsWithoutPayloads = Lists.newArrayList(); for (final TestBucket bucket : buckets) { final Payload p = bucket.getPayload(); if (p != null) { if (p.numFieldsDefined() != 1) { throw new IncompatibleTestMatrixException( "Test " + testName + " from " + matrixSource + " has a test bucket payload with multiple types: " + bucket); } if (nonEmptyPayload == null) { nonEmptyPayload = p; } else if (!nonEmptyPayload.sameType(p)) { throw new IncompatibleTestMatrixException( "Test " + testName + " from " + matrixSource + " has test bucket: " + bucket + " incompatible with type of payload: " + nonEmptyPayload); } } else { bucketsWithoutPayloads.add(bucket); } } if ((nonEmptyPayload != null) && (bucketsWithoutPayloads.size() != 0)) { throw new IncompatibleTestMatrixException( "Test " + testName + " from " + matrixSource + " has some test buckets without payloads: " + bucketsWithoutPayloads); } }
private static void verifyTest( String testName, @Nonnull ConsumableTestDefinition testDefinition, TestSpecification testSpecification, @Nonnull Map<Integer, String> knownBuckets, String matrixSource, FunctionMapper functionMapper) throws IncompatibleTestMatrixException { final List<Allocation> allocations = testDefinition.getAllocations(); verifyInternallyConsistentDefinition(testName, matrixSource, testDefinition); /* * test the matrix for adherence to this application's requirements */ final Set<Integer> unknownBuckets = Sets.newHashSet(); for (final Allocation allocation : allocations) { final List<Range> ranges = allocation.getRanges(); // ensure that each range refers to a known bucket for (final Range range : ranges) { // Externally consistent (application's requirements) if (!knownBuckets.containsKey(range.getBucketValue())) { // If the bucket has a positive allocation, add it to the list of unknownBuckets if (range.getLength() > 0) { unknownBuckets.add(range.getBucketValue()); } } } } if (unknownBuckets.size() > 0) { throw new IncompatibleTestMatrixException( "Allocation range in " + testName + " from " + matrixSource + " refers to unknown bucket value(s) " + unknownBuckets + " with length > 0"); } // TODO(pwp): add some test constants? final RuleEvaluator ruleEvaluator = makeRuleEvaluator(RuleEvaluator.EXPRESSION_FACTORY, functionMapper); PayloadSpecification payloadSpec = testSpecification.getPayload(); if (payloadSpec != null) { final String specifiedPayloadTypeName = Preconditions.checkNotNull(payloadSpec.getType(), "Missing payload spec type"); final PayloadType specifiedPayloadType = PayloadType.payloadTypeForName(specifiedPayloadTypeName); if (specifiedPayloadType == null) { // This is probably redundant vs. TestGroupsGenerator. throw new IncompatibleTestMatrixException( "For test " + testName + " from " + matrixSource + " test specification payload type unknown: " + specifiedPayloadTypeName); } final String payloadValidatorRule = payloadSpec.getValidator(); final List<TestBucket> buckets = testDefinition.getBuckets(); for (final TestBucket bucket : buckets) { Payload payload = bucket.getPayload(); if (payload != null) { if (!specifiedPayloadType.payloadHasThisType(payload)) { throw new IncompatibleTestMatrixException( "For test " + testName + " from " + matrixSource + " expected payload of type " + specifiedPayloadType.payloadTypeName + " but matrix has a test bucket payload with wrong type: " + bucket); } if (payloadValidatorRule != null) { final boolean payloadIsValid = evaluatePayloadValidator(ruleEvaluator, payloadValidatorRule, payload); if (!payloadIsValid) { throw new IncompatibleTestMatrixException( "For test " + testName + " from " + matrixSource + " payload validation rule " + payloadValidatorRule + " failed for test bucket: " + bucket); } } } } } }