コード例 #1
0
  private static void consolidate(
      @Nonnull final TestMatrixArtifact testMatrix,
      @Nonnull final Map<String, TestSpecification> requiredTests) {
    final Map<String, ConsumableTestDefinition> definedTests = testMatrix.getTests();

    // Sets.difference returns a "view" on the original set, which would require concurrent
    // modification while iterating (copying the set will prevent this)
    final Set<String> toRemove =
        ImmutableSet.copyOf(Sets.difference(definedTests.keySet(), requiredTests.keySet()));
    for (String testInMatrixNotRequired : toRemove) {
      //  we don't care about this test
      definedTests.remove(testInMatrixNotRequired);
    }

    // Next, for any required tests that are missing, ensure that
    //  there is a nonnull test definition in the matrix
    final Set<String> missing =
        ImmutableSet.copyOf(Sets.difference(requiredTests.keySet(), definedTests.keySet()));
    for (String testNotInMatrix : missing) {
      definedTests.put(
          testNotInMatrix, defaultFor(testNotInMatrix, requiredTests.get(testNotInMatrix)));
    }

    // Now go through definedTests: for each test, if the test spec
    // didn't ask for a payload, then remove any payload that is in
    // the test matrix.
    for (Entry<String, ConsumableTestDefinition> next : definedTests.entrySet()) {
      final String testName = next.getKey();
      final ConsumableTestDefinition testDefinition = next.getValue();
      final TestSpecification testSpec = requiredTests.get(testName);

      if (testSpec.getPayload() == null) {
        // No payload was requested...
        final List<TestBucket> buckets = testDefinition.getBuckets();
        for (final TestBucket bucket : buckets) {
          if (bucket.getPayload() != null) {
            // ... so stomp the unexpected payloads.
            bucket.setPayload(null);
          }
        }
      }
    }
  }
コード例 #2
0
  @Before
  public void setupMocks() throws Exception {
    expressionFactory = new ExpressionFactoryImpl();
    functionMapper = RuleEvaluator.FUNCTION_MAPPER;
    testName = "testName";
    final List<TestBucket> buckets =
        ImmutableList.of(
            new TestBucket("inactive", -1, "zoot", null),
            new TestBucket("control", 0, "zoot", null),
            new TestBucket("test", 1, "zoot", null));
    testDefinition = new ConsumableTestDefinition();
    testDefinition.setConstants(Collections.<String, Object>emptyMap());
    testDefinition.setTestType(TestType.AUTHENTICATED_USER);
    // most tests just set the salt to be the same as the test name
    testDefinition.setSalt(testName);
    testDefinition.setBuckets(buckets);

    updateAllocations(RANGES_50_50);

    final int effBuckets = buckets.size() - 1;
    counts = new int[effBuckets];
    hashes = new int[effBuckets];
  }
コード例 #3
0
  @Test
  public void testSimple_50_50() {
    testDefinition.setSalt(testName);

    final StandardTestChooser chooser = newChooser();
    exerciseChooser(chooser);

    // uncomment this if you need to recompute these values
    //        for (int i = 0; i < counts.length; i++) System.err.println(i + ": " + counts[i] + " /
    // " + hashes[i]);

    // if this ever fails, it means that something is broken about how tests are split
    // and you should investigate why!

    Assert.assertEquals("bucket0 counts", 4999412, counts[0]);
    Assert.assertEquals("bucket1 counts", 5000587, counts[1]);

    Assert.assertEquals("bucket0 hash", 1863060514, hashes[0]);
    Assert.assertEquals("bucket1 hash", 765061458, hashes[1]);
  }
コード例 #4
0
  @Test
  public void test_50_50_withMagicTestSalt() {
    // Now change the spec version and reevaluate
    testDefinition.setSalt("&" + testName);

    final StandardTestChooser chooser = newChooser();
    exerciseChooser(chooser);

    // uncomment this if you need to recompute these values
    //        for (int i = 0; i < counts.length; i++) System.err.println(i + ": " + counts[i] + " /
    // " + hashes[i]);

    // if this ever fails, it means that something is broken about how tests are split
    // and you should investigate why!

    Assert.assertEquals("bucket0 counts", COUNTS_BUCKET0_SALT_AMP_TESTNAME, counts[0]);
    Assert.assertEquals("bucket1 counts", COUNTS_BUCKET1_SALT_AMP_TESTNAME, counts[1]);

    Assert.assertEquals("bucket0 hash", HASH_BUCKET0_SALT_AMP_TESTNAME, hashes[0]);
    Assert.assertEquals("bucket1 hash", HASH_BUCKET1_SALT_AMP_TESTNAME, hashes[1]);
  }
コード例 #5
0
  @Test
  public void test50_50_withMagicTestSalt_and_unrelatedTestName() {
    final String originalTestName = testName;
    testName = "someOtherTestName";
    testDefinition.setSalt("&" + originalTestName);

    final StandardTestChooser chooser = newChooser();
    exerciseChooser(chooser);

    // uncomment this if you need to recompute these values
    //        for (int i = 0; i < counts.length; i++) System.err.println(i + ": " + counts[i] + " /
    // " + hashes[i]);

    // if this ever fails, it means that something is broken about how tests are split
    // and you should investigate why!

    // These values should be the same as in the preceding test
    Assert.assertEquals("bucket0 counts", COUNTS_BUCKET0_SALT_AMP_TESTNAME, counts[0]);
    Assert.assertEquals("bucket1 counts", COUNTS_BUCKET1_SALT_AMP_TESTNAME, counts[1]);

    Assert.assertEquals("bucket0 hash", HASH_BUCKET0_SALT_AMP_TESTNAME, hashes[0]);
    Assert.assertEquals("bucket1 hash", HASH_BUCKET1_SALT_AMP_TESTNAME, hashes[1]);
  }
コード例 #6
0
  @Test
  public void testExceptionsDealtWith() {
    final String testName = "test";

    final ConsumableTestDefinition testDefinition = new ConsumableTestDefinition();
    testDefinition.setConstants(Collections.<String, Object>emptyMap());
    testDefinition.setRule("${lang == 'en'}");

    testDefinition.setTestType(TestType.ANONYMOUS_USER);

    // most tests just set the salt to be the same as the test name
    testDefinition.setSalt(testName);
    testDefinition.setBuckets(Collections.<TestBucket>emptyList());

    final RuleEvaluator ruleEvaluator = EasyMock.createMock(RuleEvaluator.class);
    EasyMock.expect(
            ruleEvaluator.evaluateBooleanRule(
                EasyMock.<String>anyObject(), EasyMock.<Map<String, Object>>anyObject()))
        // throw an unexpected type of runtime exception
        .andThrow(new RuntimeException() {})
        // Must be evaluated, or this was not a valid test
        .once();
    EasyMock.replay(ruleEvaluator);

    final TestRangeSelector selector =
        new TestRangeSelector(ruleEvaluator, testName, testDefinition);

    // Ensure no exceptions thrown.
    final TestBucket bucket =
        new StandardTestChooser(selector)
            .choose("identifier", Collections.<String, Object>emptyMap());

    assertEquals("Expected no bucket to be found ", null, bucket);

    EasyMock.verify(ruleEvaluator);
  }
コード例 #7
0
 private void updateAllocations(final ImmutableList<Range> ranges) {
   final List<Allocation> allocations = Lists.newArrayList();
   allocations.add(new Allocation("${}", ranges));
   testDefinition.setAllocations(allocations);
 }
コード例 #8
0
  public static void verifyInternallyConsistentDefinition(
      final String testName,
      final String matrixSource,
      @Nonnull final ConsumableTestDefinition testDefinition)
      throws IncompatibleTestMatrixException {
    final List<Allocation> allocations = testDefinition.getAllocations();
    if (allocations.isEmpty()) {
      throw new IncompatibleTestMatrixException("No allocations specified in test " + testName);
    }
    final List<TestBucket> buckets = testDefinition.getBuckets();

    /*
     * test the matrix for consistency with itself
     */
    final Set<Integer> definedBuckets = Sets.newHashSet();
    for (final TestBucket bucket : buckets) {
      definedBuckets.add(bucket.getValue());
    }

    for (final Allocation allocation : allocations) {
      final List<Range> ranges = allocation.getRanges();
      //  ensure that each range refers to a known bucket
      double bucketTotal = 0;
      for (final Range range : ranges) {
        bucketTotal += range.getLength();
        // Internally consistent (within matrix itself)
        if (!definedBuckets.contains(range.getBucketValue())) {
          throw new IncompatibleTestMatrixException(
              "Allocation range in "
                  + testName
                  + " from "
                  + matrixSource
                  + " refers to unknown bucket value "
                  + range.getBucketValue());
        }
      }
      //  I hate floating points.  TODO: extract a required precision constant/parameter?
      if (bucketTotal < 0.9999
          || bucketTotal
              > 1.0001) { //  compensate for FP imprecision.  TODO: determine what these bounds
                          // really should be by testing stuff
        final StringBuilder sb =
            new StringBuilder(
                    testName
                        + " range with rule "
                        + allocation.getRule()
                        + " does not add up to 1 : ")
                .append(ranges.get(0).getLength());
        for (int i = 1; i < ranges.size(); i++) {
          sb.append(" + ").append(ranges.get(i).getLength());
        }
        sb.append(" = ").append(bucketTotal);
        throw new IncompatibleTestMatrixException(sb.toString());
      }
    }
    final Allocation lastAllocation = allocations.get(allocations.size() - 1);
    if (!CharMatcher.WHITESPACE.matchesAllOf(Strings.nullToEmpty(lastAllocation.getRule()))) {
      throw new IncompatibleTestMatrixException(
          "Final allocation for test "
              + testName
              + " from "
              + matrixSource
              + " has non-empty rule: "
              + lastAllocation.getRule());
    }

    /*
     * When defined, within a single test, all test bucket payloads
     * should be supplied; they should all have just one type each,
     * and they should all be the same type.
     */
    Payload nonEmptyPayload = null;
    final List<TestBucket> bucketsWithoutPayloads = Lists.newArrayList();
    for (final TestBucket bucket : buckets) {
      final Payload p = bucket.getPayload();
      if (p != null) {
        if (p.numFieldsDefined() != 1) {
          throw new IncompatibleTestMatrixException(
              "Test "
                  + testName
                  + " from "
                  + matrixSource
                  + " has a test bucket payload with multiple types: "
                  + bucket);
        }
        if (nonEmptyPayload == null) {
          nonEmptyPayload = p;
        } else if (!nonEmptyPayload.sameType(p)) {
          throw new IncompatibleTestMatrixException(
              "Test "
                  + testName
                  + " from "
                  + matrixSource
                  + " has test bucket: "
                  + bucket
                  + " incompatible with type of payload: "
                  + nonEmptyPayload);
        }
      } else {
        bucketsWithoutPayloads.add(bucket);
      }
    }
    if ((nonEmptyPayload != null) && (bucketsWithoutPayloads.size() != 0)) {
      throw new IncompatibleTestMatrixException(
          "Test "
              + testName
              + " from "
              + matrixSource
              + " has some test buckets without payloads: "
              + bucketsWithoutPayloads);
    }
  }
コード例 #9
0
  private static void verifyTest(
      String testName,
      @Nonnull ConsumableTestDefinition testDefinition,
      TestSpecification testSpecification,
      @Nonnull Map<Integer, String> knownBuckets,
      String matrixSource,
      FunctionMapper functionMapper)
      throws IncompatibleTestMatrixException {
    final List<Allocation> allocations = testDefinition.getAllocations();

    verifyInternallyConsistentDefinition(testName, matrixSource, testDefinition);
    /*
     * test the matrix for adherence to this application's requirements
     */
    final Set<Integer> unknownBuckets = Sets.newHashSet();

    for (final Allocation allocation : allocations) {
      final List<Range> ranges = allocation.getRanges();
      //  ensure that each range refers to a known bucket
      for (final Range range : ranges) {
        // Externally consistent (application's requirements)
        if (!knownBuckets.containsKey(range.getBucketValue())) {
          // If the bucket has a positive allocation, add it to the list of unknownBuckets
          if (range.getLength() > 0) {
            unknownBuckets.add(range.getBucketValue());
          }
        }
      }
    }
    if (unknownBuckets.size() > 0) {
      throw new IncompatibleTestMatrixException(
          "Allocation range in "
              + testName
              + " from "
              + matrixSource
              + " refers to unknown bucket value(s) "
              + unknownBuckets
              + " with length > 0");
    }

    // TODO(pwp): add some test constants?
    final RuleEvaluator ruleEvaluator =
        makeRuleEvaluator(RuleEvaluator.EXPRESSION_FACTORY, functionMapper);

    PayloadSpecification payloadSpec = testSpecification.getPayload();
    if (payloadSpec != null) {
      final String specifiedPayloadTypeName =
          Preconditions.checkNotNull(payloadSpec.getType(), "Missing payload spec type");
      final PayloadType specifiedPayloadType =
          PayloadType.payloadTypeForName(specifiedPayloadTypeName);
      if (specifiedPayloadType == null) {
        // This is probably redundant vs. TestGroupsGenerator.
        throw new IncompatibleTestMatrixException(
            "For test "
                + testName
                + " from "
                + matrixSource
                + " test specification payload type unknown: "
                + specifiedPayloadTypeName);
      }
      final String payloadValidatorRule = payloadSpec.getValidator();
      final List<TestBucket> buckets = testDefinition.getBuckets();
      for (final TestBucket bucket : buckets) {
        Payload payload = bucket.getPayload();
        if (payload != null) {
          if (!specifiedPayloadType.payloadHasThisType(payload)) {
            throw new IncompatibleTestMatrixException(
                "For test "
                    + testName
                    + " from "
                    + matrixSource
                    + " expected payload of type "
                    + specifiedPayloadType.payloadTypeName
                    + " but matrix has a test bucket payload with wrong type: "
                    + bucket);
          }
          if (payloadValidatorRule != null) {
            final boolean payloadIsValid =
                evaluatePayloadValidator(ruleEvaluator, payloadValidatorRule, payload);
            if (!payloadIsValid) {
              throw new IncompatibleTestMatrixException(
                  "For test "
                      + testName
                      + " from "
                      + matrixSource
                      + " payload validation rule "
                      + payloadValidatorRule
                      + " failed for test bucket: "
                      + bucket);
            }
          }
        }
      }
    }
  }