Esempio n. 1
0
  public static void verifyInternallyConsistentDefinition(
      final String testName,
      final String matrixSource,
      @Nonnull final ConsumableTestDefinition testDefinition)
      throws IncompatibleTestMatrixException {
    final List<Allocation> allocations = testDefinition.getAllocations();
    if (allocations.isEmpty()) {
      throw new IncompatibleTestMatrixException("No allocations specified in test " + testName);
    }
    final List<TestBucket> buckets = testDefinition.getBuckets();

    /*
     * test the matrix for consistency with itself
     */
    final Set<Integer> definedBuckets = Sets.newHashSet();
    for (final TestBucket bucket : buckets) {
      definedBuckets.add(bucket.getValue());
    }

    for (final Allocation allocation : allocations) {
      final List<Range> ranges = allocation.getRanges();
      //  ensure that each range refers to a known bucket
      double bucketTotal = 0;
      for (final Range range : ranges) {
        bucketTotal += range.getLength();
        // Internally consistent (within matrix itself)
        if (!definedBuckets.contains(range.getBucketValue())) {
          throw new IncompatibleTestMatrixException(
              "Allocation range in "
                  + testName
                  + " from "
                  + matrixSource
                  + " refers to unknown bucket value "
                  + range.getBucketValue());
        }
      }
      //  I hate floating points.  TODO: extract a required precision constant/parameter?
      if (bucketTotal < 0.9999
          || bucketTotal
              > 1.0001) { //  compensate for FP imprecision.  TODO: determine what these bounds
                          // really should be by testing stuff
        final StringBuilder sb =
            new StringBuilder(
                    testName
                        + " range with rule "
                        + allocation.getRule()
                        + " does not add up to 1 : ")
                .append(ranges.get(0).getLength());
        for (int i = 1; i < ranges.size(); i++) {
          sb.append(" + ").append(ranges.get(i).getLength());
        }
        sb.append(" = ").append(bucketTotal);
        throw new IncompatibleTestMatrixException(sb.toString());
      }
    }
    final Allocation lastAllocation = allocations.get(allocations.size() - 1);
    if (!CharMatcher.WHITESPACE.matchesAllOf(Strings.nullToEmpty(lastAllocation.getRule()))) {
      throw new IncompatibleTestMatrixException(
          "Final allocation for test "
              + testName
              + " from "
              + matrixSource
              + " has non-empty rule: "
              + lastAllocation.getRule());
    }

    /*
     * When defined, within a single test, all test bucket payloads
     * should be supplied; they should all have just one type each,
     * and they should all be the same type.
     */
    Payload nonEmptyPayload = null;
    final List<TestBucket> bucketsWithoutPayloads = Lists.newArrayList();
    for (final TestBucket bucket : buckets) {
      final Payload p = bucket.getPayload();
      if (p != null) {
        if (p.numFieldsDefined() != 1) {
          throw new IncompatibleTestMatrixException(
              "Test "
                  + testName
                  + " from "
                  + matrixSource
                  + " has a test bucket payload with multiple types: "
                  + bucket);
        }
        if (nonEmptyPayload == null) {
          nonEmptyPayload = p;
        } else if (!nonEmptyPayload.sameType(p)) {
          throw new IncompatibleTestMatrixException(
              "Test "
                  + testName
                  + " from "
                  + matrixSource
                  + " has test bucket: "
                  + bucket
                  + " incompatible with type of payload: "
                  + nonEmptyPayload);
        }
      } else {
        bucketsWithoutPayloads.add(bucket);
      }
    }
    if ((nonEmptyPayload != null) && (bucketsWithoutPayloads.size() != 0)) {
      throw new IncompatibleTestMatrixException(
          "Test "
              + testName
              + " from "
              + matrixSource
              + " has some test buckets without payloads: "
              + bucketsWithoutPayloads);
    }
  }
Esempio n. 2
0
  private static void verifyTest(
      String testName,
      @Nonnull ConsumableTestDefinition testDefinition,
      TestSpecification testSpecification,
      @Nonnull Map<Integer, String> knownBuckets,
      String matrixSource,
      FunctionMapper functionMapper)
      throws IncompatibleTestMatrixException {
    final List<Allocation> allocations = testDefinition.getAllocations();

    verifyInternallyConsistentDefinition(testName, matrixSource, testDefinition);
    /*
     * test the matrix for adherence to this application's requirements
     */
    final Set<Integer> unknownBuckets = Sets.newHashSet();

    for (final Allocation allocation : allocations) {
      final List<Range> ranges = allocation.getRanges();
      //  ensure that each range refers to a known bucket
      for (final Range range : ranges) {
        // Externally consistent (application's requirements)
        if (!knownBuckets.containsKey(range.getBucketValue())) {
          // If the bucket has a positive allocation, add it to the list of unknownBuckets
          if (range.getLength() > 0) {
            unknownBuckets.add(range.getBucketValue());
          }
        }
      }
    }
    if (unknownBuckets.size() > 0) {
      throw new IncompatibleTestMatrixException(
          "Allocation range in "
              + testName
              + " from "
              + matrixSource
              + " refers to unknown bucket value(s) "
              + unknownBuckets
              + " with length > 0");
    }

    // TODO(pwp): add some test constants?
    final RuleEvaluator ruleEvaluator =
        makeRuleEvaluator(RuleEvaluator.EXPRESSION_FACTORY, functionMapper);

    PayloadSpecification payloadSpec = testSpecification.getPayload();
    if (payloadSpec != null) {
      final String specifiedPayloadTypeName =
          Preconditions.checkNotNull(payloadSpec.getType(), "Missing payload spec type");
      final PayloadType specifiedPayloadType =
          PayloadType.payloadTypeForName(specifiedPayloadTypeName);
      if (specifiedPayloadType == null) {
        // This is probably redundant vs. TestGroupsGenerator.
        throw new IncompatibleTestMatrixException(
            "For test "
                + testName
                + " from "
                + matrixSource
                + " test specification payload type unknown: "
                + specifiedPayloadTypeName);
      }
      final String payloadValidatorRule = payloadSpec.getValidator();
      final List<TestBucket> buckets = testDefinition.getBuckets();
      for (final TestBucket bucket : buckets) {
        Payload payload = bucket.getPayload();
        if (payload != null) {
          if (!specifiedPayloadType.payloadHasThisType(payload)) {
            throw new IncompatibleTestMatrixException(
                "For test "
                    + testName
                    + " from "
                    + matrixSource
                    + " expected payload of type "
                    + specifiedPayloadType.payloadTypeName
                    + " but matrix has a test bucket payload with wrong type: "
                    + bucket);
          }
          if (payloadValidatorRule != null) {
            final boolean payloadIsValid =
                evaluatePayloadValidator(ruleEvaluator, payloadValidatorRule, payload);
            if (!payloadIsValid) {
              throw new IncompatibleTestMatrixException(
                  "For test "
                      + testName
                      + " from "
                      + matrixSource
                      + " payload validation rule "
                      + payloadValidatorRule
                      + " failed for test bucket: "
                      + bucket);
            }
          }
        }
      }
    }
  }