@SuppressWarnings("unchecked")
protected Serializer<Object> getSerializerFor(Class<?> clazz) {
Serializer<Object> s = (Serializer<Object>) serializers.get(clazz);
if (s != null) {
return s;
}
return (Serializer<Object>) Serializers.serializerFor(clazz);
}
public abstract class ProctorUtils {
private static final ObjectMapper OBJECT_MAPPER = Serializers.lenient();
private static final Logger LOGGER = Logger.getLogger(ProctorUtils.class);
public static MessageDigest createMessageDigest() {
try {
return MessageDigest.getInstance("MD5");
} catch (@Nonnull final NoSuchAlgorithmException e) {
throw new RuntimeException("Impossible no MD5", e);
}
}
@Nonnull
public static Map<String, ValueExpression> convertToValueExpressionMap(
@Nonnull final ExpressionFactory expressionFactory,
@Nonnull final Map<String, Object> values) {
final Map<String, ValueExpression> context =
new HashMap<String, ValueExpression>(values.size());
for (final Entry<String, Object> entry : values.entrySet()) {
final ValueExpression ve =
expressionFactory.createValueExpression(entry.getValue(), Object.class);
context.put(entry.getKey(), ve);
}
return context;
}
@SuppressWarnings("UnusedDeclaration") // TODO Remove?
public static String convertToArtifact(@Nonnull final TestMatrixVersion testMatrix)
throws IOException {
final StringWriter sw = new StringWriter();
final TestMatrixArtifact artifact = convertToConsumableArtifact(testMatrix);
serializeArtifact(sw, artifact);
return sw.toString();
}
public static void serializeArtifact(Writer writer, final TestMatrixArtifact artifact)
throws IOException {
serializeObject(writer, artifact);
}
@SuppressWarnings("UnusedDeclaration") // TODO Remove?
public static void serializeTestDefinition(Writer writer, final TestDefinition definition)
throws IOException {
serializeObject(writer, definition);
}
private static <T> void serializeObject(Writer writer, final T artifact) throws IOException {
OBJECT_MAPPER.defaultPrettyPrintingWriter().writeValue(writer, artifact);
}
@Nonnull
public static TestMatrixArtifact convertToConsumableArtifact(
@Nonnull final TestMatrixVersion testMatrix) {
final Audit audit = new Audit();
final Date published =
Preconditions.checkNotNull(testMatrix.getPublished(), "Missing publication date");
audit.setUpdated(published.getTime());
audit.setVersion(testMatrix.getVersion());
audit.setUpdatedBy(testMatrix.getAuthor());
final TestMatrixArtifact artifact = new TestMatrixArtifact();
artifact.setAudit(audit);
final TestMatrixDefinition testMatrixDefinition =
Preconditions.checkNotNull(
testMatrix.getTestMatrixDefinition(), "Missing test matrix definition");
final Map<String, TestDefinition> testDefinitions = testMatrixDefinition.getTests();
final Map<String, ConsumableTestDefinition> consumableTestDefinitions = Maps.newLinkedHashMap();
for (final Entry<String, TestDefinition> entry : testDefinitions.entrySet()) {
final TestDefinition td = entry.getValue();
final ConsumableTestDefinition ctd = convertToConsumableTestDefinition(td);
consumableTestDefinitions.put(entry.getKey(), ctd);
}
artifact.setTests(consumableTestDefinitions);
return artifact;
}
@Nonnull
public static ConsumableTestDefinition convertToConsumableTestDefinition(
@Nonnull final TestDefinition td) {
final Map<String, Object> specialConstants = td.getSpecialConstants();
final List<String> ruleComponents = Lists.newArrayList();
//noinspection unchecked
final List<String> countries = (List<String>) specialConstants.get("__COUNTRIES");
if (countries != null) {
ruleComponents.add("proctor:contains(__COUNTRIES, country)");
}
if (td.getRule() != null) {
ruleComponents.add(td.getRule());
}
final String rule;
if (ruleComponents.isEmpty()) {
rule = null;
} else {
final StringBuilder ruleBuilder = new StringBuilder("${");
for (int i = 0; i < ruleComponents.size(); i++) {
if (i != 0) {
ruleBuilder.append(" && ");
}
ruleBuilder.append(ruleComponents.get(i));
}
ruleBuilder.append("}");
rule = ruleBuilder.toString();
}
final List<Allocation> allocations = td.getAllocations();
for (final Allocation alloc : allocations) {
final String allocRule = alloc.getRule();
if (allocRule != null && !(allocRule.startsWith("${") && allocRule.endsWith("}"))) {
final String newAllocRule = "${" + allocRule + "}";
alloc.setRule(newAllocRule);
}
}
final Map<String, Object> constants = Maps.newLinkedHashMap();
constants.putAll(td.getConstants());
constants.putAll(specialConstants);
return new ConsumableTestDefinition(
td.getVersion(),
rule,
td.getTestType(),
td.getSalt(),
td.getBuckets(),
allocations,
constants,
td.getDescription());
}
public static ProctorSpecification readSpecification(final File inputFile) {
final ProctorSpecification spec;
InputStream stream = null;
try {
stream = new BufferedInputStream(new FileInputStream(inputFile));
spec = readSpecification(stream);
} catch (IOException e) {
throw new RuntimeException("Unable to read test set from " + inputFile, e);
} finally {
if (stream != null) {
try {
stream.close();
} catch (final IOException e) {
LOGGER.error("Suppressing throwable thrown when closing " + inputFile, e);
}
}
}
return spec;
}
public static ProctorSpecification readSpecification(final InputStream inputFile) {
final ProctorSpecification spec;
try {
spec = OBJECT_MAPPER.readValue(inputFile, ProctorSpecification.class);
} catch (@Nonnull final JsonParseException e) {
throw new RuntimeException("Unable to read test set from " + inputFile + ": ", e);
} catch (@Nonnull final JsonMappingException e) {
throw new RuntimeException("Unable to read test set from " + inputFile, e);
} catch (@Nonnull final IOException e) {
throw new RuntimeException("Unable to read test set from " + inputFile, e);
}
return spec;
}
/**
* Verifies that the TestMatrix is compatible with all the required tests. Removes non-required
* tests from the TestMatrix Replaces invalid or missing tests (buckets are not compatible) with
* default implementation returning the fallback value see defaultFor
*
* @param testMatrix
* @param matrixSource
* @param requiredTests
* @return
*/
public static ProctorLoadResult verifyAndConsolidate(
@Nonnull final TestMatrixArtifact testMatrix,
final String matrixSource,
@Nonnull final Map<String, TestSpecification> requiredTests,
@Nonnull final FunctionMapper functionMapper) {
final ProctorLoadResult result =
verify(testMatrix, matrixSource, requiredTests, functionMapper);
final Map<String, ConsumableTestDefinition> definedTests = testMatrix.getTests();
// Remove any invalid tests so that any required ones will be replaced with default values
// during the
// consolidation below (just like missing tests). Any non-required tests can safely be ignored.
for (final String invalidTest : result.getTestsWithErrors()) {
// TODO - mjs - gross that this depends on the mutability of the returned map, but then so
// does the
// consolidate method below.
definedTests.remove(invalidTest);
}
consolidate(testMatrix, requiredTests);
return result;
}
public static ProctorLoadResult verify(
@Nonnull final TestMatrixArtifact testMatrix,
final String matrixSource,
@Nonnull final Map<String, TestSpecification> requiredTests) {
return verify(
testMatrix,
matrixSource,
requiredTests,
RuleEvaluator.FUNCTION_MAPPER); // use default function mapper
}
/**
* Does not mutate the TestMatrix.
*
* <p>Verifies that the test matrix contains all the required tests and that each required test is
* valid.
*
* @param testMatrix
* @param matrixSource
* @param requiredTests
* @return
*/
public static ProctorLoadResult verify(
@Nonnull final TestMatrixArtifact testMatrix,
final String matrixSource,
@Nonnull final Map<String, TestSpecification> requiredTests,
@Nonnull final FunctionMapper functionMapper) {
final ProctorLoadResult.Builder resultBuilder = ProctorLoadResult.newBuilder();
final Map<String, Map<Integer, String>> allTestsKnownBuckets =
Maps.newHashMapWithExpectedSize(requiredTests.size());
for (final Entry<String, TestSpecification> entry : requiredTests.entrySet()) {
final Map<Integer, String> bucketValueToName = Maps.newHashMap();
for (final Entry<String, Integer> bucket : entry.getValue().getBuckets().entrySet()) {
bucketValueToName.put(bucket.getValue(), bucket.getKey());
}
allTestsKnownBuckets.put(entry.getKey(), bucketValueToName);
}
final Map<String, ConsumableTestDefinition> definedTests = testMatrix.getTests();
final SetView<String> missingTests =
Sets.difference(requiredTests.keySet(), definedTests.keySet());
resultBuilder.recordAllMissing(missingTests);
for (final Entry<String, ConsumableTestDefinition> entry : definedTests.entrySet()) {
final String testName = entry.getKey();
final Map<Integer, String> knownBuckets = allTestsKnownBuckets.remove(testName);
if (knownBuckets == null) { // we don't care about this test
// iterator.remove(); DO NOT CONSOLIDATE
continue;
}
final ConsumableTestDefinition testDefinition = entry.getValue();
try {
verifyTest(
testName,
testDefinition,
requiredTests.get(testName),
knownBuckets,
matrixSource,
functionMapper);
} catch (IncompatibleTestMatrixException e) {
LOGGER.error(String.format("Unable to load test matrix for %s", testName), e);
resultBuilder.recordError(testName);
}
}
// TODO mjs - is this check additive?
resultBuilder.recordAllMissing(allTestsKnownBuckets.keySet());
final ProctorLoadResult loadResult = resultBuilder.build();
return loadResult;
}
private static void verifyTest(
String testName,
@Nonnull ConsumableTestDefinition testDefinition,
TestSpecification testSpecification,
@Nonnull Map<Integer, String> knownBuckets,
String matrixSource,
FunctionMapper functionMapper)
throws IncompatibleTestMatrixException {
final List<Allocation> allocations = testDefinition.getAllocations();
verifyInternallyConsistentDefinition(testName, matrixSource, testDefinition);
/*
* test the matrix for adherence to this application's requirements
*/
final Set<Integer> unknownBuckets = Sets.newHashSet();
for (final Allocation allocation : allocations) {
final List<Range> ranges = allocation.getRanges();
// ensure that each range refers to a known bucket
for (final Range range : ranges) {
// Externally consistent (application's requirements)
if (!knownBuckets.containsKey(range.getBucketValue())) {
// If the bucket has a positive allocation, add it to the list of unknownBuckets
if (range.getLength() > 0) {
unknownBuckets.add(range.getBucketValue());
}
}
}
}
if (unknownBuckets.size() > 0) {
throw new IncompatibleTestMatrixException(
"Allocation range in "
+ testName
+ " from "
+ matrixSource
+ " refers to unknown bucket value(s) "
+ unknownBuckets
+ " with length > 0");
}
// TODO(pwp): add some test constants?
final RuleEvaluator ruleEvaluator =
makeRuleEvaluator(RuleEvaluator.EXPRESSION_FACTORY, functionMapper);
PayloadSpecification payloadSpec = testSpecification.getPayload();
if (payloadSpec != null) {
final String specifiedPayloadTypeName =
Preconditions.checkNotNull(payloadSpec.getType(), "Missing payload spec type");
final PayloadType specifiedPayloadType =
PayloadType.payloadTypeForName(specifiedPayloadTypeName);
if (specifiedPayloadType == null) {
// This is probably redundant vs. TestGroupsGenerator.
throw new IncompatibleTestMatrixException(
"For test "
+ testName
+ " from "
+ matrixSource
+ " test specification payload type unknown: "
+ specifiedPayloadTypeName);
}
final String payloadValidatorRule = payloadSpec.getValidator();
final List<TestBucket> buckets = testDefinition.getBuckets();
for (final TestBucket bucket : buckets) {
Payload payload = bucket.getPayload();
if (payload != null) {
if (!specifiedPayloadType.payloadHasThisType(payload)) {
throw new IncompatibleTestMatrixException(
"For test "
+ testName
+ " from "
+ matrixSource
+ " expected payload of type "
+ specifiedPayloadType.payloadTypeName
+ " but matrix has a test bucket payload with wrong type: "
+ bucket);
}
if (payloadValidatorRule != null) {
final boolean payloadIsValid =
evaluatePayloadValidator(ruleEvaluator, payloadValidatorRule, payload);
if (!payloadIsValid) {
throw new IncompatibleTestMatrixException(
"For test "
+ testName
+ " from "
+ matrixSource
+ " payload validation rule "
+ payloadValidatorRule
+ " failed for test bucket: "
+ bucket);
}
}
}
}
}
}
private static void consolidate(
@Nonnull final TestMatrixArtifact testMatrix,
@Nonnull final Map<String, TestSpecification> requiredTests) {
final Map<String, ConsumableTestDefinition> definedTests = testMatrix.getTests();
// Sets.difference returns a "view" on the original set, which would require concurrent
// modification while iterating (copying the set will prevent this)
final Set<String> toRemove =
ImmutableSet.copyOf(Sets.difference(definedTests.keySet(), requiredTests.keySet()));
for (String testInMatrixNotRequired : toRemove) {
// we don't care about this test
definedTests.remove(testInMatrixNotRequired);
}
// Next, for any required tests that are missing, ensure that
// there is a nonnull test definition in the matrix
final Set<String> missing =
ImmutableSet.copyOf(Sets.difference(requiredTests.keySet(), definedTests.keySet()));
for (String testNotInMatrix : missing) {
definedTests.put(
testNotInMatrix, defaultFor(testNotInMatrix, requiredTests.get(testNotInMatrix)));
}
// Now go through definedTests: for each test, if the test spec
// didn't ask for a payload, then remove any payload that is in
// the test matrix.
for (Entry<String, ConsumableTestDefinition> next : definedTests.entrySet()) {
final String testName = next.getKey();
final ConsumableTestDefinition testDefinition = next.getValue();
final TestSpecification testSpec = requiredTests.get(testName);
if (testSpec.getPayload() == null) {
// No payload was requested...
final List<TestBucket> buckets = testDefinition.getBuckets();
for (final TestBucket bucket : buckets) {
if (bucket.getPayload() != null) {
// ... so stomp the unexpected payloads.
bucket.setPayload(null);
}
}
}
}
}
@Nonnull
private static ConsumableTestDefinition defaultFor(
final String testName, @Nonnull final TestSpecification testSpecification) {
final String missingTestSoleBucketName = "inactive";
final String missingTestSoleBucketDescription = "inactive";
final Allocation allocation = new Allocation();
allocation.setRanges(ImmutableList.of(new Range(testSpecification.getFallbackValue(), 1.0)));
return new ConsumableTestDefinition(
testSpecification.getFallbackValue(),
null,
TestType.RANDOM,
testName,
ImmutableList.of(
new TestBucket(
missingTestSoleBucketName,
testSpecification.getFallbackValue(),
missingTestSoleBucketDescription)),
// Force a nonnull allocation just in case something somewhere assumes 1.0 total allocation
Collections.singletonList(allocation),
Collections.<String, Object>emptyMap(),
testName);
}
public static void verifyInternallyConsistentDefinition(
final String testName,
final String matrixSource,
@Nonnull final ConsumableTestDefinition testDefinition)
throws IncompatibleTestMatrixException {
final List<Allocation> allocations = testDefinition.getAllocations();
if (allocations.isEmpty()) {
throw new IncompatibleTestMatrixException("No allocations specified in test " + testName);
}
final List<TestBucket> buckets = testDefinition.getBuckets();
/*
* test the matrix for consistency with itself
*/
final Set<Integer> definedBuckets = Sets.newHashSet();
for (final TestBucket bucket : buckets) {
definedBuckets.add(bucket.getValue());
}
for (final Allocation allocation : allocations) {
final List<Range> ranges = allocation.getRanges();
// ensure that each range refers to a known bucket
double bucketTotal = 0;
for (final Range range : ranges) {
bucketTotal += range.getLength();
// Internally consistent (within matrix itself)
if (!definedBuckets.contains(range.getBucketValue())) {
throw new IncompatibleTestMatrixException(
"Allocation range in "
+ testName
+ " from "
+ matrixSource
+ " refers to unknown bucket value "
+ range.getBucketValue());
}
}
// I hate floating points. TODO: extract a required precision constant/parameter?
if (bucketTotal < 0.9999
|| bucketTotal
> 1.0001) { // compensate for FP imprecision. TODO: determine what these bounds
// really should be by testing stuff
final StringBuilder sb =
new StringBuilder(
testName
+ " range with rule "
+ allocation.getRule()
+ " does not add up to 1 : ")
.append(ranges.get(0).getLength());
for (int i = 1; i < ranges.size(); i++) {
sb.append(" + ").append(ranges.get(i).getLength());
}
sb.append(" = ").append(bucketTotal);
throw new IncompatibleTestMatrixException(sb.toString());
}
}
final Allocation lastAllocation = allocations.get(allocations.size() - 1);
if (!CharMatcher.WHITESPACE.matchesAllOf(Strings.nullToEmpty(lastAllocation.getRule()))) {
throw new IncompatibleTestMatrixException(
"Final allocation for test "
+ testName
+ " from "
+ matrixSource
+ " has non-empty rule: "
+ lastAllocation.getRule());
}
/*
* When defined, within a single test, all test bucket payloads
* should be supplied; they should all have just one type each,
* and they should all be the same type.
*/
Payload nonEmptyPayload = null;
final List<TestBucket> bucketsWithoutPayloads = Lists.newArrayList();
for (final TestBucket bucket : buckets) {
final Payload p = bucket.getPayload();
if (p != null) {
if (p.numFieldsDefined() != 1) {
throw new IncompatibleTestMatrixException(
"Test "
+ testName
+ " from "
+ matrixSource
+ " has a test bucket payload with multiple types: "
+ bucket);
}
if (nonEmptyPayload == null) {
nonEmptyPayload = p;
} else if (!nonEmptyPayload.sameType(p)) {
throw new IncompatibleTestMatrixException(
"Test "
+ testName
+ " from "
+ matrixSource
+ " has test bucket: "
+ bucket
+ " incompatible with type of payload: "
+ nonEmptyPayload);
}
} else {
bucketsWithoutPayloads.add(bucket);
}
}
if ((nonEmptyPayload != null) && (bucketsWithoutPayloads.size() != 0)) {
throw new IncompatibleTestMatrixException(
"Test "
+ testName
+ " from "
+ matrixSource
+ " has some test buckets without payloads: "
+ bucketsWithoutPayloads);
}
}
// Make a new RuleEvaluator that captures the test constants.
// TODO(pwp): add some test constants?
@Nonnull
private static RuleEvaluator makeRuleEvaluator(
final ExpressionFactory expressionFactory, final FunctionMapper functionMapper) {
// Make the expression evaluation context.
final Map<String, Object> testConstants = Collections.emptyMap();
return new RuleEvaluator(expressionFactory, functionMapper, testConstants);
}
private static boolean evaluatePayloadValidator(
@Nonnull final RuleEvaluator ruleEvaluator, final String rule, @Nonnull final Payload payload)
throws IncompatibleTestMatrixException {
Map<String, Object> values = Collections.singletonMap("value", payload.fetchAValue());
try {
return ruleEvaluator.evaluateBooleanRule(rule, values);
} catch (@Nonnull final IllegalArgumentException e) {
LOGGER.error("Unable to evaluate rule ${" + rule + "} with payload " + payload, e);
}
return false;
}
}
/**
* Tests {@linkplain Serializers#serialize(java.io.Serializable)} method.
*
* @throws Exception exception
*/
@Test
public void serialize() throws Exception {
final byte[] bytes = Serializers.serialize(new Integer(0));
assertEquals(bytes.length, INTEGER_LENGTH);
}
public abstract class AbstractJsonProctorLoader extends AbstractProctorLoader {
private static final Logger LOGGER = Logger.getLogger(FileProctorLoader.class);
@Nonnull private final ObjectMapper objectMapper = Serializers.lenient();
@Nullable private String fileContents = null;
public AbstractJsonProctorLoader(
@Nonnull final Class<?> cls,
@Nonnull final ProctorSpecification specification,
@Nonnull final FunctionMapper functionMapper) {
super(cls, specification, functionMapper);
final ProctorLoaderDetail detailObject = new ProctorLoaderDetail();
VarExporter.forNamespace(detailObject.getClass().getSimpleName()).export(detailObject, "");
}
@Nullable
protected TestMatrixArtifact loadJsonTestMatrix(@Nonnull final Reader reader) throws IOException {
final char[] buffer = new char[1024];
final StringBuilder sb = new StringBuilder();
while (true) {
final int read = reader.read(buffer);
if (read == -1) {
break;
}
if (read > 0) {
sb.append(buffer, 0, read);
}
}
reader.close();
final String newContents = sb.toString();
try {
final TestMatrixArtifact testMatrix =
objectMapper.readValue(newContents, TestMatrixArtifact.class);
if (testMatrix != null) {
// record the file contents AFTER successfully loading the matrix
fileContents = newContents;
}
return testMatrix;
} catch (@Nonnull final JsonParseException e) {
LOGGER.error("Unable to load test matrix from " + getSource(), e);
throw e;
} catch (@Nonnull final JsonMappingException e) {
LOGGER.error("Unable to load test matrix from " + getSource(), e);
throw e;
} catch (@Nonnull final IOException e) {
LOGGER.error("Unable to load test matrix from " + getSource(), e);
throw e;
}
}
@Nullable
public String getFileContents() {
return fileContents;
}
/* class ProctorLoaderDetail is public so VarExporter works correctly */
public class ProctorLoaderDetail {
@Export(name = "file-source")
public String getFileSource() {
return getSource();
}
@Nullable
@Export(
name = "file-contents",
doc =
"The file contents of a recent successful load. If the file contains invalid JSON, the file contents will not be set.")
public String getLastFileContents() {
return getFileContents();
}
}
}
