@Test
public void testSanity() throws Exception {
final AtomicLong numJobsComplete = new AtomicLong(0);
/** ** Setup mockFlow1 *** */
final Capture<FlowCallback> flow1Callback = new Capture<FlowCallback>();
mockFlow1.execute(EasyMock.eq(props), EasyMock.capture(flow1Callback));
EasyMock.expectLastCall()
.andAnswer(
new IAnswer<Void>() {
@Override
public Void answer() throws Throwable {
Assert.assertEquals(Status.RUNNING, flow.getStatus());
Assert.assertEquals(1, numJobsComplete.incrementAndGet());
flow1Callback.getValue().completed(Status.SUCCEEDED);
Assert.assertEquals(Status.RUNNING, flow.getStatus());
return null;
}
})
.once();
Props mockFlow1Props = new Props();
mockFlow1Props.put("1", "1");
mockFlow1Props.put("2", "1");
EasyMock.expect(mockFlow1.getStatus()).andReturn(Status.SUCCEEDED).times(2);
EasyMock.expect(mockFlow1.getReturnProps()).andReturn(mockFlow1Props).once();
/** ** Setup mockFlow2 *** */
final Capture<FlowCallback> flow2Callback = new Capture<FlowCallback>();
mockFlow2.execute(EasyMock.eq(props), EasyMock.capture(flow2Callback));
EasyMock.expectLastCall()
.andAnswer(
new IAnswer<Void>() {
@Override
public Void answer() throws Throwable {
Assert.assertEquals(Status.RUNNING, flow.getStatus());
Assert.assertEquals(2, numJobsComplete.incrementAndGet());
flow2Callback.getValue().completed(Status.SUCCEEDED);
Assert.assertEquals(Status.SUCCEEDED, flow.getStatus());
return null;
}
})
.once();
EasyMock.expect(mockFlow2.getStatus())
.andAnswer(
new IAnswer<Status>() {
private volatile AtomicInteger count = new AtomicInteger(0);
@Override
public Status answer() throws Throwable {
switch (count.getAndIncrement()) {
case 0:
return Status.READY;
case 1:
return Status.SUCCEEDED;
default:
Assert.fail("mockFlow2.getStatus() should only be called 2 times.");
}
return null;
}
})
.times(2);
Props mockFlow2Props = new Props();
mockFlow2Props.put("2", "2");
mockFlow2Props.put("3", "2");
EasyMock.expect(mockFlow2.getReturnProps()).andReturn(mockFlow2Props).once();
EasyMock.expect(props.equalsProps(props)).andReturn(true).once();
EasyMock.replay(mockFlow1, mockFlow2, props);
/** ** Start the test *** */
AtomicBoolean callbackRan = new AtomicBoolean(false);
flow.execute(
props,
new OneCallFlowCallback(callbackRan) {
@Override
public void theCallback(Status status) {
Assert.assertEquals(Status.SUCCEEDED, status);
Assert.assertEquals(2, numJobsComplete.get());
}
});
Assert.assertTrue("Callback wasn't run.", callbackRan.get());
Assert.assertEquals(Status.SUCCEEDED, flow.getStatus());
Assert.assertEquals(emptyExceptions, flow.getExceptions());
Assert.assertEquals(props, flow.getParentProps());
callbackRan = new AtomicBoolean(false);
flow.execute(
props,
new OneCallFlowCallback(callbackRan) {
@Override
protected void theCallback(Status status) {
Assert.assertEquals(Status.SUCCEEDED, status);
Assert.assertEquals(2, numJobsComplete.get());
}
});
Assert.assertTrue("Callback wasn't run.", callbackRan.get());
Assert.assertEquals(Status.SUCCEEDED, flow.getStatus());
Assert.assertEquals(emptyExceptions, flow.getExceptions());
Props retProps = flow.getReturnProps();
Assert.assertEquals(3, retProps.size());
Assert.assertEquals("1", retProps.get("1"));
Assert.assertEquals("2", retProps.get("2"));
Assert.assertEquals("2", retProps.get("3"));
EasyMock.verify(props);
EasyMock.reset(props);
EasyMock.expect(props.equalsProps(props)).andReturn(false).once();
EasyMock.replay(props);
boolean exceptionThrown = false;
try {
flow.execute(
props,
new FlowCallback() {
@Override
public void progressMade() {}
@Override
public void completed(Status status) {}
});
} catch (IllegalArgumentException e) {
exceptionThrown = true;
}
Assert.assertTrue(
"Expected an IllegalArgumentException to be thrown because props weren't the same.",
exceptionThrown);
}
// Verify if the new avro schema being pushed is the same one as the old one
// Does not have logic to check for Avro schema evolution yet
public void verifyAvroSchema(String url) throws Exception {
// create new n store def with schema from the metadata in the input
// path
Schema schema = AvroUtils.getAvroSchemaFromPath(getInputPath());
int replicationFactor = props.getInt("build.replication.factor", 2);
int requiredReads = props.getInt("build.required.reads", 1);
int requiredWrites = props.getInt("build.required.writes", 1);
String description = props.getString("push.store.description", "");
String owners = props.getString("push.store.owners", "");
String keySchema =
"\n\t\t<type>avro-generic</type>\n\t\t<schema-info version=\"0\">"
+ schema.getField(keyField).schema()
+ "</schema-info>\n\t";
String valSchema =
"\n\t\t<type>avro-generic</type>\n\t\t<schema-info version=\"0\">"
+ schema.getField(valueField).schema()
+ "</schema-info>\n\t";
boolean hasCompression = false;
if (props.containsKey("build.compress.value")) hasCompression = true;
if (hasCompression) {
valSchema += "\t<compression><type>gzip</type></compression>\n\t";
}
if (props.containsKey("build.force.schema.key")) {
keySchema = props.get("build.force.schema.key");
}
if (props.containsKey("build.force.schema.value")) {
valSchema = props.get("build.force.schema.value");
}
String newStoreDefXml =
VoldemortUtils.getStoreDefXml(
storeName,
replicationFactor,
requiredReads,
requiredWrites,
props.containsKey("build.preferred.reads")
? props.getInt("build.preferred.reads")
: null,
props.containsKey("build.preferred.writes")
? props.getInt("build.preferred.writes")
: null,
(props.containsKey("push.force.schema.key"))
? props.getString("push.force.schema.key")
: keySchema,
(props.containsKey("push.force.schema.value"))
? props.getString("push.force.schema.value")
: valSchema,
description,
owners);
log.info("Verifying store: \n" + newStoreDefXml.toString());
StoreDefinition newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);
// get store def from cluster
log.info("Getting store definition from: " + url + " (node id " + this.nodeId + ")");
AdminClient adminClient = new AdminClient(url, new AdminClientConfig());
try {
List<StoreDefinition> remoteStoreDefs =
adminClient.getRemoteStoreDefList(this.nodeId).getValue();
boolean foundStore = false;
// go over all store defs and see if one has the same name as the
// store we're trying
// to build
for (StoreDefinition remoteStoreDef : remoteStoreDefs) {
if (remoteStoreDef.getName().equals(storeName)) {
// if the store already exists, but doesn't match what we
// want to push, we need
// to worry
if (!remoteStoreDef.equals(newStoreDef)) {
// let's check to see if the key/value serializers are
// REALLY equal.
SerializerDefinition localKeySerializerDef = newStoreDef.getKeySerializer();
SerializerDefinition localValueSerializerDef = newStoreDef.getValueSerializer();
SerializerDefinition remoteKeySerializerDef = remoteStoreDef.getKeySerializer();
SerializerDefinition remoteValueSerializerDef = remoteStoreDef.getValueSerializer();
if (remoteKeySerializerDef.getName().equals("avro-generic")
&& remoteValueSerializerDef.getName().equals("avro-generic")
&& remoteKeySerializerDef.getAllSchemaInfoVersions().size() == 1
&& remoteValueSerializerDef.getAllSchemaInfoVersions().size() == 1) {
Schema remoteKeyDef = Schema.parse(remoteKeySerializerDef.getCurrentSchemaInfo());
Schema remoteValDef = Schema.parse(remoteValueSerializerDef.getCurrentSchemaInfo());
Schema localKeyDef = Schema.parse(localKeySerializerDef.getCurrentSchemaInfo());
Schema localValDef = Schema.parse(localValueSerializerDef.getCurrentSchemaInfo());
if (remoteKeyDef.equals(localKeyDef) && remoteValDef.equals(localValDef)) {
String compressionPolicy = "";
if (hasCompression) {
compressionPolicy = "\n\t\t<compression><type>gzip</type></compression>";
}
// if the key/value serializers are REALLY equal
// (even though the strings may not match), then
// just use the remote stores to GUARANTEE that
// they
// match, and try again.
newStoreDefXml =
VoldemortUtils.getStoreDefXml(
storeName,
replicationFactor,
requiredReads,
requiredWrites,
props.containsKey("build.preferred.reads")
? props.getInt("build.preferred.reads")
: null,
props.containsKey("build.preferred.writes")
? props.getInt("build.preferred.writes")
: null,
"\n\t\t<type>avro-generic</type>\n\t\t<schema-info version=\"0\">"
+ remoteKeySerializerDef.getCurrentSchemaInfo()
+ "</schema-info>\n\t",
"\n\t\t<type>avro-generic</type>\n\t\t<schema-info version=\"0\">"
+ remoteValueSerializerDef.getCurrentSchemaInfo()
+ "</schema-info>"
+ compressionPolicy
+ "\n\t");
newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);
if (!remoteStoreDef.equals(newStoreDef)) {
// if we still get a fail, then we know that
// the
// store defs don't match for reasons OTHER
// than
// the key/value serializer
throw new RuntimeException(
"Your store schema is identical, but the store definition does not match. Have: "
+ newStoreDef
+ "\nBut expected: "
+ remoteStoreDef);
}
} else {
// if the key/value serializers are not equal
// (even
// in java, not just json strings), then fail
throw new RuntimeException(
"Your store definition does not match the store definition that is already in the cluster. Tried to resolve identical schemas between local and remote, but failed. Have: "
+ newStoreDef
+ "\nBut expected: "
+ remoteStoreDef);
}
}
}
foundStore = true;
break;
}
}
// if the store doesn't exist yet, create it
if (!foundStore) {
// New requirement - Make sure the user had description and
// owner specified
if (description.length() == 0) {
throw new RuntimeException(
"Description field missing in store definition. "
+ "Please add \"push.store.description\" with a line describing your store");
}
if (owners.length() == 0) {
throw new RuntimeException(
"Owner field missing in store definition. "
+ "Please add \"push.store.owners\" with value being comma-separated list of LinkedIn email ids");
}
log.info("Could not find store " + storeName + " on Voldemort. Adding it to all nodes ");
adminClient.addStore(newStoreDef);
}
storeDefs =
ImmutableList.of(
VoldemortUtils.getStoreDef(
VoldemortUtils.getStoreDefXml(
storeName,
replicationFactor,
requiredReads,
requiredWrites,
props.containsKey("build.preferred.reads")
? props.getInt("build.preferred.reads")
: null,
props.containsKey("build.preferred.writes")
? props.getInt("build.preferred.writes")
: null,
keySchema,
valSchema)));
cluster = adminClient.getAdminClientCluster();
} finally {
adminClient.stop();
}
}
