public JobDescriptor(
String id, String conicalPath, String fullpath, Props props, ClassLoader classLoader) {
this._id = id;
this._path = conicalPath;
this._fullpath = fullpath;
this._props = PropsUtils.resolveProps(props);
this._jobType = props.getString(JOB_TYPE, "");
// @TODO Move this validation check in Java Job
// if(_jobType.length() == 0 || _jobType.equalsIgnoreCase("java")) {
// String className = props.getString(JOB_CLASS);
// this._class = Utils.loadClass(className, classLoader);
// }
this._readResourceLocks = props.getStringList(READ_LOCKS, ",");
this._dependencies = new HashSet<JobDescriptor>();
this._retries = props.getInt(RETRIES, 0);
this._retryBackoffMs = props.getLong(RETRY_BACKOFF, 0);
this._requiredPermits = props.getInt(JOB_PERMITS, 0);
this._classLoader = classLoader;
this._writeResourceLocks = props.getStringList(WRITE_LOCKS, ",");
this._sourceEmailList = props.getString("mail.sender", null);
// Ordered resource locking should help prevent simple deadlocking
// situations.
Collections.sort(this._readResourceLocks);
Collections.sort(this._writeResourceLocks);
this._emailList = props.getStringList(NOTIFY_EMAIL);
}
public void runPushStore(Props props, String url, String dataDir) throws Exception {
// For backwards compatibility http timeout = admin timeout
int httpTimeoutMs = 1000 * props.getInt("push.http.timeout.seconds", 24 * 60 * 60);
long pushVersion = props.getLong("push.version", -1L);
if (props.containsKey("push.version.timestamp")) {
DateFormat format = new SimpleDateFormat("yyyyMMddHHmmss");
pushVersion = Long.parseLong(format.format(new Date()));
}
int maxBackoffDelayMs = 1000 * props.getInt("push.backoff.delay.seconds", 60);
boolean rollback = props.getBoolean("push.rollback", true);
new VoldemortSwapJob(
this.getId() + "-push-store",
props,
new VoldemortSwapConf(
cluster,
dataDir,
storeName,
httpTimeoutMs,
pushVersion,
maxBackoffDelayMs,
rollback))
.run();
}
public VoldemortBuildAndPushJob(String name, Props props) {
super(name);
this.props = props;
this.storeName = props.getString("push.store.name").trim();
this.clusterUrl = new ArrayList<String>();
this.dataDirs = new ArrayList<String>();
String clusterUrlText = props.getString("push.cluster");
for (String url : Utils.COMMA_SEP.split(clusterUrlText.trim()))
if (url.trim().length() > 0) this.clusterUrl.add(url);
if (clusterUrl.size() <= 0) throw new RuntimeException("Number of urls should be atleast 1");
// Support multiple output dirs if the user mentions only "push", no
// "build".
// If user mentions both then should have only one
String dataDirText = props.getString("build.output.dir");
for (String dataDir : Utils.COMMA_SEP.split(dataDirText.trim()))
if (dataDir.trim().length() > 0) this.dataDirs.add(dataDir);
if (dataDirs.size() <= 0) throw new RuntimeException("Number of data dirs should be atleast 1");
this.nodeId = props.getInt("push.node", 0);
this.log = Logger.getLogger(name);
this.informedResults = Lists.newArrayList();
this.informedExecutor = Executors.newFixedThreadPool(2);
isAvroJob = props.getBoolean("build.type.avro", false);
keyField = props.getString("avro.key.field", null);
valueField = props.getString("avro.value.field", null);
if (isAvroJob) {
if (keyField == null)
throw new RuntimeException(
"The key field must be specified in the properties for the Avro build and push job!");
if (valueField == null)
throw new RuntimeException(
"The value field must be specified in the properties for the Avro build and push job!");
}
}
// Verify if the new avro schema being pushed is the same one as the old one
// Does not have logic to check for Avro schema evolution yet
public void verifyAvroSchema(String url) throws Exception {
// create new n store def with schema from the metadata in the input
// path
Schema schema = AvroUtils.getAvroSchemaFromPath(getInputPath());
int replicationFactor = props.getInt("build.replication.factor", 2);
int requiredReads = props.getInt("build.required.reads", 1);
int requiredWrites = props.getInt("build.required.writes", 1);
String description = props.getString("push.store.description", "");
String owners = props.getString("push.store.owners", "");
String keySchema =
"\n\t\t<type>avro-generic</type>\n\t\t<schema-info version=\"0\">"
+ schema.getField(keyField).schema()
+ "</schema-info>\n\t";
String valSchema =
"\n\t\t<type>avro-generic</type>\n\t\t<schema-info version=\"0\">"
+ schema.getField(valueField).schema()
+ "</schema-info>\n\t";
boolean hasCompression = false;
if (props.containsKey("build.compress.value")) hasCompression = true;
if (hasCompression) {
valSchema += "\t<compression><type>gzip</type></compression>\n\t";
}
if (props.containsKey("build.force.schema.key")) {
keySchema = props.get("build.force.schema.key");
}
if (props.containsKey("build.force.schema.value")) {
valSchema = props.get("build.force.schema.value");
}
String newStoreDefXml =
VoldemortUtils.getStoreDefXml(
storeName,
replicationFactor,
requiredReads,
requiredWrites,
props.containsKey("build.preferred.reads")
? props.getInt("build.preferred.reads")
: null,
props.containsKey("build.preferred.writes")
? props.getInt("build.preferred.writes")
: null,
(props.containsKey("push.force.schema.key"))
? props.getString("push.force.schema.key")
: keySchema,
(props.containsKey("push.force.schema.value"))
? props.getString("push.force.schema.value")
: valSchema,
description,
owners);
log.info("Verifying store: \n" + newStoreDefXml.toString());
StoreDefinition newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);
// get store def from cluster
log.info("Getting store definition from: " + url + " (node id " + this.nodeId + ")");
AdminClient adminClient = new AdminClient(url, new AdminClientConfig());
try {
List<StoreDefinition> remoteStoreDefs =
adminClient.getRemoteStoreDefList(this.nodeId).getValue();
boolean foundStore = false;
// go over all store defs and see if one has the same name as the
// store we're trying
// to build
for (StoreDefinition remoteStoreDef : remoteStoreDefs) {
if (remoteStoreDef.getName().equals(storeName)) {
// if the store already exists, but doesn't match what we
// want to push, we need
// to worry
if (!remoteStoreDef.equals(newStoreDef)) {
// let's check to see if the key/value serializers are
// REALLY equal.
SerializerDefinition localKeySerializerDef = newStoreDef.getKeySerializer();
SerializerDefinition localValueSerializerDef = newStoreDef.getValueSerializer();
SerializerDefinition remoteKeySerializerDef = remoteStoreDef.getKeySerializer();
SerializerDefinition remoteValueSerializerDef = remoteStoreDef.getValueSerializer();
if (remoteKeySerializerDef.getName().equals("avro-generic")
&& remoteValueSerializerDef.getName().equals("avro-generic")
&& remoteKeySerializerDef.getAllSchemaInfoVersions().size() == 1
&& remoteValueSerializerDef.getAllSchemaInfoVersions().size() == 1) {
Schema remoteKeyDef = Schema.parse(remoteKeySerializerDef.getCurrentSchemaInfo());
Schema remoteValDef = Schema.parse(remoteValueSerializerDef.getCurrentSchemaInfo());
Schema localKeyDef = Schema.parse(localKeySerializerDef.getCurrentSchemaInfo());
Schema localValDef = Schema.parse(localValueSerializerDef.getCurrentSchemaInfo());
if (remoteKeyDef.equals(localKeyDef) && remoteValDef.equals(localValDef)) {
String compressionPolicy = "";
if (hasCompression) {
compressionPolicy = "\n\t\t<compression><type>gzip</type></compression>";
}
// if the key/value serializers are REALLY equal
// (even though the strings may not match), then
// just use the remote stores to GUARANTEE that
// they
// match, and try again.
newStoreDefXml =
VoldemortUtils.getStoreDefXml(
storeName,
replicationFactor,
requiredReads,
requiredWrites,
props.containsKey("build.preferred.reads")
? props.getInt("build.preferred.reads")
: null,
props.containsKey("build.preferred.writes")
? props.getInt("build.preferred.writes")
: null,
"\n\t\t<type>avro-generic</type>\n\t\t<schema-info version=\"0\">"
+ remoteKeySerializerDef.getCurrentSchemaInfo()
+ "</schema-info>\n\t",
"\n\t\t<type>avro-generic</type>\n\t\t<schema-info version=\"0\">"
+ remoteValueSerializerDef.getCurrentSchemaInfo()
+ "</schema-info>"
+ compressionPolicy
+ "\n\t");
newStoreDef = VoldemortUtils.getStoreDef(newStoreDefXml);
if (!remoteStoreDef.equals(newStoreDef)) {
// if we still get a fail, then we know that
// the
// store defs don't match for reasons OTHER
// than
// the key/value serializer
throw new RuntimeException(
"Your store schema is identical, but the store definition does not match. Have: "
+ newStoreDef
+ "\nBut expected: "
+ remoteStoreDef);
}
} else {
// if the key/value serializers are not equal
// (even
// in java, not just json strings), then fail
throw new RuntimeException(
"Your store definition does not match the store definition that is already in the cluster. Tried to resolve identical schemas between local and remote, but failed. Have: "
+ newStoreDef
+ "\nBut expected: "
+ remoteStoreDef);
}
}
}
foundStore = true;
break;
}
}
// if the store doesn't exist yet, create it
if (!foundStore) {
// New requirement - Make sure the user had description and
// owner specified
if (description.length() == 0) {
throw new RuntimeException(
"Description field missing in store definition. "
+ "Please add \"push.store.description\" with a line describing your store");
}
if (owners.length() == 0) {
throw new RuntimeException(
"Owner field missing in store definition. "
+ "Please add \"push.store.owners\" with value being comma-separated list of LinkedIn email ids");
}
log.info("Could not find store " + storeName + " on Voldemort. Adding it to all nodes ");
adminClient.addStore(newStoreDef);
}
storeDefs =
ImmutableList.of(
VoldemortUtils.getStoreDef(
VoldemortUtils.getStoreDefXml(
storeName,
replicationFactor,
requiredReads,
requiredWrites,
props.containsKey("build.preferred.reads")
? props.getInt("build.preferred.reads")
: null,
props.containsKey("build.preferred.writes")
? props.getInt("build.preferred.writes")
: null,
keySchema,
valSchema)));
cluster = adminClient.getAdminClientCluster();
} finally {
adminClient.stop();
}
}
public String runBuildStore(Props props, String url) throws Exception {
int replicationFactor = props.getInt("build.replication.factor", 2);
int chunkSize = props.getInt("build.chunk.size", 1024 * 1024 * 1024);
Path tempDir =
new Path(
props.getString(
"build.temp.dir", "/tmp/vold-build-and-push-" + new Random().nextLong()));
URI uri = new URI(url);
Path outputDir = new Path(props.getString("build.output.dir"), uri.getHost());
Path inputPath = getInputPath();
String keySelection = props.getString("build.key.selection", null);
String valSelection = props.getString("build.value.selection", null);
CheckSumType checkSumType =
CheckSum.fromString(props.getString("checksum.type", CheckSum.toString(CheckSumType.MD5)));
boolean saveKeys = props.getBoolean("save.keys", true);
boolean reducerPerBucket = props.getBoolean("reducer.per.bucket", false);
int numChunks = props.getInt("num.chunks", -1);
if (isAvroJob) {
String recSchema = getRecordSchema();
String keySchema = getKeySchema();
String valSchema = getValueSchema();
new VoldemortStoreBuilderJob(
this.getId() + "-build-store",
props,
new VoldemortStoreBuilderConf(
replicationFactor,
chunkSize,
tempDir,
outputDir,
inputPath,
cluster,
storeDefs,
storeName,
keySelection,
valSelection,
null,
null,
checkSumType,
saveKeys,
reducerPerBucket,
numChunks,
keyField,
valueField,
recSchema,
keySchema,
valSchema),
true)
.run();
return outputDir.toString();
}
new VoldemortStoreBuilderJob(
this.getId() + "-build-store",
props,
new VoldemortStoreBuilderConf(
replicationFactor,
chunkSize,
tempDir,
outputDir,
inputPath,
cluster,
storeDefs,
storeName,
keySelection,
valSelection,
null,
null,
checkSumType,
saveKeys,
reducerPerBucket,
numChunks))
.run();
return outputDir.toString();
}
