// todo: method contains few constants that should be moved to interface
private void generateActualJobGrid(
ComputeJob computeJob, Hashtable<String, String> config, int generationCount) {
// create values hashtable to fill templates
Hashtable<String, String> values = new Hashtable<String, String>();
values.put("script_name", computeJob.getName());
values.put("error_log", "err_" + computeJob.getName() + ".log");
values.put("output_log", "out_" + computeJob.getName() + ".log");
values.put("script_location", config.get(JobGenerator.BACK_END_DIR));
values.put("node_name", computeJob.getName());
// create jdl
String jdlListing = weaveFreemarker(templateGridJDL, values);
if (workflowHasDependencies) {
String dagNodeListing = weaveFreemarker(templateGridDAGNode, values);
dagScript += dagNodeListing + "\n";
}
// write jdl
(new File(config.get(JobGenerator.OUTPUT_DIR))).mkdirs();
writeToFile(
config.get(JobGenerator.OUTPUT_DIR)
+ System.getProperty("file.separator")
+ computeJob.getName()
+ ".jdl",
jdlListing);
// create shell
String shellListing = templateGridHeader;
String initialScript = computeJob.getComputeScript();
GridTransferContainer container = pairJobTransfers.get(computeJob.getName());
// get log filename
Hashtable<String, String> logs = container.getLogs();
Enumeration logValues = logs.elements();
String logName = (String) logValues.nextElement();
String justLogName = giveJustName(logName);
// set log name to computeJob, that will be used in grid handler
computeJob.setLogFile("lfn://grid/" + logName);
// generate downloading section (transfer inputs and executable)
// and change job listing to execute in the grid
Hashtable<String, String> inputs = container.getInputs();
Enumeration actuals = inputs.elements();
while (actuals.hasMoreElements()) {
Hashtable<String, String> local = new Hashtable<String, String>();
String actualName = (String) actuals.nextElement();
String justName = giveJustName(actualName);
local.put(JobGenerator.LFN_NAME, actualName);
local.put(JobGenerator.INPUT, justName);
local.put(JobGenerator.LOG, justLogName);
String inputListing = weaveFreemarker(templateGridDownload, local);
// escapes are required to avoid $ sign processing in String replaceAll method
justName = "\\" + justName;
initialScript = initialScript.replaceAll(actualName, justName);
shellListing += inputListing;
}
Hashtable<String, String> exes = container.getExes();
actuals = exes.elements();
while (actuals.hasMoreElements()) {
Hashtable<String, String> local = new Hashtable<String, String>();
String actualName = (String) actuals.nextElement();
String justName = giveJustName(actualName);
local.put(JobGenerator.LFN_NAME, actualName);
local.put(JobGenerator.INPUT, justName);
local.put(JobGenerator.LOG, justLogName);
String inputListing = weaveFreemarker(templateGridDownloadExe, local);
logger.log(Level.DEBUG, "-----------");
logger.log(Level.DEBUG, initialScript);
logger.log(Level.DEBUG, "act " + actualName);
logger.log(Level.DEBUG, "just " + justName);
justName = "\\" + justName;
initialScript = initialScript.replaceAll(actualName, justName);
shellListing += inputListing;
}
shellListing += initialScript;
// generate uploading section
// and change job listing to execute in the grid
String outputsString = "";
Hashtable<String, String> outputs = container.getOutputs();
actuals = outputs.elements();
while (actuals.hasMoreElements()) {
Hashtable<String, String> local = new Hashtable<String, String>();
String actualName = (String) actuals.nextElement();
String justName = giveJustName(actualName);
// set output file to compute job, that will be used in grid handler
computeJob.setOutputFile("lfn://grid/" + actualName);
local.put(JobGenerator.LFN_NAME, actualName);
local.put(JobGenerator.OUTPUT, justName);
local.put(JobGenerator.LOG, justLogName);
String outputListing = weaveFreemarker(templateGridUpload, local);
justName = "\\" + justName;
shellListing = shellListing.replaceAll(actualName, justName);
// shellListing += outputListing;
outputsString += outputListing;
}
shellListing += outputsString;
// add upload log
Hashtable<String, String> local = new Hashtable<String, String>();
local.put(JobGenerator.LFN_NAME, logName);
local.put(JobGenerator.OUTPUT, justLogName);
local.put(JobGenerator.LOG, justLogName);
String outputListing = weaveFreemarker(templateGridUploadLog, local);
shellListing += outputListing;
// write shell
writeToFile(
config.get(JobGenerator.OUTPUT_DIR)
+ System.getProperty("file.separator")
+ computeJob.getName()
+ ".sh",
shellListing);
// and computeJob to grid handler
gridHandler.setComputeJob(computeJob);
// write job dependencies to DAG
WorkflowElement el = pairCJtoWE.get(computeJob);
if (el.getPreviousSteps().size() > 0) {
String jobName = computeJob.getName();
Vector<String> dependencyNames = new Vector<String>();
for (WorkflowElement wEl : el.getPreviousSteps()) {
ComputeJob cJ = pairWEtoCJ.get(wEl);
String strDependencyName = cJ.getName();
dependencyNames.addElement(strDependencyName);
}
String strDependency = "";
// more than one previous element
if (dependencyNames.size() > 1) {
for (String str : dependencyNames) {
strDependency += str + ", ";
}
// cut last coma
strDependency = strDependency.substring(0, strDependency.length() - 2);
strDependency = "{ " + strDependency + " }";
}
// only one dependency element
else {
strDependency = dependencyNames.elementAt(0);
}
// format string and add to DAg
String toAdd = "{ " + jobName + ", " + strDependency + " },\n";
dagDependencies = toAdd + dagDependencies;
}
}
public boolean generateActualJobs(
Vector<ComputeJob> computeJobs, String backend, Hashtable<String, String> config) {
int generationCount = -1;
// read templates
String templatesDir = config.get(JobGenerator.TEMPLATE_DIR);
if (backend.equalsIgnoreCase(JobGenerator.GRID)) {
readTemplatesGrid(templatesDir);
gridHandler = new CommandLineImputationGridHandler();
gridHandler.setWorksheet(worksheet);
generationCount = gridHandler.getNextJobID();
// adding dag dependencies
if (workflowHasDependencies) {
dagScript += "nodes = [\n";
}
} else if (backend.equalsIgnoreCase(JobGenerator.CLUSTER)) readTemplatesCluster(templatesDir);
for (ComputeJob computeJob : computeJobs) {
System.out.println(">>> generation job: " + computeJob.getName());
// generate files for selected back-end
if (backend.equalsIgnoreCase(JobGenerator.GRID))
generateActualJobGrid(computeJob, config, generationCount);
else if (backend.equalsIgnoreCase(JobGenerator.CLUSTER))
generateActualJobCluster(computeJob, config);
generationCount++;
}
// write cluster submit script
if (backend.equalsIgnoreCase(JobGenerator.CLUSTER))
writeToFile(
config.get(JobGenerator.OUTPUT_DIR)
+ System.getProperty("file.separator")
+ "submit_"
+ config.get(JobGenerator.GENERATION_ID)
+ ".sh",
submitScript);
else if (backend.equalsIgnoreCase(JobGenerator.GRID) && workflowHasDependencies) {
// produce targetsListFile
gridHandler.writeJobsLogsToFile(config);
// finilize dag
dagScript += "\n];";
// cut last coma and new line
dagDependencies = dagDependencies.substring(0, dagDependencies.length() - 2);
dagDependencies = "\ndependencies = {\n" + dagDependencies + "\n}";
dagScript += dagDependencies;
// write dag file
writeToFile(
config.get(JobGenerator.OUTPUT_DIR)
+ System.getProperty("file.separator")
+ "dag_"
+ config.get(JobGenerator.GENERATION_ID)
+ ".jdl",
dagScript);
}
return true;
}