// here we create macros listing
private String makeMacros(
GridTransferContainer container, String script, String type, String backend) {
String text = "";
if (type.equalsIgnoreCase("input")) {
text += JobGenerator.SOURCE_SCRIPT + "\n";
Vector<String> nameInputs = loader.findFlagValues(script, ModelLoader.FLAG_INPUTS);
Hashtable<String, String> inputs = container.getInputs();
for (String s : nameInputs) {
String macroline = makeMacroLine(s, inputs, type, backend);
text += macroline;
}
Vector<String> nameExes = loader.findFlagValues(script, ModelLoader.FLAG_EXES);
Hashtable<String, String> exes = container.getExes();
for (String s : nameExes) {
String macroline = makeMacroLine(s, exes, JobGenerator.EXE, backend);
text += macroline;
}
} else if (type.equalsIgnoreCase("output")) {
text += "\n";
Vector<String> nameOutputs = loader.findFlagValues(script, ModelLoader.FLAG_OUTPUTS);
Hashtable<String, String> outputs = container.getOutputs();
for (String s : nameOutputs) {
String macroline = makeMacroLine(s, outputs, type, backend);
text += macroline;
}
}
return text;
}
private GridTransferContainer fillContainer(
ComputeProtocol protocol, Hashtable<String, Object> values) {
GridTransferContainer container = new GridTransferContainer();
List<ComputeParameter> inputs = protocol.getInputs();
for (ComputeParameter input : inputs) {
String name = input.getName();
String value = (String) values.get(name);
container.addInput(name, value);
}
List<ComputeParameter> outputs = protocol.getOutputs();
for (ComputeParameter output : outputs) {
String name = output.getName();
String value = (String) values.get(name);
container.addOutput(name, value);
}
List<ComputeParameter> exes = protocol.getExes();
for (ComputeParameter exe : exes) {
String name = exe.getName();
String value = (String) values.get(name);
container.addExe(name, value);
}
List<ComputeParameter> logs = protocol.getLogs();
for (ComputeParameter log : logs) {
String name = log.getName();
String value = (String) values.get(name);
container.addLog(name, value);
}
return container;
}
// todo: method contains few constants that should be moved to interface
private void generateActualJobGrid(
ComputeJob computeJob, Hashtable<String, String> config, int generationCount) {
// create values hashtable to fill templates
Hashtable<String, String> values = new Hashtable<String, String>();
values.put("script_name", computeJob.getName());
values.put("error_log", "err_" + computeJob.getName() + ".log");
values.put("output_log", "out_" + computeJob.getName() + ".log");
values.put("script_location", config.get(JobGenerator.BACK_END_DIR));
values.put("node_name", computeJob.getName());
// create jdl
String jdlListing = weaveFreemarker(templateGridJDL, values);
if (workflowHasDependencies) {
String dagNodeListing = weaveFreemarker(templateGridDAGNode, values);
dagScript += dagNodeListing + "\n";
}
// write jdl
(new File(config.get(JobGenerator.OUTPUT_DIR))).mkdirs();
writeToFile(
config.get(JobGenerator.OUTPUT_DIR)
+ System.getProperty("file.separator")
+ computeJob.getName()
+ ".jdl",
jdlListing);
// create shell
String shellListing = templateGridHeader;
String initialScript = computeJob.getComputeScript();
GridTransferContainer container = pairJobTransfers.get(computeJob.getName());
// get log filename
Hashtable<String, String> logs = container.getLogs();
Enumeration logValues = logs.elements();
String logName = (String) logValues.nextElement();
String justLogName = giveJustName(logName);
// set log name to computeJob, that will be used in grid handler
computeJob.setLogFile("lfn://grid/" + logName);
// generate downloading section (transfer inputs and executable)
// and change job listing to execute in the grid
Hashtable<String, String> inputs = container.getInputs();
Enumeration actuals = inputs.elements();
while (actuals.hasMoreElements()) {
Hashtable<String, String> local = new Hashtable<String, String>();
String actualName = (String) actuals.nextElement();
String justName = giveJustName(actualName);
local.put(JobGenerator.LFN_NAME, actualName);
local.put(JobGenerator.INPUT, justName);
local.put(JobGenerator.LOG, justLogName);
String inputListing = weaveFreemarker(templateGridDownload, local);
// escapes are required to avoid $ sign processing in String replaceAll method
justName = "\\" + justName;
initialScript = initialScript.replaceAll(actualName, justName);
shellListing += inputListing;
}
Hashtable<String, String> exes = container.getExes();
actuals = exes.elements();
while (actuals.hasMoreElements()) {
Hashtable<String, String> local = new Hashtable<String, String>();
String actualName = (String) actuals.nextElement();
String justName = giveJustName(actualName);
local.put(JobGenerator.LFN_NAME, actualName);
local.put(JobGenerator.INPUT, justName);
local.put(JobGenerator.LOG, justLogName);
String inputListing = weaveFreemarker(templateGridDownloadExe, local);
logger.log(Level.DEBUG, "-----------");
logger.log(Level.DEBUG, initialScript);
logger.log(Level.DEBUG, "act " + actualName);
logger.log(Level.DEBUG, "just " + justName);
justName = "\\" + justName;
initialScript = initialScript.replaceAll(actualName, justName);
shellListing += inputListing;
}
shellListing += initialScript;
// generate uploading section
// and change job listing to execute in the grid
String outputsString = "";
Hashtable<String, String> outputs = container.getOutputs();
actuals = outputs.elements();
while (actuals.hasMoreElements()) {
Hashtable<String, String> local = new Hashtable<String, String>();
String actualName = (String) actuals.nextElement();
String justName = giveJustName(actualName);
// set output file to compute job, that will be used in grid handler
computeJob.setOutputFile("lfn://grid/" + actualName);
local.put(JobGenerator.LFN_NAME, actualName);
local.put(JobGenerator.OUTPUT, justName);
local.put(JobGenerator.LOG, justLogName);
String outputListing = weaveFreemarker(templateGridUpload, local);
justName = "\\" + justName;
shellListing = shellListing.replaceAll(actualName, justName);
// shellListing += outputListing;
outputsString += outputListing;
}
shellListing += outputsString;
// add upload log
Hashtable<String, String> local = new Hashtable<String, String>();
local.put(JobGenerator.LFN_NAME, logName);
local.put(JobGenerator.OUTPUT, justLogName);
local.put(JobGenerator.LOG, justLogName);
String outputListing = weaveFreemarker(templateGridUploadLog, local);
shellListing += outputListing;
// write shell
writeToFile(
config.get(JobGenerator.OUTPUT_DIR)
+ System.getProperty("file.separator")
+ computeJob.getName()
+ ".sh",
shellListing);
// and computeJob to grid handler
gridHandler.setComputeJob(computeJob);
// write job dependencies to DAG
WorkflowElement el = pairCJtoWE.get(computeJob);
if (el.getPreviousSteps().size() > 0) {
String jobName = computeJob.getName();
Vector<String> dependencyNames = new Vector<String>();
for (WorkflowElement wEl : el.getPreviousSteps()) {
ComputeJob cJ = pairWEtoCJ.get(wEl);
String strDependencyName = cJ.getName();
dependencyNames.addElement(strDependencyName);
}
String strDependency = "";
// more than one previous element
if (dependencyNames.size() > 1) {
for (String str : dependencyNames) {
strDependency += str + ", ";
}
// cut last coma
strDependency = strDependency.substring(0, strDependency.length() - 2);
strDependency = "{ " + strDependency + " }";
}
// only one dependency element
else {
strDependency = dependencyNames.elementAt(0);
}
// format string and add to DAg
String toAdd = "{ " + jobName + ", " + strDependency + " },\n";
dagDependencies = toAdd + dagDependencies;
}
}
private void generateActualJobGridMacro(
ComputeJob computeJob,
Hashtable<String, String> config,
String strMacrosInput,
String strMacrosOutput) {
Hashtable<String, String> values = new Hashtable<String, String>();
values.put("script_name", computeJob.getName());
values.put("error_log", "err_" + computeJob.getName() + ".log");
values.put("output_log", "out_" + computeJob.getName() + ".log");
values.put("script_location", config.get(JobGenerator.BACK_END_DIR));
values.put("node_name", computeJob.getName());
// create jdl
String jdlListing = weaveFreemarker(templateGridJDL, values);
// write jdl
(new File(config.get(JobGenerator.OUTPUT_DIR))).mkdirs();
writeToFile(
config.get(JobGenerator.OUTPUT_DIR)
+ System.getProperty("file.separator")
+ computeJob.getName()
+ ".jdl",
jdlListing);
// create shell
String shellListing = templateGridHeader;
String initialScript = computeJob.getComputeScript();
GridTransferContainer container = pairJobTransfers.get(computeJob.getName());
// generate downloading section (transfer inputs and executable)
// and change job listing to execute in the grid
Hashtable<String, String> inputs = container.getInputs();
Enumeration actuals = inputs.elements();
while (actuals.hasMoreElements()) {
String actualName = (String) actuals.nextElement();
String justName = giveJustName(actualName);
// escapes are required to avoid $ sign processing in String replaceAll method
justName = "\\" + justName;
// commented out - but I keep it if I need to sho somebody the problem of replacement with
// wild cards
// System.out.println("actual " + actualName);
// System.out.println("just " + justName);
//
// System.out.println("------------- before");
// System.out.println(initialScript);
initialScript = initialScript.replaceAll(actualName, justName);
// System.out.println("------------- after");
// System.out.println(initialScript);
}
Hashtable<String, String> exes = container.getExes();
actuals = exes.elements();
while (actuals.hasMoreElements()) {
String actualName = (String) actuals.nextElement();
String justName = giveJustName(actualName);
justName = "\\" + justName;
initialScript = initialScript.replaceAll(actualName, justName);
}
shellListing += strMacrosInput;
shellListing += initialScript;
String outputsString = "";
Hashtable<String, String> outputs = container.getOutputs();
actuals = outputs.elements();
while (actuals.hasMoreElements()) {
String actualName = (String) actuals.nextElement();
String justName = giveJustName(actualName);
justName = "\\" + justName;
shellListing = shellListing.replaceAll(actualName, justName);
}
shellListing += strMacrosOutput;
// write shell
writeToFile(
config.get(JobGenerator.OUTPUT_DIR)
+ System.getProperty("file.separator")
+ computeJob.getName()
+ ".sh",
shellListing);
}
