private GridTransferContainer fillContainer(
ComputeProtocol protocol, Hashtable<String, Object> values) {
GridTransferContainer container = new GridTransferContainer();
List<ComputeParameter> inputs = protocol.getInputs();
for (ComputeParameter input : inputs) {
String name = input.getName();
String value = (String) values.get(name);
container.addInput(name, value);
}
List<ComputeParameter> outputs = protocol.getOutputs();
for (ComputeParameter output : outputs) {
String name = output.getName();
String value = (String) values.get(name);
container.addOutput(name, value);
}
List<ComputeParameter> exes = protocol.getExes();
for (ComputeParameter exe : exes) {
String name = exe.getName();
String value = (String) values.get(name);
container.addExe(name, value);
}
List<ComputeParameter> logs = protocol.getLogs();
for (ComputeParameter log : logs) {
String name = log.getName();
String value = (String) values.get(name);
container.addLog(name, value);
}
return container;
}
// here, we also identify what parameters should be foldered
private Pair<String, Object> processDependentParameter(
ComputeParameter par,
Hashtable<String, Object> line,
Hashtable<String, String> simpleValues) {
Pair<String, Object> pair = new Pair<String, Object>();
pair.setA(par.getName());
int lineFolderedSize = foldingParser.getFolderedLineSize(line);
String parTemplate = par.getDefaultValue();
foldingParser.setNotList();
foldingParser.checkIsList(parTemplate);
boolean isList = foldingParser.getIsList();
if (lineFolderedSize > 1 && isList) {
List<String> values = new ArrayList<String>();
for (int i = 0; i < lineFolderedSize; i++) {
String value = foldingParser.parseTemplateLineByHand(parTemplate, line, i, simpleValues);
values.add(value);
}
pair.setB(values);
} else {
String value = foldingParser.parseTemplateLineByHand(parTemplate, line, 0, simpleValues);
pair.setB(value);
}
return pair;
}
private boolean isTarget(String ekey, List<ComputeParameter> targets) {
for (ComputeParameter par : targets) {
String name = par.getName();
if (name.equalsIgnoreCase(ekey)) return true;
}
return false;
}
public Vector<ComputeJob> generateComputeJobsFoldedWorksheet(
Workflow workflow, List<Tuple> f, String backend) {
// create the table with targets, which is equal to worksheet if there are no targets
List<Hashtable> table = null;
// check if workflow elements have dependencies
workflowHasDependencies = hasDependencies(workflow);
// remove unused parameters from the worksheet
List<Hashtable> worksheet = foldingMaker.transformToTable(f);
foldingMaker.setWorkflow(workflow);
worksheet =
foldingMaker.removeUnused(
worksheet, (List<ComputeParameter>) workflow.getWorkflowComputeParameterCollection());
// result jobs
Vector<ComputeJob> computeJobs = new Vector<ComputeJob>();
// if (backend.equalsIgnoreCase(JobGenerator.GRID))
pairJobTransfers = new Hashtable<String, GridTransferContainer>();
pairWEtoCJ = new Hashtable<WorkflowElement, ComputeJob>();
pairCJtoWE = new Hashtable<ComputeJob, WorkflowElement>();
if (backend.equalsIgnoreCase(JobGenerator.CLUSTER)) {
submitScript = "";
} else if (backend.equalsIgnoreCase(JobGenerator.GRID) && workflowHasDependencies) {
dagScript = "Type=\"dag\";\n\n";
dagDependencies = "";
}
Collection<ComputeParameter> parameters = workflow.getWorkflowComputeParameterCollection();
Collection<WorkflowElement> workflowElements = workflow.getWorkflowWorkflowElementCollection();
for (WorkflowElement el : workflowElements) {
ComputeProtocol protocol = (ComputeProtocol) el.getProtocol();
String template = protocol.getScriptTemplate();
// chack if we have any targets
List<ComputeParameter> targets = foldingMaker.findTargets(protocol.getScriptTemplate());
if (targets != null) {
table = foldingMaker.fold(targets, worksheet);
} else table = worksheet;
// here, we prapare some information about folded table
// in particular, we find what parameters are Martijn's folded constants
foldingParser.evaluateTable(table);
// we set all parameters to foldered parser, which are used for reducing foldered constants
// this comment does give any clue to what is going on :)
foldingParser.setParametersList(parameters);
// now we start to use foldered worksheet
// all our parameters, that depend on foldered worksheet, will become lists as well, that
// upset me a lot :(
for (int i = 0; i < table.size(); i++) {
// because Hashtable does not allow null keys or values used for weaving
Hashtable<String, Object> values = new Hashtable<String, Object>();
// parameters which are templates and do directly depend on worksheet values
Vector<ComputeParameter> complexParameters = new Vector<ComputeParameter>();
// hashtable with simple values, we need it for initial weaving
Hashtable<String, String> simpleValues = new Hashtable<String, String>();
// job naming
String id = "id";
Hashtable<String, Object> line = table.get(i);
if (targets != null) {
// use targets to create name
Enumeration ekeys = line.keys();
while (ekeys.hasMoreElements()) {
String ekey = (String) ekeys.nextElement();
if (isTarget(ekey, targets)) {
Object eValues = line.get(ekey);
values.put(ekey, eValues);
String vvv = eValues.toString();
vvv = vvv.replaceAll(" ", "_");
id += "_" + vvv;
}
}
} else {
// use the whole line to create name
Enumeration ekeys = line.keys();
while (ekeys.hasMoreElements()) {
String ekey = (String) ekeys.nextElement();
Object eValues = line.get(ekey);
values.put(ekey, eValues);
String vvv = eValues.toString();
vvv = vvv.replaceAll(" ", "_");
id += "_" + vvv;
}
}
for (ComputeParameter parameter : parameters) {
if (parameter.getDefaultValue() != null) {
if (parameter.getDefaultValue().contains("${")) {
complexParameters.addElement(parameter);
} else {
values.put(parameter.getName(), parameter.getDefaultValue());
simpleValues.put(parameter.getName(), parameter.getDefaultValue());
}
}
}
logger.log(Level.DEBUG, "simple parameters before: " + values.size());
// we transform complex parameters to unweaved values, because unweaved value can folded
Hashtable<String, Object> unweavedValues = new Hashtable<String, Object>();
for (ComputeParameter par : complexParameters) {
Pair<String, Object> value = processDependentParameter(par, line, simpleValues);
if (foldingParser.isValueSimple(value)) {
values.put(par.getName(), value.getB());
} else {
unweavedValues.put(par.getName(), value.getB());
}
}
logger.log(
Level.DEBUG,
"simple parameters after "
+ values.size()
+ " parameters to weave: "
+ complexParameters.size());
Vector<String> vecToRemove = new Vector<String>();
int weavingCount = 0;
logger.log(Level.DEBUG, "loop " + weavingCount + " -> " + unweavedValues.size());
// in a loop weave complex parameters with values
while (unweavedValues.size() > 0 && weavingCount < 100) {
Enumeration unkeys = unweavedValues.keys();
while (unkeys.hasMoreElements()) {
String unkey = (String) unkeys.nextElement();
Object eValue = unweavedValues.get(unkey);
Pair<String, Object> value = null;
if (eValue instanceof Collection<?>) {
List<String> unweavedLines = (List<String>) eValue;
value = processUnweavedCollection(unkey, unweavedLines, values);
} else {
String unweavedLine = (String) eValue;
value = processUnweavedLine(unkey, unweavedLine, values, 0);
}
if (foldingParser.isValueSimple(value)) {
values.put(value.getA(), value.getB());
vecToRemove.add(unkey);
} else unweavedValues.put(value.getA(), value.getB());
}
for (String str : vecToRemove) unweavedValues.remove(str);
weavingCount++;
logger.log(
Level.DEBUG,
"loop/parameters to weave " + weavingCount + " -> " + unweavedValues.size());
}
String jobListing = foldingMaker.weaveFreemarker(template, values);
ComputeJob job = new ComputeJob();
String jobName =
config.get(JobGenerator.GENERATION_ID)
+ "_"
+ workflow.getName()
+ "_"
+ el.getName()
+ "_"
+ id;
job.setName(jobName);
job.setProtocol(protocol);
job.setComputeScript(jobListing);
computeJobs.add(job);
// fill containers for grid jobs to ensure correct data transfer
// and for cluster to generate submit script
// if (backend.equalsIgnoreCase(JobGenerator.GRID))
{
GridTransferContainer container = fillContainer(protocol, values);
pairJobTransfers.put(job.getName(), container);
}
// else if (backend.equalsIgnoreCase(JobGenerator.CLUSTER))
{
pairWEtoCJ.put(el, job);
pairCJtoWE.put(job, el);
}
logger.log(
Level.DEBUG, "----------------------------------------------------------------------");
logger.log(Level.DEBUG, el.getName());
logger.log(Level.DEBUG, jobListing);
logger.log(
Level.DEBUG, "----------------------------------------------------------------------");
}
}
return computeJobs;
}