private Pair<String, Object> processUnweavedLine( String unkey, String unweavedLine, Hashtable<String, Object> values, int i) { Pair<String, Object> pair = new Pair<String, Object>(); Hashtable<String, String> hashtable = prepareSimpleValues(values, i); String value = foldingParser.doByHand(unweavedLine, hashtable); pair.setA(unkey); pair.setB(value); return pair; }
// here, we also identify what parameters should be foldered private Pair<String, Object> processDependentParameter( ComputeParameter par, Hashtable<String, Object> line, Hashtable<String, String> simpleValues) { Pair<String, Object> pair = new Pair<String, Object>(); pair.setA(par.getName()); int lineFolderedSize = foldingParser.getFolderedLineSize(line); String parTemplate = par.getDefaultValue(); foldingParser.setNotList(); foldingParser.checkIsList(parTemplate); boolean isList = foldingParser.getIsList(); if (lineFolderedSize > 1 && isList) { List<String> values = new ArrayList<String>(); for (int i = 0; i < lineFolderedSize; i++) { String value = foldingParser.parseTemplateLineByHand(parTemplate, line, i, simpleValues); values.add(value); } pair.setB(values); } else { String value = foldingParser.parseTemplateLineByHand(parTemplate, line, 0, simpleValues); pair.setB(value); } return pair; }
public Vector<ComputeJob> generateComputeJobsFoldedWorksheet( Workflow workflow, List<Tuple> f, String backend) { // create the table with targets, which is equal to worksheet if there are no targets List<Hashtable> table = null; // check if workflow elements have dependencies workflowHasDependencies = hasDependencies(workflow); // remove unused parameters from the worksheet List<Hashtable> worksheet = foldingMaker.transformToTable(f); foldingMaker.setWorkflow(workflow); worksheet = foldingMaker.removeUnused( worksheet, (List<ComputeParameter>) workflow.getWorkflowComputeParameterCollection()); // result jobs Vector<ComputeJob> computeJobs = new Vector<ComputeJob>(); // if (backend.equalsIgnoreCase(JobGenerator.GRID)) pairJobTransfers = new Hashtable<String, GridTransferContainer>(); pairWEtoCJ = new Hashtable<WorkflowElement, ComputeJob>(); pairCJtoWE = new Hashtable<ComputeJob, WorkflowElement>(); if (backend.equalsIgnoreCase(JobGenerator.CLUSTER)) { submitScript = ""; } else if (backend.equalsIgnoreCase(JobGenerator.GRID) && workflowHasDependencies) { dagScript = "Type=\"dag\";\n\n"; dagDependencies = ""; } Collection<ComputeParameter> parameters = workflow.getWorkflowComputeParameterCollection(); Collection<WorkflowElement> workflowElements = workflow.getWorkflowWorkflowElementCollection(); for (WorkflowElement el : workflowElements) { ComputeProtocol protocol = (ComputeProtocol) el.getProtocol(); String template = protocol.getScriptTemplate(); // chack if we have any targets List<ComputeParameter> targets = foldingMaker.findTargets(protocol.getScriptTemplate()); if (targets != null) { table = foldingMaker.fold(targets, worksheet); } else table = worksheet; // here, we prapare some information about folded table // in particular, we find what parameters are Martijn's folded constants foldingParser.evaluateTable(table); // we set all parameters to foldered parser, which are used for reducing foldered constants // this comment does give any clue to what is going on :) foldingParser.setParametersList(parameters); // now we start to use foldered worksheet // all our parameters, that depend on foldered worksheet, will become lists as well, that // upset me a lot :( for (int i = 0; i < table.size(); i++) { // because Hashtable does not allow null keys or values used for weaving Hashtable<String, Object> values = new Hashtable<String, Object>(); // parameters which are templates and do directly depend on worksheet values Vector<ComputeParameter> complexParameters = new Vector<ComputeParameter>(); // hashtable with simple values, we need it for initial weaving Hashtable<String, String> simpleValues = new Hashtable<String, String>(); // job naming String id = "id"; Hashtable<String, Object> line = table.get(i); if (targets != null) { // use targets to create name Enumeration ekeys = line.keys(); while (ekeys.hasMoreElements()) { String ekey = (String) ekeys.nextElement(); if (isTarget(ekey, targets)) { Object eValues = line.get(ekey); values.put(ekey, eValues); String vvv = eValues.toString(); vvv = vvv.replaceAll(" ", "_"); id += "_" + vvv; } } } else { // use the whole line to create name Enumeration ekeys = line.keys(); while (ekeys.hasMoreElements()) { String ekey = (String) ekeys.nextElement(); Object eValues = line.get(ekey); values.put(ekey, eValues); String vvv = eValues.toString(); vvv = vvv.replaceAll(" ", "_"); id += "_" + vvv; } } for (ComputeParameter parameter : parameters) { if (parameter.getDefaultValue() != null) { if (parameter.getDefaultValue().contains("${")) { complexParameters.addElement(parameter); } else { values.put(parameter.getName(), parameter.getDefaultValue()); simpleValues.put(parameter.getName(), parameter.getDefaultValue()); } } } logger.log(Level.DEBUG, "simple parameters before: " + values.size()); // we transform complex parameters to unweaved values, because unweaved value can folded Hashtable<String, Object> unweavedValues = new Hashtable<String, Object>(); for (ComputeParameter par : complexParameters) { Pair<String, Object> value = processDependentParameter(par, line, simpleValues); if (foldingParser.isValueSimple(value)) { values.put(par.getName(), value.getB()); } else { unweavedValues.put(par.getName(), value.getB()); } } logger.log( Level.DEBUG, "simple parameters after " + values.size() + " parameters to weave: " + complexParameters.size()); Vector<String> vecToRemove = new Vector<String>(); int weavingCount = 0; logger.log(Level.DEBUG, "loop " + weavingCount + " -> " + unweavedValues.size()); // in a loop weave complex parameters with values while (unweavedValues.size() > 0 && weavingCount < 100) { Enumeration unkeys = unweavedValues.keys(); while (unkeys.hasMoreElements()) { String unkey = (String) unkeys.nextElement(); Object eValue = unweavedValues.get(unkey); Pair<String, Object> value = null; if (eValue instanceof Collection<?>) { List<String> unweavedLines = (List<String>) eValue; value = processUnweavedCollection(unkey, unweavedLines, values); } else { String unweavedLine = (String) eValue; value = processUnweavedLine(unkey, unweavedLine, values, 0); } if (foldingParser.isValueSimple(value)) { values.put(value.getA(), value.getB()); vecToRemove.add(unkey); } else unweavedValues.put(value.getA(), value.getB()); } for (String str : vecToRemove) unweavedValues.remove(str); weavingCount++; logger.log( Level.DEBUG, "loop/parameters to weave " + weavingCount + " -> " + unweavedValues.size()); } String jobListing = foldingMaker.weaveFreemarker(template, values); ComputeJob job = new ComputeJob(); String jobName = config.get(JobGenerator.GENERATION_ID) + "_" + workflow.getName() + "_" + el.getName() + "_" + id; job.setName(jobName); job.setProtocol(protocol); job.setComputeScript(jobListing); computeJobs.add(job); // fill containers for grid jobs to ensure correct data transfer // and for cluster to generate submit script // if (backend.equalsIgnoreCase(JobGenerator.GRID)) { GridTransferContainer container = fillContainer(protocol, values); pairJobTransfers.put(job.getName(), container); } // else if (backend.equalsIgnoreCase(JobGenerator.CLUSTER)) { pairWEtoCJ.put(el, job); pairCJtoWE.put(job, el); } logger.log( Level.DEBUG, "----------------------------------------------------------------------"); logger.log(Level.DEBUG, el.getName()); logger.log(Level.DEBUG, jobListing); logger.log( Level.DEBUG, "----------------------------------------------------------------------"); } } return computeJobs; }