コード例 #1
0
 private boolean hasDependencies(Workflow workflow) {
   Collection<WorkflowElement> workflowElements = workflow.getWorkflowWorkflowElementCollection();
   for (WorkflowElement el : workflowElements) {
     if (el.getPreviousSteps().size() > 0) return true;
   }
   return false;
 }
コード例 #2
0
  public Vector<ComputeJob> generateComputeJobsFoldedWorksheet(
      Workflow workflow, List<Tuple> f, String backend) {
    // create the table with targets, which is equal to worksheet if there are no targets
    List<Hashtable> table = null;

    // check if workflow elements have dependencies
    workflowHasDependencies = hasDependencies(workflow);

    // remove unused parameters from the worksheet
    List<Hashtable> worksheet = foldingMaker.transformToTable(f);
    foldingMaker.setWorkflow(workflow);
    worksheet =
        foldingMaker.removeUnused(
            worksheet, (List<ComputeParameter>) workflow.getWorkflowComputeParameterCollection());

    // result jobs
    Vector<ComputeJob> computeJobs = new Vector<ComputeJob>();

    // if (backend.equalsIgnoreCase(JobGenerator.GRID))
    pairJobTransfers = new Hashtable<String, GridTransferContainer>();

    pairWEtoCJ = new Hashtable<WorkflowElement, ComputeJob>();
    pairCJtoWE = new Hashtable<ComputeJob, WorkflowElement>();

    if (backend.equalsIgnoreCase(JobGenerator.CLUSTER)) {
      submitScript = "";
    } else if (backend.equalsIgnoreCase(JobGenerator.GRID) && workflowHasDependencies) {
      dagScript = "Type=\"dag\";\n\n";
      dagDependencies = "";
    }

    Collection<ComputeParameter> parameters = workflow.getWorkflowComputeParameterCollection();
    Collection<WorkflowElement> workflowElements = workflow.getWorkflowWorkflowElementCollection();
    for (WorkflowElement el : workflowElements) {

      ComputeProtocol protocol = (ComputeProtocol) el.getProtocol();
      String template = protocol.getScriptTemplate();

      // chack if we have any targets
      List<ComputeParameter> targets = foldingMaker.findTargets(protocol.getScriptTemplate());
      if (targets != null) {
        table = foldingMaker.fold(targets, worksheet);
      } else table = worksheet;

      // here, we prapare some information about folded table
      // in particular, we find what parameters are Martijn's folded constants
      foldingParser.evaluateTable(table);

      // we set all parameters to foldered parser, which are used for reducing foldered constants
      // this comment does give any clue to what is going on :)
      foldingParser.setParametersList(parameters);

      // now we start to use foldered worksheet
      // all our parameters, that depend on foldered worksheet, will become lists as well, that
      // upset me a lot :(
      for (int i = 0; i < table.size(); i++) {
        // because Hashtable does not allow null keys or values used for weaving
        Hashtable<String, Object> values = new Hashtable<String, Object>();

        // parameters which are templates and do directly depend on worksheet values
        Vector<ComputeParameter> complexParameters = new Vector<ComputeParameter>();

        // hashtable with simple values, we need it for initial weaving
        Hashtable<String, String> simpleValues = new Hashtable<String, String>();

        // job naming
        String id = "id";
        Hashtable<String, Object> line = table.get(i);

        if (targets != null) {
          // use targets to create name
          Enumeration ekeys = line.keys();
          while (ekeys.hasMoreElements()) {
            String ekey = (String) ekeys.nextElement();
            if (isTarget(ekey, targets)) {
              Object eValues = line.get(ekey);
              values.put(ekey, eValues);
              String vvv = eValues.toString();
              vvv = vvv.replaceAll(" ", "_");
              id += "_" + vvv;
            }
          }

        } else {
          // use the whole line to create name
          Enumeration ekeys = line.keys();
          while (ekeys.hasMoreElements()) {
            String ekey = (String) ekeys.nextElement();
            Object eValues = line.get(ekey);
            values.put(ekey, eValues);
            String vvv = eValues.toString();
            vvv = vvv.replaceAll(" ", "_");
            id += "_" + vvv;
          }
        }

        for (ComputeParameter parameter : parameters) {
          if (parameter.getDefaultValue() != null) {
            if (parameter.getDefaultValue().contains("${")) {
              complexParameters.addElement(parameter);
            } else {
              values.put(parameter.getName(), parameter.getDefaultValue());
              simpleValues.put(parameter.getName(), parameter.getDefaultValue());
            }
          }
        }
        logger.log(Level.DEBUG, "simple parameters before: " + values.size());

        // we transform complex parameters to unweaved values, because unweaved value can folded
        Hashtable<String, Object> unweavedValues = new Hashtable<String, Object>();

        for (ComputeParameter par : complexParameters) {
          Pair<String, Object> value = processDependentParameter(par, line, simpleValues);
          if (foldingParser.isValueSimple(value)) {
            values.put(par.getName(), value.getB());
          } else {
            unweavedValues.put(par.getName(), value.getB());
          }
        }

        logger.log(
            Level.DEBUG,
            "simple parameters after "
                + values.size()
                + "  parameters to weave: "
                + complexParameters.size());

        Vector<String> vecToRemove = new Vector<String>();

        int weavingCount = 0;
        logger.log(Level.DEBUG, "loop " + weavingCount + " -> " + unweavedValues.size());
        // in a loop weave complex parameters with values
        while (unweavedValues.size() > 0 && weavingCount < 100) {
          Enumeration unkeys = unweavedValues.keys();
          while (unkeys.hasMoreElements()) {
            String unkey = (String) unkeys.nextElement();
            Object eValue = unweavedValues.get(unkey);

            Pair<String, Object> value = null;

            if (eValue instanceof Collection<?>) {
              List<String> unweavedLines = (List<String>) eValue;
              value = processUnweavedCollection(unkey, unweavedLines, values);
            } else {
              String unweavedLine = (String) eValue;
              value = processUnweavedLine(unkey, unweavedLine, values, 0);
            }

            if (foldingParser.isValueSimple(value)) {
              values.put(value.getA(), value.getB());
              vecToRemove.add(unkey);
            } else unweavedValues.put(value.getA(), value.getB());
          }

          for (String str : vecToRemove) unweavedValues.remove(str);
          weavingCount++;

          logger.log(
              Level.DEBUG,
              "loop/parameters to weave " + weavingCount + " -> " + unweavedValues.size());
        }

        String jobListing = foldingMaker.weaveFreemarker(template, values);

        ComputeJob job = new ComputeJob();

        String jobName =
            config.get(JobGenerator.GENERATION_ID)
                + "_"
                + workflow.getName()
                + "_"
                + el.getName()
                + "_"
                + id;

        job.setName(jobName);
        job.setProtocol(protocol);
        job.setComputeScript(jobListing);
        computeJobs.add(job);

        // fill containers for grid jobs to ensure correct data transfer
        // and for cluster to generate submit script
        // if (backend.equalsIgnoreCase(JobGenerator.GRID))
        {
          GridTransferContainer container = fillContainer(protocol, values);
          pairJobTransfers.put(job.getName(), container);
        }
        // else if (backend.equalsIgnoreCase(JobGenerator.CLUSTER))
        {
          pairWEtoCJ.put(el, job);
          pairCJtoWE.put(job, el);
        }
        logger.log(
            Level.DEBUG, "----------------------------------------------------------------------");
        logger.log(Level.DEBUG, el.getName());
        logger.log(Level.DEBUG, jobListing);
        logger.log(
            Level.DEBUG, "----------------------------------------------------------------------");
      }
    }
    return computeJobs;
  }