コード例 #1
0
  // here we create macros listing
  private String makeMacros(
      GridTransferContainer container, String script, String type, String backend) {
    String text = "";
    if (type.equalsIgnoreCase("input")) {
      text += JobGenerator.SOURCE_SCRIPT + "\n";

      Vector<String> nameInputs = loader.findFlagValues(script, ModelLoader.FLAG_INPUTS);
      Hashtable<String, String> inputs = container.getInputs();

      for (String s : nameInputs) {
        String macroline = makeMacroLine(s, inputs, type, backend);
        text += macroline;
      }

      Vector<String> nameExes = loader.findFlagValues(script, ModelLoader.FLAG_EXES);
      Hashtable<String, String> exes = container.getExes();

      for (String s : nameExes) {
        String macroline = makeMacroLine(s, exes, JobGenerator.EXE, backend);
        text += macroline;
      }

    } else if (type.equalsIgnoreCase("output")) {
      text += "\n";

      Vector<String> nameOutputs = loader.findFlagValues(script, ModelLoader.FLAG_OUTPUTS);
      Hashtable<String, String> outputs = container.getOutputs();
      for (String s : nameOutputs) {
        String macroline = makeMacroLine(s, outputs, type, backend);
        text += macroline;
      }
    }
    return text;
  }
コード例 #2
0
  // todo: method contains few constants that should be moved to interface
  private void generateActualJobGrid(
      ComputeJob computeJob, Hashtable<String, String> config, int generationCount) {
    // create values hashtable to fill templates
    Hashtable<String, String> values = new Hashtable<String, String>();

    values.put("script_name", computeJob.getName());
    values.put("error_log", "err_" + computeJob.getName() + ".log");
    values.put("output_log", "out_" + computeJob.getName() + ".log");
    values.put("script_location", config.get(JobGenerator.BACK_END_DIR));
    values.put("node_name", computeJob.getName());
    // create jdl
    String jdlListing = weaveFreemarker(templateGridJDL, values);

    if (workflowHasDependencies) {
      String dagNodeListing = weaveFreemarker(templateGridDAGNode, values);
      dagScript += dagNodeListing + "\n";
    }

    // write jdl
    (new File(config.get(JobGenerator.OUTPUT_DIR))).mkdirs();
    writeToFile(
        config.get(JobGenerator.OUTPUT_DIR)
            + System.getProperty("file.separator")
            + computeJob.getName()
            + ".jdl",
        jdlListing);

    // create shell
    String shellListing = templateGridHeader;
    String initialScript = computeJob.getComputeScript();
    GridTransferContainer container = pairJobTransfers.get(computeJob.getName());

    // get log filename
    Hashtable<String, String> logs = container.getLogs();
    Enumeration logValues = logs.elements();
    String logName = (String) logValues.nextElement();
    String justLogName = giveJustName(logName);

    // set log name to computeJob, that will be used in grid handler
    computeJob.setLogFile("lfn://grid/" + logName);

    // generate downloading section (transfer inputs and executable)
    // and change job listing to execute in the grid
    Hashtable<String, String> inputs = container.getInputs();
    Enumeration actuals = inputs.elements();
    while (actuals.hasMoreElements()) {
      Hashtable<String, String> local = new Hashtable<String, String>();

      String actualName = (String) actuals.nextElement();
      String justName = giveJustName(actualName);

      local.put(JobGenerator.LFN_NAME, actualName);
      local.put(JobGenerator.INPUT, justName);
      local.put(JobGenerator.LOG, justLogName);

      String inputListing = weaveFreemarker(templateGridDownload, local);
      // escapes are required to avoid $ sign processing in String replaceAll method
      justName = "\\" + justName;
      initialScript = initialScript.replaceAll(actualName, justName);

      shellListing += inputListing;
    }

    Hashtable<String, String> exes = container.getExes();
    actuals = exes.elements();
    while (actuals.hasMoreElements()) {
      Hashtable<String, String> local = new Hashtable<String, String>();

      String actualName = (String) actuals.nextElement();
      String justName = giveJustName(actualName);

      local.put(JobGenerator.LFN_NAME, actualName);
      local.put(JobGenerator.INPUT, justName);
      local.put(JobGenerator.LOG, justLogName);

      String inputListing = weaveFreemarker(templateGridDownloadExe, local);

      logger.log(Level.DEBUG, "-----------");
      logger.log(Level.DEBUG, initialScript);
      logger.log(Level.DEBUG, "act " + actualName);
      logger.log(Level.DEBUG, "just " + justName);
      justName = "\\" + justName;
      initialScript = initialScript.replaceAll(actualName, justName);
      shellListing += inputListing;
    }

    shellListing += initialScript;

    // generate uploading section
    // and change job listing to execute in the grid

    String outputsString = "";

    Hashtable<String, String> outputs = container.getOutputs();
    actuals = outputs.elements();
    while (actuals.hasMoreElements()) {
      Hashtable<String, String> local = new Hashtable<String, String>();

      String actualName = (String) actuals.nextElement();
      String justName = giveJustName(actualName);

      // set output file to compute job, that will be used in grid handler
      computeJob.setOutputFile("lfn://grid/" + actualName);

      local.put(JobGenerator.LFN_NAME, actualName);
      local.put(JobGenerator.OUTPUT, justName);
      local.put(JobGenerator.LOG, justLogName);

      String outputListing = weaveFreemarker(templateGridUpload, local);
      justName = "\\" + justName;
      shellListing = shellListing.replaceAll(actualName, justName);

      // shellListing += outputListing;
      outputsString += outputListing;
    }

    shellListing += outputsString;

    // add upload log
    Hashtable<String, String> local = new Hashtable<String, String>();

    local.put(JobGenerator.LFN_NAME, logName);
    local.put(JobGenerator.OUTPUT, justLogName);
    local.put(JobGenerator.LOG, justLogName);

    String outputListing = weaveFreemarker(templateGridUploadLog, local);
    shellListing += outputListing;

    // write shell
    writeToFile(
        config.get(JobGenerator.OUTPUT_DIR)
            + System.getProperty("file.separator")
            + computeJob.getName()
            + ".sh",
        shellListing);

    // and computeJob to grid handler
    gridHandler.setComputeJob(computeJob);

    // write job dependencies to DAG
    WorkflowElement el = pairCJtoWE.get(computeJob);
    if (el.getPreviousSteps().size() > 0) {
      String jobName = computeJob.getName();
      Vector<String> dependencyNames = new Vector<String>();
      for (WorkflowElement wEl : el.getPreviousSteps()) {
        ComputeJob cJ = pairWEtoCJ.get(wEl);
        String strDependencyName = cJ.getName();
        dependencyNames.addElement(strDependencyName);
      }

      String strDependency = "";

      // more than one previous element
      if (dependencyNames.size() > 1) {
        for (String str : dependencyNames) {
          strDependency += str + ", ";
        }
        // cut last coma
        strDependency = strDependency.substring(0, strDependency.length() - 2);
        strDependency = "{ " + strDependency + " }";
      }
      // only one dependency element
      else {
        strDependency = dependencyNames.elementAt(0);
      }

      // format string and add to DAg
      String toAdd = "{ " + jobName + ", " + strDependency + " },\n";
      dagDependencies = toAdd + dagDependencies;
    }
  }
コード例 #3
0
  private void generateActualJobGridMacro(
      ComputeJob computeJob,
      Hashtable<String, String> config,
      String strMacrosInput,
      String strMacrosOutput) {
    Hashtable<String, String> values = new Hashtable<String, String>();

    values.put("script_name", computeJob.getName());
    values.put("error_log", "err_" + computeJob.getName() + ".log");
    values.put("output_log", "out_" + computeJob.getName() + ".log");
    values.put("script_location", config.get(JobGenerator.BACK_END_DIR));
    values.put("node_name", computeJob.getName());
    // create jdl
    String jdlListing = weaveFreemarker(templateGridJDL, values);

    // write jdl
    (new File(config.get(JobGenerator.OUTPUT_DIR))).mkdirs();
    writeToFile(
        config.get(JobGenerator.OUTPUT_DIR)
            + System.getProperty("file.separator")
            + computeJob.getName()
            + ".jdl",
        jdlListing);

    // create shell
    String shellListing = templateGridHeader;
    String initialScript = computeJob.getComputeScript();
    GridTransferContainer container = pairJobTransfers.get(computeJob.getName());

    // generate downloading section (transfer inputs and executable)
    // and change job listing to execute in the grid
    Hashtable<String, String> inputs = container.getInputs();
    Enumeration actuals = inputs.elements();
    while (actuals.hasMoreElements()) {
      String actualName = (String) actuals.nextElement();
      String justName = giveJustName(actualName);

      // escapes are required to avoid $ sign processing in String replaceAll method
      justName = "\\" + justName;

      // commented out - but I keep it if I need to sho somebody the problem of replacement with
      // wild cards
      //            System.out.println("actual " + actualName);
      //            System.out.println("just " + justName);
      //
      //            System.out.println("------------- before");
      //            System.out.println(initialScript);

      initialScript = initialScript.replaceAll(actualName, justName);

      //            System.out.println("------------- after");
      //            System.out.println(initialScript);

    }

    Hashtable<String, String> exes = container.getExes();
    actuals = exes.elements();
    while (actuals.hasMoreElements()) {
      String actualName = (String) actuals.nextElement();
      String justName = giveJustName(actualName);

      justName = "\\" + justName;
      initialScript = initialScript.replaceAll(actualName, justName);
    }
    shellListing += strMacrosInput;
    shellListing += initialScript;
    String outputsString = "";

    Hashtable<String, String> outputs = container.getOutputs();
    actuals = outputs.elements();
    while (actuals.hasMoreElements()) {
      String actualName = (String) actuals.nextElement();
      String justName = giveJustName(actualName);

      justName = "\\" + justName;
      shellListing = shellListing.replaceAll(actualName, justName);
    }

    shellListing += strMacrosOutput;

    // write shell
    writeToFile(
        config.get(JobGenerator.OUTPUT_DIR)
            + System.getProperty("file.separator")
            + computeJob.getName()
            + ".sh",
        shellListing);
  }