예제 #1
0
  /**
   * Resolve datasets using job configuration.
   *
   * @param dsElems : Data set XML element.
   * @throws CoordinatorJobException thrown if failed to resolve datasets
   */
  private void resolveDataSets(List<Element> dsElems) throws CoordinatorJobException {
    for (Element dsElem : dsElems) {
      // Setting up default TimeUnit and EndOFDuraion
      evalFreq.setVariable("timeunit", TimeUnit.MINUTE);
      evalFreq.setVariable("endOfDuration", TimeUnit.NONE);

      String val = resolveAttribute("frequency", dsElem, evalFreq);
      int ival = ParamChecker.checkInteger(val, "frequency");
      ParamChecker.checkGTZero(ival, "frequency");
      addAnAttribute(
          "freq_timeunit",
          dsElem,
          evalFreq.getVariable("timeunit") == null
              ? TimeUnit.MINUTE.toString()
              : ((TimeUnit) evalFreq.getVariable("timeunit")).toString());
      addAnAttribute(
          "end_of_duration",
          dsElem,
          evalFreq.getVariable("endOfDuration") == null
              ? TimeUnit.NONE.toString()
              : ((TimeUnit) evalFreq.getVariable("endOfDuration")).toString());
      val = resolveAttribute("initial-instance", dsElem, evalNofuncs);
      ParamChecker.checkUTC(val, "initial-instance");
      val = resolveAttribute("timezone", dsElem, evalNofuncs);
      ParamChecker.checkTimeZone(val, "timezone");
      resolveTagContents("uri-template", dsElem, evalNofuncs);
      resolveTagContents("done-flag", dsElem, evalNofuncs);
    }
  }
예제 #2
0
  /**
   * Resolve basic entities using job Configuration.
   *
   * @param conf :Job configuration
   * @param appXml : Original job XML
   * @param coordJob : Coordinator job bean to be populated.
   * @return Resolved job XML element.
   * @throws CoordinatorJobException thrown if failed to resolve basic entities
   * @throws Exception thrown if failed to resolve basic entities
   */
  @SuppressWarnings("unchecked")
  protected Element resolveInitial(Configuration conf, String appXml, CoordinatorJobBean coordJob)
      throws CoordinatorJobException, Exception {
    Element eAppXml = XmlUtils.parseXml(appXml);
    // job's main attributes
    // frequency
    String val = resolveAttribute("frequency", eAppXml, evalFreq);
    int ival = ParamChecker.checkInteger(val, "frequency");
    ParamChecker.checkGTZero(ival, "frequency");
    coordJob.setFrequency(ival);
    TimeUnit tmp =
        (evalFreq.getVariable("timeunit") == null)
            ? TimeUnit.MINUTE
            : ((TimeUnit) evalFreq.getVariable("timeunit"));
    addAnAttribute("freq_timeunit", eAppXml, tmp.toString());
    // TimeUnit
    coordJob.setTimeUnit(CoordinatorJob.Timeunit.valueOf(tmp.toString()));
    // End Of Duration
    tmp =
        evalFreq.getVariable("endOfDuration") == null
            ? TimeUnit.NONE
            : ((TimeUnit) evalFreq.getVariable("endOfDuration"));
    addAnAttribute("end_of_duration", eAppXml, tmp.toString());
    // coordJob.setEndOfDuration(tmp) // TODO: Add new attribute in Job bean

    // Application name
    if (this.coordName == null) {
      val = resolveAttribute("name", eAppXml, evalNofuncs);
      coordJob.setAppName(val);
    } else {
      // this coord job is created from bundle
      coordJob.setAppName(this.coordName);
    }

    // start time
    val = resolveAttribute("start", eAppXml, evalNofuncs);
    ParamChecker.checkUTC(val, "start");
    coordJob.setStartTime(DateUtils.parseDateUTC(val));
    // end time
    val = resolveAttribute("end", eAppXml, evalNofuncs);
    ParamChecker.checkUTC(val, "end");
    coordJob.setEndTime(DateUtils.parseDateUTC(val));
    // Time zone
    val = resolveAttribute("timezone", eAppXml, evalNofuncs);
    ParamChecker.checkTimeZone(val, "timezone");
    coordJob.setTimeZone(val);

    // controls
    val =
        resolveTagContents(
            "timeout", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
    if (val == "") {
      val = Services.get().getConf().get(CONF_DEFAULT_TIMEOUT_NORMAL);
    }

    ival = ParamChecker.checkInteger(val, "timeout");
    if (ival < 0 || ival > Services.get().getConf().getInt(CONF_DEFAULT_MAX_TIMEOUT, 129600)) {
      ival = Services.get().getConf().getInt(CONF_DEFAULT_MAX_TIMEOUT, 129600);
    }
    coordJob.setTimeout(ival);

    val =
        resolveTagContents(
            "concurrency", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
    if (val == null || val.isEmpty()) {
      val = Services.get().getConf().get(CONF_DEFAULT_CONCURRENCY, "-1");
    }
    ival = ParamChecker.checkInteger(val, "concurrency");
    coordJob.setConcurrency(ival);

    val =
        resolveTagContents(
            "throttle", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
    if (val == null || val.isEmpty()) {
      int defaultThrottle = Services.get().getConf().getInt(CONF_DEFAULT_THROTTLE, 12);
      ival = defaultThrottle;
    } else {
      ival = ParamChecker.checkInteger(val, "throttle");
    }
    int maxQueue = Services.get().getConf().getInt(CONF_QUEUE_SIZE, 10000);
    float factor = Services.get().getConf().getFloat(CONF_MAT_THROTTLING_FACTOR, 0.10f);
    int maxThrottle = (int) (maxQueue * factor);
    if (ival > maxThrottle || ival < 1) {
      ival = maxThrottle;
    }
    LOG.debug("max throttle " + ival);
    coordJob.setMatThrottling(ival);

    val =
        resolveTagContents(
            "execution", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
    if (val == "") {
      val = Execution.FIFO.toString();
    }
    coordJob.setExecution(Execution.valueOf(val));
    String[] acceptedVals = {
      Execution.LIFO.toString(), Execution.FIFO.toString(), Execution.LAST_ONLY.toString()
    };
    ParamChecker.isMember(val, acceptedVals, "execution");

    // datasets
    resolveTagContents(
        "include", eAppXml.getChild("datasets", eAppXml.getNamespace()), evalNofuncs);
    // for each data set
    resolveDataSets(eAppXml);
    HashMap<String, String> dataNameList = new HashMap<String, String>();
    resolveIOEvents(eAppXml, dataNameList);

    resolveTagContents(
        "app-path",
        eAppXml
            .getChild("action", eAppXml.getNamespace())
            .getChild("workflow", eAppXml.getNamespace()),
        evalNofuncs);
    // TODO: If action or workflow tag is missing, NullPointerException will
    // occur
    Element configElem =
        eAppXml
            .getChild("action", eAppXml.getNamespace())
            .getChild("workflow", eAppXml.getNamespace())
            .getChild("configuration", eAppXml.getNamespace());
    evalData =
        CoordELEvaluator.createELEvaluatorForDataEcho(conf, "coord-job-submit-data", dataNameList);
    if (configElem != null) {
      for (Element propElem :
          (List<Element>) configElem.getChildren("property", configElem.getNamespace())) {
        resolveTagContents("name", propElem, evalData);
        // Want to check the data-integrity but don't want to modify the
        // XML
        // for properties only
        Element tmpProp = (Element) propElem.clone();
        resolveTagContents("value", tmpProp, evalData);
      }
    }
    resolveSLA(eAppXml, coordJob);
    return eAppXml;
  }