Example #1
0
  /**
   * Run recipe with different frequencies. Submission should go through. Check frequency of the
   * launched oozie job
   */
  @Test(dataProvider = "frequencyGenerator")
  public void differentRecipeFrequenciesTest(String frequency) throws Exception {
    setUp(RecipeExecLocation.SourceCluster);
    LOGGER.info("Testing with frequency: " + frequency);
    String tblName = "myTable";
    recipeMerlin
        .withSourceDb(DB_NAME)
        .withSourceTable(tblName)
        .withFrequency(new Frequency(frequency));
    runSql(connection, "create table " + tblName + "(comment string)");
    final List<String> command = recipeMerlin.getSubmissionCommand();
    Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed.");
    LOGGER.info("Submission went through.");

    InstanceUtil.waitTillInstanceReachState(
        clusterOC, recipeMerlin.getName(), 1, CoordinatorAction.Status.RUNNING, EntityType.PROCESS);
    String filter = "name=FALCON_PROCESS_" + recipeMerlin.getName();
    List<BundleJob> bundleJobs = OozieUtil.getBundles(clusterOC, filter, 0, 10);
    List<String> bundleIds = OozieUtil.getBundleIds(bundleJobs);
    String bundleId = OozieUtil.getMaxId(bundleIds);
    List<CoordinatorJob> coords = clusterOC.getBundleJobInfo(bundleId).getCoordinators();
    List<String> cIds = new ArrayList<String>();
    for (CoordinatorJob coord : coords) {
      cIds.add(coord.getId());
    }
    String coordId = OozieUtil.getMinId(cIds);
    CoordinatorJob job = clusterOC.getCoordJobInfo(coordId);
    CoordinatorJob.Timeunit timeUnit = job.getTimeUnit();
    String freq = job.getFrequency();
    LOGGER.info("Frequency of running job: " + timeUnit + " " + freq);
    Assert.assertTrue(
        frequency.contains(timeUnit.name().toLowerCase().replace("_", ""))
            && frequency.contains(freq),
        "Running job has different frequency.");
  }
Example #2
0
  /**
   * Resolve basic entities using job Configuration.
   *
   * @param conf :Job configuration
   * @param appXml : Original job XML
   * @param coordJob : Coordinator job bean to be populated.
   * @return Resolved job XML element.
   * @throws CoordinatorJobException thrown if failed to resolve basic entities
   * @throws Exception thrown if failed to resolve basic entities
   */
  @SuppressWarnings("unchecked")
  protected Element resolveInitial(Configuration conf, String appXml, CoordinatorJobBean coordJob)
      throws CoordinatorJobException, Exception {
    Element eAppXml = XmlUtils.parseXml(appXml);
    // job's main attributes
    // frequency
    String val = resolveAttribute("frequency", eAppXml, evalFreq);
    int ival = ParamChecker.checkInteger(val, "frequency");
    ParamChecker.checkGTZero(ival, "frequency");
    coordJob.setFrequency(ival);
    TimeUnit tmp =
        (evalFreq.getVariable("timeunit") == null)
            ? TimeUnit.MINUTE
            : ((TimeUnit) evalFreq.getVariable("timeunit"));
    addAnAttribute("freq_timeunit", eAppXml, tmp.toString());
    // TimeUnit
    coordJob.setTimeUnit(CoordinatorJob.Timeunit.valueOf(tmp.toString()));
    // End Of Duration
    tmp =
        evalFreq.getVariable("endOfDuration") == null
            ? TimeUnit.NONE
            : ((TimeUnit) evalFreq.getVariable("endOfDuration"));
    addAnAttribute("end_of_duration", eAppXml, tmp.toString());
    // coordJob.setEndOfDuration(tmp) // TODO: Add new attribute in Job bean

    // Application name
    if (this.coordName == null) {
      val = resolveAttribute("name", eAppXml, evalNofuncs);
      coordJob.setAppName(val);
    } else {
      // this coord job is created from bundle
      coordJob.setAppName(this.coordName);
    }

    // start time
    val = resolveAttribute("start", eAppXml, evalNofuncs);
    ParamChecker.checkUTC(val, "start");
    coordJob.setStartTime(DateUtils.parseDateUTC(val));
    // end time
    val = resolveAttribute("end", eAppXml, evalNofuncs);
    ParamChecker.checkUTC(val, "end");
    coordJob.setEndTime(DateUtils.parseDateUTC(val));
    // Time zone
    val = resolveAttribute("timezone", eAppXml, evalNofuncs);
    ParamChecker.checkTimeZone(val, "timezone");
    coordJob.setTimeZone(val);

    // controls
    val =
        resolveTagContents(
            "timeout", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
    if (val == "") {
      val = Services.get().getConf().get(CONF_DEFAULT_TIMEOUT_NORMAL);
    }

    ival = ParamChecker.checkInteger(val, "timeout");
    if (ival < 0 || ival > Services.get().getConf().getInt(CONF_DEFAULT_MAX_TIMEOUT, 129600)) {
      ival = Services.get().getConf().getInt(CONF_DEFAULT_MAX_TIMEOUT, 129600);
    }
    coordJob.setTimeout(ival);

    val =
        resolveTagContents(
            "concurrency", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
    if (val == null || val.isEmpty()) {
      val = Services.get().getConf().get(CONF_DEFAULT_CONCURRENCY, "-1");
    }
    ival = ParamChecker.checkInteger(val, "concurrency");
    coordJob.setConcurrency(ival);

    val =
        resolveTagContents(
            "throttle", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
    if (val == null || val.isEmpty()) {
      int defaultThrottle = Services.get().getConf().getInt(CONF_DEFAULT_THROTTLE, 12);
      ival = defaultThrottle;
    } else {
      ival = ParamChecker.checkInteger(val, "throttle");
    }
    int maxQueue = Services.get().getConf().getInt(CONF_QUEUE_SIZE, 10000);
    float factor = Services.get().getConf().getFloat(CONF_MAT_THROTTLING_FACTOR, 0.10f);
    int maxThrottle = (int) (maxQueue * factor);
    if (ival > maxThrottle || ival < 1) {
      ival = maxThrottle;
    }
    LOG.debug("max throttle " + ival);
    coordJob.setMatThrottling(ival);

    val =
        resolveTagContents(
            "execution", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
    if (val == "") {
      val = Execution.FIFO.toString();
    }
    coordJob.setExecution(Execution.valueOf(val));
    String[] acceptedVals = {
      Execution.LIFO.toString(), Execution.FIFO.toString(), Execution.LAST_ONLY.toString()
    };
    ParamChecker.isMember(val, acceptedVals, "execution");

    // datasets
    resolveTagContents(
        "include", eAppXml.getChild("datasets", eAppXml.getNamespace()), evalNofuncs);
    // for each data set
    resolveDataSets(eAppXml);
    HashMap<String, String> dataNameList = new HashMap<String, String>();
    resolveIOEvents(eAppXml, dataNameList);

    resolveTagContents(
        "app-path",
        eAppXml
            .getChild("action", eAppXml.getNamespace())
            .getChild("workflow", eAppXml.getNamespace()),
        evalNofuncs);
    // TODO: If action or workflow tag is missing, NullPointerException will
    // occur
    Element configElem =
        eAppXml
            .getChild("action", eAppXml.getNamespace())
            .getChild("workflow", eAppXml.getNamespace())
            .getChild("configuration", eAppXml.getNamespace());
    evalData =
        CoordELEvaluator.createELEvaluatorForDataEcho(conf, "coord-job-submit-data", dataNameList);
    if (configElem != null) {
      for (Element propElem :
          (List<Element>) configElem.getChildren("property", configElem.getNamespace())) {
        resolveTagContents("name", propElem, evalData);
        // Want to check the data-integrity but don't want to modify the
        // XML
        // for properties only
        Element tmpProp = (Element) propElem.clone();
        resolveTagContents("value", tmpProp, evalData);
      }
    }
    resolveSLA(eAppXml, coordJob);
    return eAppXml;
  }