@Test
  public void execute_blocking_FAIL() throws KettleException {
    final long waitTime = 0;

    OozieJobExecutorConfig config = new OozieJobExecutorConfig();
    config.setOozieUrl(
        "http://localhost:11000/oozie"); // don't worry if it isn't running, we fake out our test
                                         // connection to it anyway
    config.setOozieWorkflowConfig("test-src/job.properties");
    config.setJobEntryName("name");
    config.setBlockingPollingInterval("1000");

    TestOozieJobExecutorJobEntry je = new TestOozieJobExecutorJobEntry(getFailingTest());

    je.setParentJob(new Job("test", null, null));
    je.setJobConfig(config);

    Result result = new Result();
    long start = System.currentTimeMillis();
    je.execute(result, 0);
    long end = System.currentTimeMillis();
    assertTrue(
        "Total runtime should be >= the wait time if we are blocking", (end - start) >= waitTime);

    Assert.assertEquals(1, result.getNrErrors());
    assertFalse(result.getResult());
  }
  @Test
  public void testNrErrorsSuccess() throws Exception {
    entry.limit = "5";
    entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL;
    entry.tablename = "table";

    Result res = entry.execute(new Result(), 0);

    assertTrue("Eval number of rows should be suceeded", res.getResult());
    assertEquals("Apparently there should no error", res.getNrErrors(), 0);

    // that should work regardless of old/new behavior flag
    entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y");

    res = entry.execute(new Result(), 0);

    assertTrue("Eval number of rows should be suceeded", res.getResult());
    assertEquals("Apparently there should no error", res.getNrErrors(), 0);
  }
  @Test
  public void testExecuteWithException() throws Exception {
    entry.arguments = new String[] {null};

    Result res = entry.execute(new Result(), 0);

    assertFalse("Entry should fail", res.getResult());
    assertEquals(
        "File with wrong name was specified. One error should be reported", 1, res.getNrErrors());
  }
  @Test
  public void testNrErrorsNoCustomSql() throws Exception {
    entry.limit = "5";
    entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL;
    entry.iscustomSQL = true;
    entry.customSQL = null;

    Result res = entry.execute(new Result(), 0);

    assertFalse("Eval number of rows should fail", res.getResult());
    assertEquals("Apparently there should be an error", res.getNrErrors(), 1);

    // that should work regardless of old/new behavior flag
    entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y");

    res = entry.execute(new Result(), 0);

    assertFalse("Eval number of rows should fail", res.getResult());
    assertEquals("Apparently there should be an error", res.getNrErrors(), 1);
  }
  @Test
  public void testSetNrErrorsNewBehaviorFalseResult() throws Exception {
    // this tests fix for PDI-10270
    entry.arguments = new String[] {"nonExistingFile.ext"};

    Result res = entry.execute(new Result(), 0);

    assertFalse("Entry should fail", res.getResult());
    assertEquals(
        "Files not found. Result is false. But... No of errors should be zero",
        0,
        res.getNrErrors());
  }
  @Test
  public void testNrErrorsFailureNewBehavior() throws Exception {
    entry.limit = "1";
    entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL;
    entry.tablename = "table";

    Result res = entry.execute(new Result(), 0);

    assertFalse("Eval number of rows should fail", res.getResult());
    assertEquals(
        "No errors should be reported in result object accoding to the new behavior",
        res.getNrErrors(),
        0);
  }
  @Test
  public void testSetNrErrorsOldBehaviorFalseResult() throws Exception {
    // this tests backward compatibility settings for PDI-10270
    entry.arguments = new String[] {"nonExistingFile1.ext", "nonExistingFile2.ext"};

    entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y");

    Result res = entry.execute(new Result(), 0);

    assertFalse("Entry should fail", res.getResult());
    assertEquals(
        "Files not found. Result is false. And... Number of errors should be the same as number of not found files",
        entry.arguments.length,
        res.getNrErrors());
  }
  @Test
  public void testNrErrorsFailureOldBehavior() throws Exception {
    entry.limit = "1";
    entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL;
    entry.tablename = "table";

    entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y");

    Result res = entry.execute(new Result(), 0);

    assertFalse("Eval number of rows should fail", res.getResult());
    assertEquals(
        "An error should be reported in result object accoding to the old behavior",
        res.getNrErrors(),
        1);
  }
Exemplo n.º 9
0
  private void executeShell(Result result, List<RowMetaAndData> cmdRows, String[] args) {
    FileObject fileObject = null;
    String realScript = null;
    FileObject tempFile = null;

    try {
      // What's the exact command?
      String[] base = null;
      List<String> cmds = new ArrayList<String>();

      if (log.isBasic()) {
        logBasic(BaseMessages.getString(PKG, "JobShell.RunningOn", Const.getOS()));
      }

      if (insertScript) {
        realScript = environmentSubstitute(script);
      } else {
        String realFilename = environmentSubstitute(getFilename());
        fileObject = KettleVFS.getFileObject(realFilename, this);
      }

      if (Const.getOS().equals("Windows 95")) {
        base = new String[] {"command.com", "/C"};
        if (insertScript) {
          tempFile =
              KettleVFS.createTempFile(
                  "kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
          fileObject = createTemporaryShellFile(tempFile, realScript);
        }
      } else if (Const.getOS().startsWith("Windows")) {
        base = new String[] {"cmd.exe", "/C"};
        if (insertScript) {
          tempFile =
              KettleVFS.createTempFile(
                  "kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
          fileObject = createTemporaryShellFile(tempFile, realScript);
        }
      } else {
        if (insertScript) {
          tempFile =
              KettleVFS.createTempFile(
                  "kettle", "shell", System.getProperty("java.io.tmpdir"), this);
          fileObject = createTemporaryShellFile(tempFile, realScript);
        }
        base = new String[] {KettleVFS.getFilename(fileObject)};
      }

      // Construct the arguments...
      if (argFromPrevious && cmdRows != null) {
        // Add the base command...
        for (int i = 0; i < base.length; i++) {
          cmds.add(base[i]);
        }

        if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
          // for windows all arguments including the command itself
          // need to be
          // included in 1 argument to cmd/command.

          StringBuffer cmdline = new StringBuffer(300);

          cmdline.append('"');
          cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
          // Add the arguments from previous results...
          for (int i = 0; i < cmdRows.size(); i++) {
            // Normally just one row, but once in a while to remain compatible we have multiple.

            RowMetaAndData r = cmdRows.get(i);
            for (int j = 0; j < r.size(); j++) {
              cmdline.append(' ');
              cmdline.append(Const.optionallyQuoteStringByOS(r.getString(j, null)));
            }
          }
          cmdline.append('"');
          cmds.add(cmdline.toString());
        } else {
          // Add the arguments from previous results...
          for (int i = 0; i < cmdRows.size(); i++) {
            // Normally just one row, but once in a while to remain compatible we have multiple.

            RowMetaAndData r = cmdRows.get(i);
            for (int j = 0; j < r.size(); j++) {
              cmds.add(Const.optionallyQuoteStringByOS(r.getString(j, null)));
            }
          }
        }
      } else if (args != null) {
        // Add the base command...
        for (int i = 0; i < base.length; i++) {
          cmds.add(base[i]);
        }

        if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
          // for windows all arguments including the command itself
          // need to be
          // included in 1 argument to cmd/command.

          StringBuffer cmdline = new StringBuffer(300);

          cmdline.append('"');
          cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));

          for (int i = 0; i < args.length; i++) {
            cmdline.append(' ');
            cmdline.append(Const.optionallyQuoteStringByOS(args[i]));
          }
          cmdline.append('"');
          cmds.add(cmdline.toString());
        } else {
          for (int i = 0; i < args.length; i++) {
            cmds.add(args[i]);
          }
        }
      }

      StringBuffer command = new StringBuffer();

      Iterator<String> it = cmds.iterator();
      boolean first = true;
      while (it.hasNext()) {
        if (!first) {
          command.append(' ');
        } else {
          first = false;
        }
        command.append(it.next());
      }
      if (log.isBasic()) {
        logBasic(BaseMessages.getString(PKG, "JobShell.ExecCommand", command.toString()));
      }

      // Build the environment variable list...
      ProcessBuilder procBuilder = new ProcessBuilder(cmds);
      Map<String, String> env = procBuilder.environment();
      String[] variables = listVariables();
      for (int i = 0; i < variables.length; i++) {
        env.put(variables[i], getVariable(variables[i]));
      }

      if (getWorkDirectory() != null && !Const.isEmpty(Const.rtrim(getWorkDirectory()))) {
        String vfsFilename = environmentSubstitute(getWorkDirectory());
        File file = new File(KettleVFS.getFilename(KettleVFS.getFileObject(vfsFilename, this)));
        procBuilder.directory(file);
      }
      Process proc = procBuilder.start();

      // any error message?
      StreamLogger errorLogger = new StreamLogger(log, proc.getErrorStream(), "(stderr)", true);

      // any output?
      StreamLogger outputLogger = new StreamLogger(log, proc.getInputStream(), "(stdout)");

      // kick them off
      Thread errorLoggerThread = new Thread(errorLogger);
      errorLoggerThread.start();
      Thread outputLoggerThread = new Thread(outputLogger);
      outputLoggerThread.start();

      proc.waitFor();
      if (log.isDetailed()) {
        logDetailed(BaseMessages.getString(PKG, "JobShell.CommandFinished", command.toString()));
      }

      // What's the exit status?
      result.setExitStatus(proc.exitValue());
      if (result.getExitStatus() != 0) {
        if (log.isDetailed()) {
          logDetailed(
              BaseMessages.getString(
                  PKG,
                  "JobShell.ExitStatus",
                  environmentSubstitute(getFilename()),
                  "" + result.getExitStatus()));
        }

        result.setNrErrors(1);
      }

      // wait until loggers read all data from stdout and stderr
      errorLoggerThread.join();
      outputLoggerThread.join();

      // close the streams
      // otherwise you get "Too many open files, java.io.IOException" after a lot of iterations
      proc.getErrorStream().close();
      proc.getOutputStream().close();

    } catch (IOException ioe) {
      logError(
          BaseMessages.getString(
              PKG,
              "JobShell.ErrorRunningShell",
              environmentSubstitute(getFilename()),
              ioe.toString()),
          ioe);
      result.setNrErrors(1);
    } catch (InterruptedException ie) {
      logError(
          BaseMessages.getString(
              PKG, "JobShell.Shellinterupted", environmentSubstitute(getFilename()), ie.toString()),
          ie);
      result.setNrErrors(1);
    } catch (Exception e) {
      logError(
          BaseMessages.getString(
              PKG, "JobShell.UnexpectedError", environmentSubstitute(getFilename()), e.toString()),
          e);
      result.setNrErrors(1);
    } finally {
      // If we created a temporary file, remove it...
      //
      if (tempFile != null) {
        try {
          tempFile.delete();
        } catch (Exception e) {
          BaseMessages.getString(
              PKG, "JobShell.UnexpectedError", tempFile.toString(), e.toString());
        }
      }
    }

    if (result.getNrErrors() > 0) {
      result.setResult(false);
    } else {
      result.setResult(true);
    }
  }
Exemplo n.º 10
0
  public Result execute(Result result, int nr) throws KettleException {
    FileLoggingEventListener loggingEventListener = null;
    LogLevel shellLogLevel = parentJob.getLogLevel();
    if (setLogfile) {
      String realLogFilename = environmentSubstitute(getLogFilename());
      // We need to check here the log filename
      // if we do not have one, we must fail
      if (Const.isEmpty(realLogFilename)) {
        logError(BaseMessages.getString(PKG, "JobEntryShell.Exception.LogFilenameMissing"));
        result.setNrErrors(1);
        result.setResult(false);
        return result;
      }

      try {
        loggingEventListener =
            new FileLoggingEventListener(getLogChannelId(), realLogFilename, setAppendLogfile);
        KettleLogStore.getAppender().addLoggingEventListener(loggingEventListener);
      } catch (KettleException e) {
        logError(
            BaseMessages.getString(
                PKG, "JobEntryShell.Error.UnableopenAppenderFile", getLogFilename(), e.toString()));
        logError(Const.getStackTracker(e));
        result.setNrErrors(1);
        result.setResult(false);
        return result;
      }
      shellLogLevel = logFileLevel;
    }

    log.setLogLevel(shellLogLevel);

    result.setEntryNr(nr);

    // "Translate" the arguments for later
    String[] substArgs = null;
    if (arguments != null) {
      substArgs = new String[arguments.length];
      for (int idx = 0; idx < arguments.length; idx++) {
        substArgs[idx] = environmentSubstitute(arguments[idx]);
      }
    }

    int iteration = 0;
    String[] args = substArgs;
    RowMetaAndData resultRow = null;
    boolean first = true;
    List<RowMetaAndData> rows = result.getRows();

    if (log.isDetailed()) {
      logDetailed(
          BaseMessages.getString(
              PKG, "JobEntryShell.Log.FoundPreviousRows", "" + (rows != null ? rows.size() : 0)));
    }

    while ((first && !execPerRow)
        || (execPerRow && rows != null && iteration < rows.size() && result.getNrErrors() == 0)) {
      first = false;
      if (rows != null && execPerRow) {
        resultRow = rows.get(iteration);
      } else {
        resultRow = null;
      }

      List<RowMetaAndData> cmdRows = null;

      if (execPerRow) {
        // Execute for each input row

        if (argFromPrevious) {
          // Copy the input row to the (command line) arguments

          if (resultRow != null) {
            args = new String[resultRow.size()];
            for (int i = 0; i < resultRow.size(); i++) {
              args[i] = resultRow.getString(i, null);
            }
          }
        } else {
          // Just pass a single row
          List<RowMetaAndData> newList = new ArrayList<RowMetaAndData>();
          newList.add(resultRow);
          cmdRows = newList;
        }
      } else {
        if (argFromPrevious) {
          // Only put the first Row on the arguments
          args = null;
          if (resultRow != null) {
            args = new String[resultRow.size()];
            for (int i = 0; i < resultRow.size(); i++) {
              args[i] = resultRow.getString(i, null);
            }
          } else {
            cmdRows = rows;
          }
        } else {
          // Keep it as it was...
          cmdRows = rows;
        }
      }

      executeShell(result, cmdRows, args);

      iteration++;
    }

    if (setLogfile) {
      if (loggingEventListener != null) {
        KettleLogStore.getAppender().removeLoggingEventListener(loggingEventListener);
        loggingEventListener.close();

        ResultFile resultFile =
            new ResultFile(
                ResultFile.FILE_TYPE_LOG,
                loggingEventListener.getFile(),
                parentJob.getJobname(),
                getName());
        result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
      }
    }

    return result;
  }