Пример #1
0
  /** Check the 2 lists comparing the rows in order. If they are not the same fail the test. */
  public void checkRows(List<RowMetaAndData> rows1, List<RowMetaAndData> rows2) {
    int idx = 1;
    if (rows1.size() != rows2.size()) {
      fail("Number of rows is not the same: " + rows1.size() + " and " + rows2.size());
    }
    Iterator<RowMetaAndData> it1 = rows1.iterator();
    Iterator<RowMetaAndData> it2 = rows2.iterator();

    while (it1.hasNext() && it2.hasNext()) {
      RowMetaAndData rm1 = it1.next();
      RowMetaAndData rm2 = it2.next();

      Object[] r1 = rm1.getData();
      Object[] r2 = rm2.getData();

      if (rm1.size() != rm2.size()) {
        fail("row nr " + idx + " is not equal");
      }
      int fields[] = new int[rm1.size()];
      for (int ydx = 0; ydx < rm1.size(); ydx++) {
        fields[ydx] = ydx;
      }
      try {
        if (rm1.getRowMeta().compare(r1, r2, fields) != 0) {
          fail("row nr " + idx + " is not equal");
        }
      } catch (KettleValueException e) {
        fail("row nr " + idx + " is not equal");
      }

      idx++;
    }
  }
Пример #2
0
  public void test01_BasicSelectFrom() throws Exception {
    KettleEnvironment.init();

    String sqlQuery = "SELECT * FROM Service";

    SqlTransExecutor executor = new SqlTransExecutor(sqlQuery, getServices());

    final List<RowMetaAndData> rows = new ArrayList<RowMetaAndData>();

    // print the eventual result rows...
    //
    executor.executeQuery(
        new RowAdapter() {
          @Override
          public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
              throws KettleStepException {
            rows.add(new RowMetaAndData(rowMeta, row));
          }
        });

    // Now the generated transformation is waiting for input so we
    // can start the service transformation
    //
    executor.waitUntilFinished();

    assertEquals(8, rows.size());
    RowMetaAndData row = rows.get(0);
    assertEquals(4, row.size());
  }
  /**
   * Check the 2 lists comparing the rows in order. If they are not the same fail the test.
   *
   * @param rows1 set 1 of rows to compare
   * @param rows2 set 2 of rows to compare
   * @param fileNameColumn Number of the column containing the filename. This is only checked for
   *     being non-null (some systems maybe canonize names differently than we input).
   */
  public static void checkRows(
      List<RowMetaAndData> rows1, List<RowMetaAndData> rows2, int fileNameColumn)
      throws TestFailedException {

    int idx = 1;
    if (rows1.size() != rows2.size()) {
      throw new TestFailedException(
          "Number of rows is not the same: " + rows1.size() + " and " + rows2.size());
    }
    Iterator<RowMetaAndData> itrRows1 = rows1.iterator();
    Iterator<RowMetaAndData> itrRows2 = rows2.iterator();

    while (itrRows1.hasNext() && itrRows2.hasNext()) {
      RowMetaAndData rowMetaAndData1 = itrRows1.next();
      RowMetaAndData rowMetaAndData2 = itrRows2.next();

      RowMetaInterface rowMetaInterface1 = rowMetaAndData1.getRowMeta();

      Object[] rowObject1 = rowMetaAndData1.getData();
      Object[] rowObject2 = rowMetaAndData2.getData();

      if (rowMetaAndData1.size() != rowMetaAndData2.size()) {
        throw new TestFailedException("row number " + idx + " is not equal");
      }

      int[] fields = new int[rowMetaInterface1.size()];
      for (int ydx = 0; ydx < rowMetaInterface1.size(); ydx++) {
        fields[ydx] = ydx;
      }

      try {
        if (fileNameColumn >= 0) {
          rowObject1[fileNameColumn] = rowObject2[fileNameColumn];
        }
        if (rowMetaAndData1.getRowMeta().compare(rowObject1, rowObject2, fields) != 0) {
          throw new ComparisonFailure(
              "row nr " + idx + " is not equal",
              rowMetaInterface1.getString(rowObject1),
              rowMetaInterface1.getString(rowObject2));
        }
      } catch (KettleValueException e) {
        throw new TestFailedException("row nr " + idx + " is not equal");
      }
      idx++;
    }
  }
Пример #4
0
  private void executeShell(Result result, List<RowMetaAndData> cmdRows, String[] args) {
    FileObject fileObject = null;
    String realScript = null;
    FileObject tempFile = null;

    try {
      // What's the exact command?
      String[] base = null;
      List<String> cmds = new ArrayList<String>();

      if (log.isBasic()) {
        logBasic(BaseMessages.getString(PKG, "JobShell.RunningOn", Const.getOS()));
      }

      if (insertScript) {
        realScript = environmentSubstitute(script);
      } else {
        String realFilename = environmentSubstitute(getFilename());
        fileObject = KettleVFS.getFileObject(realFilename, this);
      }

      if (Const.getOS().equals("Windows 95")) {
        base = new String[] {"command.com", "/C"};
        if (insertScript) {
          tempFile =
              KettleVFS.createTempFile(
                  "kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
          fileObject = createTemporaryShellFile(tempFile, realScript);
        }
      } else if (Const.getOS().startsWith("Windows")) {
        base = new String[] {"cmd.exe", "/C"};
        if (insertScript) {
          tempFile =
              KettleVFS.createTempFile(
                  "kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
          fileObject = createTemporaryShellFile(tempFile, realScript);
        }
      } else {
        if (insertScript) {
          tempFile =
              KettleVFS.createTempFile(
                  "kettle", "shell", System.getProperty("java.io.tmpdir"), this);
          fileObject = createTemporaryShellFile(tempFile, realScript);
        }
        base = new String[] {KettleVFS.getFilename(fileObject)};
      }

      // Construct the arguments...
      if (argFromPrevious && cmdRows != null) {
        // Add the base command...
        for (int i = 0; i < base.length; i++) {
          cmds.add(base[i]);
        }

        if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
          // for windows all arguments including the command itself
          // need to be
          // included in 1 argument to cmd/command.

          StringBuffer cmdline = new StringBuffer(300);

          cmdline.append('"');
          cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
          // Add the arguments from previous results...
          for (int i = 0; i < cmdRows.size(); i++) {
            // Normally just one row, but once in a while to remain compatible we have multiple.

            RowMetaAndData r = cmdRows.get(i);
            for (int j = 0; j < r.size(); j++) {
              cmdline.append(' ');
              cmdline.append(Const.optionallyQuoteStringByOS(r.getString(j, null)));
            }
          }
          cmdline.append('"');
          cmds.add(cmdline.toString());
        } else {
          // Add the arguments from previous results...
          for (int i = 0; i < cmdRows.size(); i++) {
            // Normally just one row, but once in a while to remain compatible we have multiple.

            RowMetaAndData r = cmdRows.get(i);
            for (int j = 0; j < r.size(); j++) {
              cmds.add(Const.optionallyQuoteStringByOS(r.getString(j, null)));
            }
          }
        }
      } else if (args != null) {
        // Add the base command...
        for (int i = 0; i < base.length; i++) {
          cmds.add(base[i]);
        }

        if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
          // for windows all arguments including the command itself
          // need to be
          // included in 1 argument to cmd/command.

          StringBuffer cmdline = new StringBuffer(300);

          cmdline.append('"');
          cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));

          for (int i = 0; i < args.length; i++) {
            cmdline.append(' ');
            cmdline.append(Const.optionallyQuoteStringByOS(args[i]));
          }
          cmdline.append('"');
          cmds.add(cmdline.toString());
        } else {
          for (int i = 0; i < args.length; i++) {
            cmds.add(args[i]);
          }
        }
      }

      StringBuffer command = new StringBuffer();

      Iterator<String> it = cmds.iterator();
      boolean first = true;
      while (it.hasNext()) {
        if (!first) {
          command.append(' ');
        } else {
          first = false;
        }
        command.append(it.next());
      }
      if (log.isBasic()) {
        logBasic(BaseMessages.getString(PKG, "JobShell.ExecCommand", command.toString()));
      }

      // Build the environment variable list...
      ProcessBuilder procBuilder = new ProcessBuilder(cmds);
      Map<String, String> env = procBuilder.environment();
      String[] variables = listVariables();
      for (int i = 0; i < variables.length; i++) {
        env.put(variables[i], getVariable(variables[i]));
      }

      if (getWorkDirectory() != null && !Const.isEmpty(Const.rtrim(getWorkDirectory()))) {
        String vfsFilename = environmentSubstitute(getWorkDirectory());
        File file = new File(KettleVFS.getFilename(KettleVFS.getFileObject(vfsFilename, this)));
        procBuilder.directory(file);
      }
      Process proc = procBuilder.start();

      // any error message?
      StreamLogger errorLogger = new StreamLogger(log, proc.getErrorStream(), "(stderr)", true);

      // any output?
      StreamLogger outputLogger = new StreamLogger(log, proc.getInputStream(), "(stdout)");

      // kick them off
      Thread errorLoggerThread = new Thread(errorLogger);
      errorLoggerThread.start();
      Thread outputLoggerThread = new Thread(outputLogger);
      outputLoggerThread.start();

      proc.waitFor();
      if (log.isDetailed()) {
        logDetailed(BaseMessages.getString(PKG, "JobShell.CommandFinished", command.toString()));
      }

      // What's the exit status?
      result.setExitStatus(proc.exitValue());
      if (result.getExitStatus() != 0) {
        if (log.isDetailed()) {
          logDetailed(
              BaseMessages.getString(
                  PKG,
                  "JobShell.ExitStatus",
                  environmentSubstitute(getFilename()),
                  "" + result.getExitStatus()));
        }

        result.setNrErrors(1);
      }

      // wait until loggers read all data from stdout and stderr
      errorLoggerThread.join();
      outputLoggerThread.join();

      // close the streams
      // otherwise you get "Too many open files, java.io.IOException" after a lot of iterations
      proc.getErrorStream().close();
      proc.getOutputStream().close();

    } catch (IOException ioe) {
      logError(
          BaseMessages.getString(
              PKG,
              "JobShell.ErrorRunningShell",
              environmentSubstitute(getFilename()),
              ioe.toString()),
          ioe);
      result.setNrErrors(1);
    } catch (InterruptedException ie) {
      logError(
          BaseMessages.getString(
              PKG, "JobShell.Shellinterupted", environmentSubstitute(getFilename()), ie.toString()),
          ie);
      result.setNrErrors(1);
    } catch (Exception e) {
      logError(
          BaseMessages.getString(
              PKG, "JobShell.UnexpectedError", environmentSubstitute(getFilename()), e.toString()),
          e);
      result.setNrErrors(1);
    } finally {
      // If we created a temporary file, remove it...
      //
      if (tempFile != null) {
        try {
          tempFile.delete();
        } catch (Exception e) {
          BaseMessages.getString(
              PKG, "JobShell.UnexpectedError", tempFile.toString(), e.toString());
        }
      }
    }

    if (result.getNrErrors() > 0) {
      result.setResult(false);
    } else {
      result.setResult(true);
    }
  }
Пример #5
0
  public Result execute(Result result, int nr) throws KettleException {
    FileLoggingEventListener loggingEventListener = null;
    LogLevel shellLogLevel = parentJob.getLogLevel();
    if (setLogfile) {
      String realLogFilename = environmentSubstitute(getLogFilename());
      // We need to check here the log filename
      // if we do not have one, we must fail
      if (Const.isEmpty(realLogFilename)) {
        logError(BaseMessages.getString(PKG, "JobEntryShell.Exception.LogFilenameMissing"));
        result.setNrErrors(1);
        result.setResult(false);
        return result;
      }

      try {
        loggingEventListener =
            new FileLoggingEventListener(getLogChannelId(), realLogFilename, setAppendLogfile);
        KettleLogStore.getAppender().addLoggingEventListener(loggingEventListener);
      } catch (KettleException e) {
        logError(
            BaseMessages.getString(
                PKG, "JobEntryShell.Error.UnableopenAppenderFile", getLogFilename(), e.toString()));
        logError(Const.getStackTracker(e));
        result.setNrErrors(1);
        result.setResult(false);
        return result;
      }
      shellLogLevel = logFileLevel;
    }

    log.setLogLevel(shellLogLevel);

    result.setEntryNr(nr);

    // "Translate" the arguments for later
    String[] substArgs = null;
    if (arguments != null) {
      substArgs = new String[arguments.length];
      for (int idx = 0; idx < arguments.length; idx++) {
        substArgs[idx] = environmentSubstitute(arguments[idx]);
      }
    }

    int iteration = 0;
    String[] args = substArgs;
    RowMetaAndData resultRow = null;
    boolean first = true;
    List<RowMetaAndData> rows = result.getRows();

    if (log.isDetailed()) {
      logDetailed(
          BaseMessages.getString(
              PKG, "JobEntryShell.Log.FoundPreviousRows", "" + (rows != null ? rows.size() : 0)));
    }

    while ((first && !execPerRow)
        || (execPerRow && rows != null && iteration < rows.size() && result.getNrErrors() == 0)) {
      first = false;
      if (rows != null && execPerRow) {
        resultRow = rows.get(iteration);
      } else {
        resultRow = null;
      }

      List<RowMetaAndData> cmdRows = null;

      if (execPerRow) {
        // Execute for each input row

        if (argFromPrevious) {
          // Copy the input row to the (command line) arguments

          if (resultRow != null) {
            args = new String[resultRow.size()];
            for (int i = 0; i < resultRow.size(); i++) {
              args[i] = resultRow.getString(i, null);
            }
          }
        } else {
          // Just pass a single row
          List<RowMetaAndData> newList = new ArrayList<RowMetaAndData>();
          newList.add(resultRow);
          cmdRows = newList;
        }
      } else {
        if (argFromPrevious) {
          // Only put the first Row on the arguments
          args = null;
          if (resultRow != null) {
            args = new String[resultRow.size()];
            for (int i = 0; i < resultRow.size(); i++) {
              args[i] = resultRow.getString(i, null);
            }
          } else {
            cmdRows = rows;
          }
        } else {
          // Keep it as it was...
          cmdRows = rows;
        }
      }

      executeShell(result, cmdRows, args);

      iteration++;
    }

    if (setLogfile) {
      if (loggingEventListener != null) {
        KettleLogStore.getAppender().removeLoggingEventListener(loggingEventListener);
        loggingEventListener.close();

        ResultFile resultFile =
            new ResultFile(
                ResultFile.FILE_TYPE_LOG,
                loggingEventListener.getFile(),
                parentJob.getJobname(),
                getName());
        result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
      }
    }

    return result;
  }