/** Check the 2 lists comparing the rows in order. If they are not the same fail the test. */
public void checkRows(List<RowMetaAndData> rows1, List<RowMetaAndData> rows2) {
int idx = 1;
if (rows1.size() != rows2.size()) {
fail("Number of rows is not the same: " + rows1.size() + " and " + rows2.size());
}
Iterator<RowMetaAndData> it1 = rows1.iterator();
Iterator<RowMetaAndData> it2 = rows2.iterator();
while (it1.hasNext() && it2.hasNext()) {
RowMetaAndData rm1 = it1.next();
RowMetaAndData rm2 = it2.next();
Object[] r1 = rm1.getData();
Object[] r2 = rm2.getData();
if (rm1.size() != rm2.size()) {
fail("row nr " + idx + " is not equal");
}
int fields[] = new int[rm1.size()];
for (int ydx = 0; ydx < rm1.size(); ydx++) {
fields[ydx] = ydx;
}
try {
if (rm1.getRowMeta().compare(r1, r2, fields) != 0) {
fail("row nr " + idx + " is not equal");
}
} catch (KettleValueException e) {
fail("row nr " + idx + " is not equal");
}
idx++;
}
}
public void test01_BasicSelectFrom() throws Exception {
KettleEnvironment.init();
String sqlQuery = "SELECT * FROM Service";
SqlTransExecutor executor = new SqlTransExecutor(sqlQuery, getServices());
final List<RowMetaAndData> rows = new ArrayList<RowMetaAndData>();
// print the eventual result rows...
//
executor.executeQuery(
new RowAdapter() {
@Override
public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
throws KettleStepException {
rows.add(new RowMetaAndData(rowMeta, row));
}
});
// Now the generated transformation is waiting for input so we
// can start the service transformation
//
executor.waitUntilFinished();
assertEquals(8, rows.size());
RowMetaAndData row = rows.get(0);
assertEquals(4, row.size());
}
/**
* Check the 2 lists comparing the rows in order. If they are not the same fail the test.
*
* @param rows1 set 1 of rows to compare
* @param rows2 set 2 of rows to compare
* @param fileNameColumn Number of the column containing the filename. This is only checked for
* being non-null (some systems maybe canonize names differently than we input).
*/
public static void checkRows(
List<RowMetaAndData> rows1, List<RowMetaAndData> rows2, int fileNameColumn)
throws TestFailedException {
int idx = 1;
if (rows1.size() != rows2.size()) {
throw new TestFailedException(
"Number of rows is not the same: " + rows1.size() + " and " + rows2.size());
}
Iterator<RowMetaAndData> itrRows1 = rows1.iterator();
Iterator<RowMetaAndData> itrRows2 = rows2.iterator();
while (itrRows1.hasNext() && itrRows2.hasNext()) {
RowMetaAndData rowMetaAndData1 = itrRows1.next();
RowMetaAndData rowMetaAndData2 = itrRows2.next();
RowMetaInterface rowMetaInterface1 = rowMetaAndData1.getRowMeta();
Object[] rowObject1 = rowMetaAndData1.getData();
Object[] rowObject2 = rowMetaAndData2.getData();
if (rowMetaAndData1.size() != rowMetaAndData2.size()) {
throw new TestFailedException("row number " + idx + " is not equal");
}
int[] fields = new int[rowMetaInterface1.size()];
for (int ydx = 0; ydx < rowMetaInterface1.size(); ydx++) {
fields[ydx] = ydx;
}
try {
if (fileNameColumn >= 0) {
rowObject1[fileNameColumn] = rowObject2[fileNameColumn];
}
if (rowMetaAndData1.getRowMeta().compare(rowObject1, rowObject2, fields) != 0) {
throw new ComparisonFailure(
"row nr " + idx + " is not equal",
rowMetaInterface1.getString(rowObject1),
rowMetaInterface1.getString(rowObject2));
}
} catch (KettleValueException e) {
throw new TestFailedException("row nr " + idx + " is not equal");
}
idx++;
}
}
private void executeShell(Result result, List<RowMetaAndData> cmdRows, String[] args) {
FileObject fileObject = null;
String realScript = null;
FileObject tempFile = null;
try {
// What's the exact command?
String[] base = null;
List<String> cmds = new ArrayList<String>();
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobShell.RunningOn", Const.getOS()));
}
if (insertScript) {
realScript = environmentSubstitute(script);
} else {
String realFilename = environmentSubstitute(getFilename());
fileObject = KettleVFS.getFileObject(realFilename, this);
}
if (Const.getOS().equals("Windows 95")) {
base = new String[] {"command.com", "/C"};
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
} else if (Const.getOS().startsWith("Windows")) {
base = new String[] {"cmd.exe", "/C"};
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
} else {
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
base = new String[] {KettleVFS.getFilename(fileObject)};
}
// Construct the arguments...
if (argFromPrevious && cmdRows != null) {
// Add the base command...
for (int i = 0; i < base.length; i++) {
cmds.add(base[i]);
}
if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
// for windows all arguments including the command itself
// need to be
// included in 1 argument to cmd/command.
StringBuffer cmdline = new StringBuffer(300);
cmdline.append('"');
cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
// Add the arguments from previous results...
for (int i = 0; i < cmdRows.size(); i++) {
// Normally just one row, but once in a while to remain compatible we have multiple.
RowMetaAndData r = cmdRows.get(i);
for (int j = 0; j < r.size(); j++) {
cmdline.append(' ');
cmdline.append(Const.optionallyQuoteStringByOS(r.getString(j, null)));
}
}
cmdline.append('"');
cmds.add(cmdline.toString());
} else {
// Add the arguments from previous results...
for (int i = 0; i < cmdRows.size(); i++) {
// Normally just one row, but once in a while to remain compatible we have multiple.
RowMetaAndData r = cmdRows.get(i);
for (int j = 0; j < r.size(); j++) {
cmds.add(Const.optionallyQuoteStringByOS(r.getString(j, null)));
}
}
}
} else if (args != null) {
// Add the base command...
for (int i = 0; i < base.length; i++) {
cmds.add(base[i]);
}
if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
// for windows all arguments including the command itself
// need to be
// included in 1 argument to cmd/command.
StringBuffer cmdline = new StringBuffer(300);
cmdline.append('"');
cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
for (int i = 0; i < args.length; i++) {
cmdline.append(' ');
cmdline.append(Const.optionallyQuoteStringByOS(args[i]));
}
cmdline.append('"');
cmds.add(cmdline.toString());
} else {
for (int i = 0; i < args.length; i++) {
cmds.add(args[i]);
}
}
}
StringBuffer command = new StringBuffer();
Iterator<String> it = cmds.iterator();
boolean first = true;
while (it.hasNext()) {
if (!first) {
command.append(' ');
} else {
first = false;
}
command.append(it.next());
}
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobShell.ExecCommand", command.toString()));
}
// Build the environment variable list...
ProcessBuilder procBuilder = new ProcessBuilder(cmds);
Map<String, String> env = procBuilder.environment();
String[] variables = listVariables();
for (int i = 0; i < variables.length; i++) {
env.put(variables[i], getVariable(variables[i]));
}
if (getWorkDirectory() != null && !Const.isEmpty(Const.rtrim(getWorkDirectory()))) {
String vfsFilename = environmentSubstitute(getWorkDirectory());
File file = new File(KettleVFS.getFilename(KettleVFS.getFileObject(vfsFilename, this)));
procBuilder.directory(file);
}
Process proc = procBuilder.start();
// any error message?
StreamLogger errorLogger = new StreamLogger(log, proc.getErrorStream(), "(stderr)", true);
// any output?
StreamLogger outputLogger = new StreamLogger(log, proc.getInputStream(), "(stdout)");
// kick them off
Thread errorLoggerThread = new Thread(errorLogger);
errorLoggerThread.start();
Thread outputLoggerThread = new Thread(outputLogger);
outputLoggerThread.start();
proc.waitFor();
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobShell.CommandFinished", command.toString()));
}
// What's the exit status?
result.setExitStatus(proc.exitValue());
if (result.getExitStatus() != 0) {
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobShell.ExitStatus",
environmentSubstitute(getFilename()),
"" + result.getExitStatus()));
}
result.setNrErrors(1);
}
// wait until loggers read all data from stdout and stderr
errorLoggerThread.join();
outputLoggerThread.join();
// close the streams
// otherwise you get "Too many open files, java.io.IOException" after a lot of iterations
proc.getErrorStream().close();
proc.getOutputStream().close();
} catch (IOException ioe) {
logError(
BaseMessages.getString(
PKG,
"JobShell.ErrorRunningShell",
environmentSubstitute(getFilename()),
ioe.toString()),
ioe);
result.setNrErrors(1);
} catch (InterruptedException ie) {
logError(
BaseMessages.getString(
PKG, "JobShell.Shellinterupted", environmentSubstitute(getFilename()), ie.toString()),
ie);
result.setNrErrors(1);
} catch (Exception e) {
logError(
BaseMessages.getString(
PKG, "JobShell.UnexpectedError", environmentSubstitute(getFilename()), e.toString()),
e);
result.setNrErrors(1);
} finally {
// If we created a temporary file, remove it...
//
if (tempFile != null) {
try {
tempFile.delete();
} catch (Exception e) {
BaseMessages.getString(
PKG, "JobShell.UnexpectedError", tempFile.toString(), e.toString());
}
}
}
if (result.getNrErrors() > 0) {
result.setResult(false);
} else {
result.setResult(true);
}
}
public Result execute(Result result, int nr) throws KettleException {
FileLoggingEventListener loggingEventListener = null;
LogLevel shellLogLevel = parentJob.getLogLevel();
if (setLogfile) {
String realLogFilename = environmentSubstitute(getLogFilename());
// We need to check here the log filename
// if we do not have one, we must fail
if (Const.isEmpty(realLogFilename)) {
logError(BaseMessages.getString(PKG, "JobEntryShell.Exception.LogFilenameMissing"));
result.setNrErrors(1);
result.setResult(false);
return result;
}
try {
loggingEventListener =
new FileLoggingEventListener(getLogChannelId(), realLogFilename, setAppendLogfile);
KettleLogStore.getAppender().addLoggingEventListener(loggingEventListener);
} catch (KettleException e) {
logError(
BaseMessages.getString(
PKG, "JobEntryShell.Error.UnableopenAppenderFile", getLogFilename(), e.toString()));
logError(Const.getStackTracker(e));
result.setNrErrors(1);
result.setResult(false);
return result;
}
shellLogLevel = logFileLevel;
}
log.setLogLevel(shellLogLevel);
result.setEntryNr(nr);
// "Translate" the arguments for later
String[] substArgs = null;
if (arguments != null) {
substArgs = new String[arguments.length];
for (int idx = 0; idx < arguments.length; idx++) {
substArgs[idx] = environmentSubstitute(arguments[idx]);
}
}
int iteration = 0;
String[] args = substArgs;
RowMetaAndData resultRow = null;
boolean first = true;
List<RowMetaAndData> rows = result.getRows();
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryShell.Log.FoundPreviousRows", "" + (rows != null ? rows.size() : 0)));
}
while ((first && !execPerRow)
|| (execPerRow && rows != null && iteration < rows.size() && result.getNrErrors() == 0)) {
first = false;
if (rows != null && execPerRow) {
resultRow = rows.get(iteration);
} else {
resultRow = null;
}
List<RowMetaAndData> cmdRows = null;
if (execPerRow) {
// Execute for each input row
if (argFromPrevious) {
// Copy the input row to the (command line) arguments
if (resultRow != null) {
args = new String[resultRow.size()];
for (int i = 0; i < resultRow.size(); i++) {
args[i] = resultRow.getString(i, null);
}
}
} else {
// Just pass a single row
List<RowMetaAndData> newList = new ArrayList<RowMetaAndData>();
newList.add(resultRow);
cmdRows = newList;
}
} else {
if (argFromPrevious) {
// Only put the first Row on the arguments
args = null;
if (resultRow != null) {
args = new String[resultRow.size()];
for (int i = 0; i < resultRow.size(); i++) {
args[i] = resultRow.getString(i, null);
}
} else {
cmdRows = rows;
}
} else {
// Keep it as it was...
cmdRows = rows;
}
}
executeShell(result, cmdRows, args);
iteration++;
}
if (setLogfile) {
if (loggingEventListener != null) {
KettleLogStore.getAppender().removeLoggingEventListener(loggingEventListener);
loggingEventListener.close();
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_LOG,
loggingEventListener.getFile(),
parentJob.getJobname(),
getName());
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
}
}
return result;
}