public void openNewFile(String baseFilename) throws KettleException {
data.writer = null;
ResultFile resultFile = null;
String filename = buildFilename(environmentSubstitute(baseFilename), true);
try {
if (meta.isFileAsCommand()) {
if (log.isDebug()) logDebug("Spawning external process");
if (data.cmdProc != null) {
logError("Previous command not correctly terminated");
setErrors(1);
}
String cmdstr = environmentSubstitute(meta.getFileName());
if (Const.getOS().equals("Windows 95")) {
cmdstr = "command.com /C " + cmdstr;
} else {
if (Const.getOS().startsWith("Windows")) {
cmdstr = "cmd.exe /C " + cmdstr;
}
}
if (log.isDetailed()) logDetailed("Starting: " + cmdstr);
Runtime r = Runtime.getRuntime();
data.cmdProc = r.exec(cmdstr, EnvUtil.getEnvironmentVariablesForRuntimeExec());
data.writer = data.cmdProc.getOutputStream();
StreamLogger stdoutLogger = new StreamLogger(data.cmdProc.getInputStream(), "(stdout)");
StreamLogger stderrLogger = new StreamLogger(data.cmdProc.getErrorStream(), "(stderr)");
new Thread(stdoutLogger).start();
new Thread(stderrLogger).start();
} else {
// Add this to the result file names...
resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL,
KettleVFS.getFileObject(filename),
getTransMeta().getName(),
getStepname());
resultFile.setComment("This file was created with a text file output step");
addResultFile(resultFile);
OutputStream outputStream;
if (!Const.isEmpty(meta.getFileCompression())
&& !meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_NONE)) {
if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_ZIP)) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in zipped mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
data.zip = new ZipOutputStream(data.fos);
File entry = new File(filename);
ZipEntry zipentry = new ZipEntry(entry.getName());
zipentry.setComment("Compressed by Kettle");
data.zip.putNextEntry(zipentry);
outputStream = data.zip;
} else if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_GZIP)) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in gzipped mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
data.gzip = new GZIPOutputStream(data.fos);
outputStream = data.gzip;
} else {
throw new KettleFileException("No compression method specified!");
}
} else {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in nocompress mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
outputStream = data.fos;
}
if (!Const.isEmpty(meta.getEncoding())) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in encoding: " + meta.getEncoding());
data.writer = new BufferedOutputStream(outputStream, 5000);
} else {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in default encoding");
data.writer = new BufferedOutputStream(outputStream, 5000);
}
if (log.isDetailed()) logDetailed("Opened new file with name [" + filename + "]");
}
} catch (Exception e) {
throw new KettleException("Error opening new file : " + e.toString());
}
// System.out.println("end of newFile(), splitnr="+splitnr);
data.splitnr++;
if (resultFile != null && meta.isAddToResultFiles()) {
// Add this to the result file names...
addResultFile(resultFile);
}
}
private void executeShell(Result result, List<RowMetaAndData> cmdRows, String[] args) {
FileObject fileObject = null;
String realScript = null;
FileObject tempFile = null;
try {
// What's the exact command?
String[] base = null;
List<String> cmds = new ArrayList<String>();
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobShell.RunningOn", Const.getOS()));
}
if (insertScript) {
realScript = environmentSubstitute(script);
} else {
String realFilename = environmentSubstitute(getFilename());
fileObject = KettleVFS.getFileObject(realFilename, this);
}
if (Const.getOS().equals("Windows 95")) {
base = new String[] {"command.com", "/C"};
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
} else if (Const.getOS().startsWith("Windows")) {
base = new String[] {"cmd.exe", "/C"};
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
} else {
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
base = new String[] {KettleVFS.getFilename(fileObject)};
}
// Construct the arguments...
if (argFromPrevious && cmdRows != null) {
// Add the base command...
for (int i = 0; i < base.length; i++) {
cmds.add(base[i]);
}
if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
// for windows all arguments including the command itself
// need to be
// included in 1 argument to cmd/command.
StringBuffer cmdline = new StringBuffer(300);
cmdline.append('"');
cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
// Add the arguments from previous results...
for (int i = 0; i < cmdRows.size(); i++) {
// Normally just one row, but once in a while to remain compatible we have multiple.
RowMetaAndData r = cmdRows.get(i);
for (int j = 0; j < r.size(); j++) {
cmdline.append(' ');
cmdline.append(Const.optionallyQuoteStringByOS(r.getString(j, null)));
}
}
cmdline.append('"');
cmds.add(cmdline.toString());
} else {
// Add the arguments from previous results...
for (int i = 0; i < cmdRows.size(); i++) {
// Normally just one row, but once in a while to remain compatible we have multiple.
RowMetaAndData r = cmdRows.get(i);
for (int j = 0; j < r.size(); j++) {
cmds.add(Const.optionallyQuoteStringByOS(r.getString(j, null)));
}
}
}
} else if (args != null) {
// Add the base command...
for (int i = 0; i < base.length; i++) {
cmds.add(base[i]);
}
if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
// for windows all arguments including the command itself
// need to be
// included in 1 argument to cmd/command.
StringBuffer cmdline = new StringBuffer(300);
cmdline.append('"');
cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
for (int i = 0; i < args.length; i++) {
cmdline.append(' ');
cmdline.append(Const.optionallyQuoteStringByOS(args[i]));
}
cmdline.append('"');
cmds.add(cmdline.toString());
} else {
for (int i = 0; i < args.length; i++) {
cmds.add(args[i]);
}
}
}
StringBuffer command = new StringBuffer();
Iterator<String> it = cmds.iterator();
boolean first = true;
while (it.hasNext()) {
if (!first) {
command.append(' ');
} else {
first = false;
}
command.append(it.next());
}
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobShell.ExecCommand", command.toString()));
}
// Build the environment variable list...
ProcessBuilder procBuilder = new ProcessBuilder(cmds);
Map<String, String> env = procBuilder.environment();
String[] variables = listVariables();
for (int i = 0; i < variables.length; i++) {
env.put(variables[i], getVariable(variables[i]));
}
if (getWorkDirectory() != null && !Const.isEmpty(Const.rtrim(getWorkDirectory()))) {
String vfsFilename = environmentSubstitute(getWorkDirectory());
File file = new File(KettleVFS.getFilename(KettleVFS.getFileObject(vfsFilename, this)));
procBuilder.directory(file);
}
Process proc = procBuilder.start();
// any error message?
StreamLogger errorLogger = new StreamLogger(log, proc.getErrorStream(), "(stderr)", true);
// any output?
StreamLogger outputLogger = new StreamLogger(log, proc.getInputStream(), "(stdout)");
// kick them off
Thread errorLoggerThread = new Thread(errorLogger);
errorLoggerThread.start();
Thread outputLoggerThread = new Thread(outputLogger);
outputLoggerThread.start();
proc.waitFor();
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobShell.CommandFinished", command.toString()));
}
// What's the exit status?
result.setExitStatus(proc.exitValue());
if (result.getExitStatus() != 0) {
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobShell.ExitStatus",
environmentSubstitute(getFilename()),
"" + result.getExitStatus()));
}
result.setNrErrors(1);
}
// wait until loggers read all data from stdout and stderr
errorLoggerThread.join();
outputLoggerThread.join();
// close the streams
// otherwise you get "Too many open files, java.io.IOException" after a lot of iterations
proc.getErrorStream().close();
proc.getOutputStream().close();
} catch (IOException ioe) {
logError(
BaseMessages.getString(
PKG,
"JobShell.ErrorRunningShell",
environmentSubstitute(getFilename()),
ioe.toString()),
ioe);
result.setNrErrors(1);
} catch (InterruptedException ie) {
logError(
BaseMessages.getString(
PKG, "JobShell.Shellinterupted", environmentSubstitute(getFilename()), ie.toString()),
ie);
result.setNrErrors(1);
} catch (Exception e) {
logError(
BaseMessages.getString(
PKG, "JobShell.UnexpectedError", environmentSubstitute(getFilename()), e.toString()),
e);
result.setNrErrors(1);
} finally {
// If we created a temporary file, remove it...
//
if (tempFile != null) {
try {
tempFile.delete();
} catch (Exception e) {
BaseMessages.getString(
PKG, "JobShell.UnexpectedError", tempFile.toString(), e.toString());
}
}
}
if (result.getNrErrors() > 0) {
result.setResult(false);
} else {
result.setResult(true);
}
}
