public void openNewFile(String baseFilename) throws KettleException { data.writer = null; ResultFile resultFile = null; String filename = buildFilename(environmentSubstitute(baseFilename), true); try { if (meta.isFileAsCommand()) { if (log.isDebug()) logDebug("Spawning external process"); if (data.cmdProc != null) { logError("Previous command not correctly terminated"); setErrors(1); } String cmdstr = environmentSubstitute(meta.getFileName()); if (Const.getOS().equals("Windows 95")) { cmdstr = "command.com /C " + cmdstr; } else { if (Const.getOS().startsWith("Windows")) { cmdstr = "cmd.exe /C " + cmdstr; } } if (log.isDetailed()) logDetailed("Starting: " + cmdstr); Runtime r = Runtime.getRuntime(); data.cmdProc = r.exec(cmdstr, EnvUtil.getEnvironmentVariablesForRuntimeExec()); data.writer = data.cmdProc.getOutputStream(); StreamLogger stdoutLogger = new StreamLogger(data.cmdProc.getInputStream(), "(stdout)"); StreamLogger stderrLogger = new StreamLogger(data.cmdProc.getErrorStream(), "(stderr)"); new Thread(stdoutLogger).start(); new Thread(stderrLogger).start(); } else { // Add this to the result file names... resultFile = new ResultFile( ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject(filename), getTransMeta().getName(), getStepname()); resultFile.setComment("This file was created with a text file output step"); addResultFile(resultFile); OutputStream outputStream; if (!Const.isEmpty(meta.getFileCompression()) && !meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_NONE)) { if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_ZIP)) { if (log.isDetailed()) log.logDetailed(toString(), "Opening output stream in zipped mode"); if (checkPreviouslyOpened(filename)) { data.fos = KettleVFS.getOutputStream(filename, true); } else { data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended()); } data.zip = new ZipOutputStream(data.fos); File entry = new File(filename); ZipEntry zipentry = new ZipEntry(entry.getName()); zipentry.setComment("Compressed by Kettle"); data.zip.putNextEntry(zipentry); outputStream = data.zip; } else if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_GZIP)) { if (log.isDetailed()) log.logDetailed(toString(), "Opening output stream in gzipped mode"); if (checkPreviouslyOpened(filename)) { data.fos = KettleVFS.getOutputStream(filename, true); } else { data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended()); } data.gzip = new GZIPOutputStream(data.fos); outputStream = data.gzip; } else { throw new KettleFileException("No compression method specified!"); } } else { if (log.isDetailed()) log.logDetailed(toString(), "Opening output stream in nocompress mode"); if (checkPreviouslyOpened(filename)) { data.fos = KettleVFS.getOutputStream(filename, true); } else { data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended()); } outputStream = data.fos; } if (!Const.isEmpty(meta.getEncoding())) { if (log.isDetailed()) log.logDetailed(toString(), "Opening output stream in encoding: " + meta.getEncoding()); data.writer = new BufferedOutputStream(outputStream, 5000); } else { if (log.isDetailed()) log.logDetailed(toString(), "Opening output stream in default encoding"); data.writer = new BufferedOutputStream(outputStream, 5000); } if (log.isDetailed()) logDetailed("Opened new file with name [" + filename + "]"); } } catch (Exception e) { throw new KettleException("Error opening new file : " + e.toString()); } // System.out.println("end of newFile(), splitnr="+splitnr); data.splitnr++; if (resultFile != null && meta.isAddToResultFiles()) { // Add this to the result file names... addResultFile(resultFile); } }
private void executeShell(Result result, List<RowMetaAndData> cmdRows, String[] args) { FileObject fileObject = null; String realScript = null; FileObject tempFile = null; try { // What's the exact command? String[] base = null; List<String> cmds = new ArrayList<String>(); if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "JobShell.RunningOn", Const.getOS())); } if (insertScript) { realScript = environmentSubstitute(script); } else { String realFilename = environmentSubstitute(getFilename()); fileObject = KettleVFS.getFileObject(realFilename, this); } if (Const.getOS().equals("Windows 95")) { base = new String[] {"command.com", "/C"}; if (insertScript) { tempFile = KettleVFS.createTempFile( "kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this); fileObject = createTemporaryShellFile(tempFile, realScript); } } else if (Const.getOS().startsWith("Windows")) { base = new String[] {"cmd.exe", "/C"}; if (insertScript) { tempFile = KettleVFS.createTempFile( "kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this); fileObject = createTemporaryShellFile(tempFile, realScript); } } else { if (insertScript) { tempFile = KettleVFS.createTempFile( "kettle", "shell", System.getProperty("java.io.tmpdir"), this); fileObject = createTemporaryShellFile(tempFile, realScript); } base = new String[] {KettleVFS.getFilename(fileObject)}; } // Construct the arguments... if (argFromPrevious && cmdRows != null) { // Add the base command... for (int i = 0; i < base.length; i++) { cmds.add(base[i]); } if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) { // for windows all arguments including the command itself // need to be // included in 1 argument to cmd/command. StringBuffer cmdline = new StringBuffer(300); cmdline.append('"'); cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject))); // Add the arguments from previous results... for (int i = 0; i < cmdRows.size(); i++) { // Normally just one row, but once in a while to remain compatible we have multiple. RowMetaAndData r = cmdRows.get(i); for (int j = 0; j < r.size(); j++) { cmdline.append(' '); cmdline.append(Const.optionallyQuoteStringByOS(r.getString(j, null))); } } cmdline.append('"'); cmds.add(cmdline.toString()); } else { // Add the arguments from previous results... for (int i = 0; i < cmdRows.size(); i++) { // Normally just one row, but once in a while to remain compatible we have multiple. RowMetaAndData r = cmdRows.get(i); for (int j = 0; j < r.size(); j++) { cmds.add(Const.optionallyQuoteStringByOS(r.getString(j, null))); } } } } else if (args != null) { // Add the base command... for (int i = 0; i < base.length; i++) { cmds.add(base[i]); } if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) { // for windows all arguments including the command itself // need to be // included in 1 argument to cmd/command. StringBuffer cmdline = new StringBuffer(300); cmdline.append('"'); cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject))); for (int i = 0; i < args.length; i++) { cmdline.append(' '); cmdline.append(Const.optionallyQuoteStringByOS(args[i])); } cmdline.append('"'); cmds.add(cmdline.toString()); } else { for (int i = 0; i < args.length; i++) { cmds.add(args[i]); } } } StringBuffer command = new StringBuffer(); Iterator<String> it = cmds.iterator(); boolean first = true; while (it.hasNext()) { if (!first) { command.append(' '); } else { first = false; } command.append(it.next()); } if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "JobShell.ExecCommand", command.toString())); } // Build the environment variable list... ProcessBuilder procBuilder = new ProcessBuilder(cmds); Map<String, String> env = procBuilder.environment(); String[] variables = listVariables(); for (int i = 0; i < variables.length; i++) { env.put(variables[i], getVariable(variables[i])); } if (getWorkDirectory() != null && !Const.isEmpty(Const.rtrim(getWorkDirectory()))) { String vfsFilename = environmentSubstitute(getWorkDirectory()); File file = new File(KettleVFS.getFilename(KettleVFS.getFileObject(vfsFilename, this))); procBuilder.directory(file); } Process proc = procBuilder.start(); // any error message? StreamLogger errorLogger = new StreamLogger(log, proc.getErrorStream(), "(stderr)", true); // any output? StreamLogger outputLogger = new StreamLogger(log, proc.getInputStream(), "(stdout)"); // kick them off Thread errorLoggerThread = new Thread(errorLogger); errorLoggerThread.start(); Thread outputLoggerThread = new Thread(outputLogger); outputLoggerThread.start(); proc.waitFor(); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobShell.CommandFinished", command.toString())); } // What's the exit status? result.setExitStatus(proc.exitValue()); if (result.getExitStatus() != 0) { if (log.isDetailed()) { logDetailed( BaseMessages.getString( PKG, "JobShell.ExitStatus", environmentSubstitute(getFilename()), "" + result.getExitStatus())); } result.setNrErrors(1); } // wait until loggers read all data from stdout and stderr errorLoggerThread.join(); outputLoggerThread.join(); // close the streams // otherwise you get "Too many open files, java.io.IOException" after a lot of iterations proc.getErrorStream().close(); proc.getOutputStream().close(); } catch (IOException ioe) { logError( BaseMessages.getString( PKG, "JobShell.ErrorRunningShell", environmentSubstitute(getFilename()), ioe.toString()), ioe); result.setNrErrors(1); } catch (InterruptedException ie) { logError( BaseMessages.getString( PKG, "JobShell.Shellinterupted", environmentSubstitute(getFilename()), ie.toString()), ie); result.setNrErrors(1); } catch (Exception e) { logError( BaseMessages.getString( PKG, "JobShell.UnexpectedError", environmentSubstitute(getFilename()), e.toString()), e); result.setNrErrors(1); } finally { // If we created a temporary file, remove it... // if (tempFile != null) { try { tempFile.delete(); } catch (Exception e) { BaseMessages.getString( PKG, "JobShell.UnexpectedError", tempFile.toString(), e.toString()); } } } if (result.getNrErrors() > 0) { result.setResult(false); } else { result.setResult(true); } }