public boolean init(StepMetaInterface smi, StepDataInterface sdi) { meta = (TextFileOutputMeta) smi; data = (TextFileOutputData) sdi; if (super.init(smi, sdi)) { data.splitnr = 0; // In case user want to create file at first row // In that case, DO NOT create file at Init if (!meta.isDoNotOpenNewFileInit()) { try { if (!meta.isFileNameInField()) { openNewFile(meta.getFileName()); } data.oneFileOpened = true; } catch (Exception e) { logError("Couldn't open file " + meta.getFileName(), e); setErrors(1L); stopAll(); } } try { initBinaryDataFields(); } catch (Exception e) { logError("Couldn't initialize binary data fields", e); setErrors(1L); stopAll(); } return true; } return false; }
private boolean closeFile() { boolean retval = false; try { if (data.writer != null) { if (log.isDebug()) logDebug("Closing output stream"); data.writer.close(); if (log.isDebug()) logDebug("Closed output stream"); } data.writer = null; if (data.cmdProc != null) { if (log.isDebug()) logDebug("Ending running external command"); int procStatus = data.cmdProc.waitFor(); // close the streams // otherwise you get "Too many open files, java.io.IOException" after a lot of iterations try { data.cmdProc.getErrorStream().close(); data.cmdProc.getOutputStream().close(); data.cmdProc.getInputStream().close(); } catch (IOException e) { if (log.isDetailed()) logDetailed("Warning: Error closing streams: " + e.getMessage()); } data.cmdProc = null; if (log.isBasic() && procStatus != 0) logBasic("Command exit status: " + procStatus); } else { if (log.isDebug()) logDebug("Closing normal file ..."); if (FILE_COMPRESSION_TYPE_ZIP.equals(meta.getFileCompression())) { data.zip.closeEntry(); data.zip.finish(); data.zip.close(); } else if (FILE_COMPRESSION_TYPE_GZIP.equals(meta.getFileCompression())) { data.gzip.finish(); } if (data.fos != null) { data.fos.close(); data.fos = null; } } retval = true; } catch (Exception e) { logError("Exception trying to close file: " + e.toString()); setErrors(1); retval = false; } return retval; }
/** * This method should only be used when you have a filename in the input stream. * * @param filename the filename to set the data.writer field for * @throws KettleException */ private void setDataWriterForFilename(String filename) throws KettleException { // First handle the writers themselves. // If we didn't have a writer yet, we create one. // Basically we open a new file // data.writer = data.fileWriterMap.get(filename); if (data.writer == null) { openNewFile(filename); data.fileWriterMap.put(filename, data.writer); // If it's the first time we open it and we have a header, we write a header... // if (!meta.isFileAppended() && meta.isHeaderEnabled()) { if (writeHeader()) { incrementLinesOutput(); } } } }
private void initBinaryDataFields() throws KettleException { try { data.hasEncoding = !Const.isEmpty(meta.getEncoding()); data.binarySeparator = new byte[] {}; data.binaryEnclosure = new byte[] {}; data.binaryNewline = new byte[] {}; if (data.hasEncoding) { if (!Const.isEmpty(meta.getSeparator())) data.binarySeparator = environmentSubstitute(meta.getSeparator()).getBytes(meta.getEncoding()); if (!Const.isEmpty(meta.getEnclosure())) data.binaryEnclosure = environmentSubstitute(meta.getEnclosure()).getBytes(meta.getEncoding()); if (!Const.isEmpty(meta.getNewline())) data.binaryNewline = meta.getNewline().getBytes(meta.getEncoding()); } else { if (!Const.isEmpty(meta.getSeparator())) data.binarySeparator = environmentSubstitute(meta.getSeparator()).getBytes(); if (!Const.isEmpty(meta.getEnclosure())) data.binaryEnclosure = environmentSubstitute(meta.getEnclosure()).getBytes(); if (!Const.isEmpty(meta.getNewline())) data.binaryNewline = environmentSubstitute(meta.getNewline()).getBytes(); } data.binaryNullValue = new byte[meta.getOutputFields().length][]; for (int i = 0; i < meta.getOutputFields().length; i++) { data.binaryNullValue[i] = null; String nullString = meta.getOutputFields()[i].getNullString(); if (!Const.isEmpty(nullString)) { if (data.hasEncoding) { data.binaryNullValue[i] = nullString.getBytes(meta.getEncoding()); } else { data.binaryNullValue[i] = nullString.getBytes(); } } } } catch (Exception e) { throw new KettleException("Unexpected error while encoding binary fields", e); } }
public synchronized boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException { meta = (TextFileOutputMeta) smi; data = (TextFileOutputData) sdi; boolean result = true; boolean bEndedLineWrote = false; Object[] r = getRow(); // This also waits for a row to be finished. if (r != null && first) { first = false; data.outputRowMeta = getInputRowMeta().clone(); meta.getFields(data.outputRowMeta, getStepname(), null, null, this); // if file name in field is enabled then set field name and open file // if (meta.isFileNameInField()) { // find and set index of file name field in input stream // data.fileNameFieldIndex = getInputRowMeta().indexOfValue(meta.getFileNameField()); // set the file name for this row // if (data.fileNameFieldIndex < 0) { throw new KettleStepException( Messages.getString( "TextFileOutput.Exception.FileNameFieldNotFound", meta.getFileNameField())); // $NON-NLS-1$ } data.fileNameMeta = getInputRowMeta().getValueMeta(data.fileNameFieldIndex); data.fileName = data.fileNameMeta.getString(r[data.fileNameFieldIndex]); setDataWriterForFilename(data.fileName); } else if (meta.isDoNotOpenNewFileInit() && !meta.isFileNameInField()) { // Open a new file here // openNewFile(meta.getFileName()); data.oneFileOpened = true; initBinaryDataFields(); } if (!meta.isFileAppended() && (meta.isHeaderEnabled() || meta.isFooterEnabled())) // See if we have to write a header-line) { if (!meta.isFileNameInField() && meta.isHeaderEnabled() && data.outputRowMeta != null) { writeHeader(); } } data.fieldnrs = new int[meta.getOutputFields().length]; for (int i = 0; i < meta.getOutputFields().length; i++) { data.fieldnrs[i] = data.outputRowMeta.indexOfValue(meta.getOutputFields()[i].getName()); if (data.fieldnrs[i] < 0) { throw new KettleStepException( "Field [" + meta.getOutputFields()[i].getName() + "] couldn't be found in the input stream!"); } } } if ((r == null && data.outputRowMeta != null && meta.isFooterEnabled()) || (r != null && getLinesOutput() > 0 && meta.getSplitEvery() > 0 && ((getLinesOutput() + 1) % meta.getSplitEvery()) == 0)) { if (data.outputRowMeta != null) { if (meta.isFooterEnabled()) { writeHeader(); } } if (r == null) { // add tag to last line if needed writeEndedLine(); bEndedLineWrote = true; } // Done with this part or with everything. closeFile(); // Not finished: open another file... if (r != null) { openNewFile(meta.getFileName()); if (meta.isHeaderEnabled() && data.outputRowMeta != null) if (writeHeader()) incrementLinesOutput(); } } if (r == null) // no more input to be expected... { if (false == bEndedLineWrote) { // add tag to last line if needed writeEndedLine(); bEndedLineWrote = true; } setOutputDone(); return false; } // First handle the file name in field // Write a header line as well if needed // if (meta.isFileNameInField()) { String baseFilename = data.fileNameMeta.getString(r[data.fileNameFieldIndex]); setDataWriterForFilename(baseFilename); } writeRowToFile(data.outputRowMeta, r); putRow(data.outputRowMeta, r); // in case we want it to go further... if (checkFeedback(getLinesOutput())) logBasic("linenr " + getLinesOutput()); return result; }
public void openNewFile(String baseFilename) throws KettleException { data.writer = null; ResultFile resultFile = null; String filename = buildFilename(environmentSubstitute(baseFilename), true); try { if (meta.isFileAsCommand()) { if (log.isDebug()) logDebug("Spawning external process"); if (data.cmdProc != null) { logError("Previous command not correctly terminated"); setErrors(1); } String cmdstr = environmentSubstitute(meta.getFileName()); if (Const.getOS().equals("Windows 95")) { cmdstr = "command.com /C " + cmdstr; } else { if (Const.getOS().startsWith("Windows")) { cmdstr = "cmd.exe /C " + cmdstr; } } if (log.isDetailed()) logDetailed("Starting: " + cmdstr); Runtime r = Runtime.getRuntime(); data.cmdProc = r.exec(cmdstr, EnvUtil.getEnvironmentVariablesForRuntimeExec()); data.writer = data.cmdProc.getOutputStream(); StreamLogger stdoutLogger = new StreamLogger(data.cmdProc.getInputStream(), "(stdout)"); StreamLogger stderrLogger = new StreamLogger(data.cmdProc.getErrorStream(), "(stderr)"); new Thread(stdoutLogger).start(); new Thread(stderrLogger).start(); } else { // Add this to the result file names... resultFile = new ResultFile( ResultFile.FILE_TYPE_GENERAL, KettleVFS.getFileObject(filename), getTransMeta().getName(), getStepname()); resultFile.setComment("This file was created with a text file output step"); addResultFile(resultFile); OutputStream outputStream; if (!Const.isEmpty(meta.getFileCompression()) && !meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_NONE)) { if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_ZIP)) { if (log.isDetailed()) log.logDetailed(toString(), "Opening output stream in zipped mode"); if (checkPreviouslyOpened(filename)) { data.fos = KettleVFS.getOutputStream(filename, true); } else { data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended()); } data.zip = new ZipOutputStream(data.fos); File entry = new File(filename); ZipEntry zipentry = new ZipEntry(entry.getName()); zipentry.setComment("Compressed by Kettle"); data.zip.putNextEntry(zipentry); outputStream = data.zip; } else if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_GZIP)) { if (log.isDetailed()) log.logDetailed(toString(), "Opening output stream in gzipped mode"); if (checkPreviouslyOpened(filename)) { data.fos = KettleVFS.getOutputStream(filename, true); } else { data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended()); } data.gzip = new GZIPOutputStream(data.fos); outputStream = data.gzip; } else { throw new KettleFileException("No compression method specified!"); } } else { if (log.isDetailed()) log.logDetailed(toString(), "Opening output stream in nocompress mode"); if (checkPreviouslyOpened(filename)) { data.fos = KettleVFS.getOutputStream(filename, true); } else { data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended()); } outputStream = data.fos; } if (!Const.isEmpty(meta.getEncoding())) { if (log.isDetailed()) log.logDetailed(toString(), "Opening output stream in encoding: " + meta.getEncoding()); data.writer = new BufferedOutputStream(outputStream, 5000); } else { if (log.isDetailed()) log.logDetailed(toString(), "Opening output stream in default encoding"); data.writer = new BufferedOutputStream(outputStream, 5000); } if (log.isDetailed()) logDetailed("Opened new file with name [" + filename + "]"); } } catch (Exception e) { throw new KettleException("Error opening new file : " + e.toString()); } // System.out.println("end of newFile(), splitnr="+splitnr); data.splitnr++; if (resultFile != null && meta.isAddToResultFiles()) { // Add this to the result file names... addResultFile(resultFile); } }