public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
meta = (TextFileOutputMeta) smi;
data = (TextFileOutputData) sdi;
if (super.init(smi, sdi)) {
data.splitnr = 0;
// In case user want to create file at first row
// In that case, DO NOT create file at Init
if (!meta.isDoNotOpenNewFileInit()) {
try {
if (!meta.isFileNameInField()) {
openNewFile(meta.getFileName());
}
data.oneFileOpened = true;
} catch (Exception e) {
logError("Couldn't open file " + meta.getFileName(), e);
setErrors(1L);
stopAll();
}
}
try {
initBinaryDataFields();
} catch (Exception e) {
logError("Couldn't initialize binary data fields", e);
setErrors(1L);
stopAll();
}
return true;
}
return false;
}
private boolean closeFile() {
boolean retval = false;
try {
if (data.writer != null) {
if (log.isDebug()) logDebug("Closing output stream");
data.writer.close();
if (log.isDebug()) logDebug("Closed output stream");
}
data.writer = null;
if (data.cmdProc != null) {
if (log.isDebug()) logDebug("Ending running external command");
int procStatus = data.cmdProc.waitFor();
// close the streams
// otherwise you get "Too many open files, java.io.IOException" after a lot of iterations
try {
data.cmdProc.getErrorStream().close();
data.cmdProc.getOutputStream().close();
data.cmdProc.getInputStream().close();
} catch (IOException e) {
if (log.isDetailed()) logDetailed("Warning: Error closing streams: " + e.getMessage());
}
data.cmdProc = null;
if (log.isBasic() && procStatus != 0) logBasic("Command exit status: " + procStatus);
} else {
if (log.isDebug()) logDebug("Closing normal file ...");
if (FILE_COMPRESSION_TYPE_ZIP.equals(meta.getFileCompression())) {
data.zip.closeEntry();
data.zip.finish();
data.zip.close();
} else if (FILE_COMPRESSION_TYPE_GZIP.equals(meta.getFileCompression())) {
data.gzip.finish();
}
if (data.fos != null) {
data.fos.close();
data.fos = null;
}
}
retval = true;
} catch (Exception e) {
logError("Exception trying to close file: " + e.toString());
setErrors(1);
retval = false;
}
return retval;
}
/**
* This method should only be used when you have a filename in the input stream.
*
* @param filename the filename to set the data.writer field for
* @throws KettleException
*/
private void setDataWriterForFilename(String filename) throws KettleException {
// First handle the writers themselves.
// If we didn't have a writer yet, we create one.
// Basically we open a new file
//
data.writer = data.fileWriterMap.get(filename);
if (data.writer == null) {
openNewFile(filename);
data.fileWriterMap.put(filename, data.writer);
// If it's the first time we open it and we have a header, we write a header...
//
if (!meta.isFileAppended() && meta.isHeaderEnabled()) {
if (writeHeader()) {
incrementLinesOutput();
}
}
}
}
private void initBinaryDataFields() throws KettleException {
try {
data.hasEncoding = !Const.isEmpty(meta.getEncoding());
data.binarySeparator = new byte[] {};
data.binaryEnclosure = new byte[] {};
data.binaryNewline = new byte[] {};
if (data.hasEncoding) {
if (!Const.isEmpty(meta.getSeparator()))
data.binarySeparator =
environmentSubstitute(meta.getSeparator()).getBytes(meta.getEncoding());
if (!Const.isEmpty(meta.getEnclosure()))
data.binaryEnclosure =
environmentSubstitute(meta.getEnclosure()).getBytes(meta.getEncoding());
if (!Const.isEmpty(meta.getNewline()))
data.binaryNewline = meta.getNewline().getBytes(meta.getEncoding());
} else {
if (!Const.isEmpty(meta.getSeparator()))
data.binarySeparator = environmentSubstitute(meta.getSeparator()).getBytes();
if (!Const.isEmpty(meta.getEnclosure()))
data.binaryEnclosure = environmentSubstitute(meta.getEnclosure()).getBytes();
if (!Const.isEmpty(meta.getNewline()))
data.binaryNewline = environmentSubstitute(meta.getNewline()).getBytes();
}
data.binaryNullValue = new byte[meta.getOutputFields().length][];
for (int i = 0; i < meta.getOutputFields().length; i++) {
data.binaryNullValue[i] = null;
String nullString = meta.getOutputFields()[i].getNullString();
if (!Const.isEmpty(nullString)) {
if (data.hasEncoding) {
data.binaryNullValue[i] = nullString.getBytes(meta.getEncoding());
} else {
data.binaryNullValue[i] = nullString.getBytes();
}
}
}
} catch (Exception e) {
throw new KettleException("Unexpected error while encoding binary fields", e);
}
}
public synchronized boolean processRow(StepMetaInterface smi, StepDataInterface sdi)
throws KettleException {
meta = (TextFileOutputMeta) smi;
data = (TextFileOutputData) sdi;
boolean result = true;
boolean bEndedLineWrote = false;
Object[] r = getRow(); // This also waits for a row to be finished.
if (r != null && first) {
first = false;
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
// if file name in field is enabled then set field name and open file
//
if (meta.isFileNameInField()) {
// find and set index of file name field in input stream
//
data.fileNameFieldIndex = getInputRowMeta().indexOfValue(meta.getFileNameField());
// set the file name for this row
//
if (data.fileNameFieldIndex < 0) {
throw new KettleStepException(
Messages.getString(
"TextFileOutput.Exception.FileNameFieldNotFound",
meta.getFileNameField())); // $NON-NLS-1$
}
data.fileNameMeta = getInputRowMeta().getValueMeta(data.fileNameFieldIndex);
data.fileName = data.fileNameMeta.getString(r[data.fileNameFieldIndex]);
setDataWriterForFilename(data.fileName);
} else if (meta.isDoNotOpenNewFileInit() && !meta.isFileNameInField()) {
// Open a new file here
//
openNewFile(meta.getFileName());
data.oneFileOpened = true;
initBinaryDataFields();
}
if (!meta.isFileAppended()
&& (meta.isHeaderEnabled()
|| meta.isFooterEnabled())) // See if we have to write a header-line)
{
if (!meta.isFileNameInField() && meta.isHeaderEnabled() && data.outputRowMeta != null) {
writeHeader();
}
}
data.fieldnrs = new int[meta.getOutputFields().length];
for (int i = 0; i < meta.getOutputFields().length; i++) {
data.fieldnrs[i] = data.outputRowMeta.indexOfValue(meta.getOutputFields()[i].getName());
if (data.fieldnrs[i] < 0) {
throw new KettleStepException(
"Field ["
+ meta.getOutputFields()[i].getName()
+ "] couldn't be found in the input stream!");
}
}
}
if ((r == null && data.outputRowMeta != null && meta.isFooterEnabled())
|| (r != null
&& getLinesOutput() > 0
&& meta.getSplitEvery() > 0
&& ((getLinesOutput() + 1) % meta.getSplitEvery()) == 0)) {
if (data.outputRowMeta != null) {
if (meta.isFooterEnabled()) {
writeHeader();
}
}
if (r == null) {
// add tag to last line if needed
writeEndedLine();
bEndedLineWrote = true;
}
// Done with this part or with everything.
closeFile();
// Not finished: open another file...
if (r != null) {
openNewFile(meta.getFileName());
if (meta.isHeaderEnabled() && data.outputRowMeta != null)
if (writeHeader()) incrementLinesOutput();
}
}
if (r == null) // no more input to be expected...
{
if (false == bEndedLineWrote) {
// add tag to last line if needed
writeEndedLine();
bEndedLineWrote = true;
}
setOutputDone();
return false;
}
// First handle the file name in field
// Write a header line as well if needed
//
if (meta.isFileNameInField()) {
String baseFilename = data.fileNameMeta.getString(r[data.fileNameFieldIndex]);
setDataWriterForFilename(baseFilename);
}
writeRowToFile(data.outputRowMeta, r);
putRow(data.outputRowMeta, r); // in case we want it to go further...
if (checkFeedback(getLinesOutput())) logBasic("linenr " + getLinesOutput());
return result;
}
public void openNewFile(String baseFilename) throws KettleException {
data.writer = null;
ResultFile resultFile = null;
String filename = buildFilename(environmentSubstitute(baseFilename), true);
try {
if (meta.isFileAsCommand()) {
if (log.isDebug()) logDebug("Spawning external process");
if (data.cmdProc != null) {
logError("Previous command not correctly terminated");
setErrors(1);
}
String cmdstr = environmentSubstitute(meta.getFileName());
if (Const.getOS().equals("Windows 95")) {
cmdstr = "command.com /C " + cmdstr;
} else {
if (Const.getOS().startsWith("Windows")) {
cmdstr = "cmd.exe /C " + cmdstr;
}
}
if (log.isDetailed()) logDetailed("Starting: " + cmdstr);
Runtime r = Runtime.getRuntime();
data.cmdProc = r.exec(cmdstr, EnvUtil.getEnvironmentVariablesForRuntimeExec());
data.writer = data.cmdProc.getOutputStream();
StreamLogger stdoutLogger = new StreamLogger(data.cmdProc.getInputStream(), "(stdout)");
StreamLogger stderrLogger = new StreamLogger(data.cmdProc.getErrorStream(), "(stderr)");
new Thread(stdoutLogger).start();
new Thread(stderrLogger).start();
} else {
// Add this to the result file names...
resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL,
KettleVFS.getFileObject(filename),
getTransMeta().getName(),
getStepname());
resultFile.setComment("This file was created with a text file output step");
addResultFile(resultFile);
OutputStream outputStream;
if (!Const.isEmpty(meta.getFileCompression())
&& !meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_NONE)) {
if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_ZIP)) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in zipped mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
data.zip = new ZipOutputStream(data.fos);
File entry = new File(filename);
ZipEntry zipentry = new ZipEntry(entry.getName());
zipentry.setComment("Compressed by Kettle");
data.zip.putNextEntry(zipentry);
outputStream = data.zip;
} else if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_GZIP)) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in gzipped mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
data.gzip = new GZIPOutputStream(data.fos);
outputStream = data.gzip;
} else {
throw new KettleFileException("No compression method specified!");
}
} else {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in nocompress mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
outputStream = data.fos;
}
if (!Const.isEmpty(meta.getEncoding())) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in encoding: " + meta.getEncoding());
data.writer = new BufferedOutputStream(outputStream, 5000);
} else {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in default encoding");
data.writer = new BufferedOutputStream(outputStream, 5000);
}
if (log.isDetailed()) logDetailed("Opened new file with name [" + filename + "]");
}
} catch (Exception e) {
throw new KettleException("Error opening new file : " + e.toString());
}
// System.out.println("end of newFile(), splitnr="+splitnr);
data.splitnr++;
if (resultFile != null && meta.isAddToResultFiles()) {
// Add this to the result file names...
addResultFile(resultFile);
}
}
