public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
meta = (TextFileOutputMeta) smi;
data = (TextFileOutputData) sdi;
if (super.init(smi, sdi)) {
data.splitnr = 0;
// In case user want to create file at first row
// In that case, DO NOT create file at Init
if (!meta.isDoNotOpenNewFileInit()) {
try {
if (!meta.isFileNameInField()) {
openNewFile(meta.getFileName());
}
data.oneFileOpened = true;
} catch (Exception e) {
logError("Couldn't open file " + meta.getFileName(), e);
setErrors(1L);
stopAll();
}
}
try {
initBinaryDataFields();
} catch (Exception e) {
logError("Couldn't initialize binary data fields", e);
setErrors(1L);
stopAll();
}
return true;
}
return false;
}
private void writeField(ValueMetaInterface v, Object valueData, byte[] nullString)
throws KettleStepException {
try {
byte[] str;
// First check whether or not we have a null string set
// These values should be set when a null value passes
//
if (nullString != null && v.isNull(valueData)) {
str = nullString;
} else {
if (meta.isFastDump()) {
if (valueData instanceof byte[]) {
str = (byte[]) valueData;
} else {
str = getBinaryString((valueData == null) ? "" : valueData.toString());
}
} else {
str = formatField(v, valueData);
}
}
if (str != null && str.length > 0) {
List<Integer> enclosures = null;
if (v.isString() && meta.isEnclosureForced() && !meta.isPadded()) {
data.writer.write(data.binaryEnclosure);
// Also check for the existence of the enclosure character.
// If needed we double (escape) the enclosure character.
//
enclosures = getEnclosurePositions(str);
}
if (enclosures == null) {
data.writer.write(str);
} else {
// Skip the enclosures, double them instead...
int from = 0;
for (int i = 0; i < enclosures.size(); i++) {
int position = enclosures.get(i);
data.writer.write(str, from, position + data.binaryEnclosure.length - from);
data.writer.write(data.binaryEnclosure); // write enclosure a second time
from = position + data.binaryEnclosure.length;
}
if (from < str.length) {
data.writer.write(str, from, str.length - from);
}
}
if (v.isString() && meta.isEnclosureForced() && !meta.isPadded()) {
data.writer.write(data.binaryEnclosure);
}
}
} catch (Exception e) {
throw new KettleStepException("Error writing field content to file", e);
}
}
public String buildFilename(String filename, boolean ziparchive) {
return TextFileOutputMeta.buildFilename(
filename,
meta.getExtension(),
this,
getCopy(),
getPartitionID(),
data.splitnr,
ziparchive,
meta);
}
public Object clone() {
TextFileOutputMeta retval = (TextFileOutputMeta) super.clone();
int nrfields = outputFields.length;
retval.allocate(nrfields);
for (int i = 0; i < nrfields; i++) {
retval.outputFields[i] = (TextFileField) outputFields[i].clone();
}
return retval;
}
private boolean closeFile() {
boolean retval = false;
try {
if (data.writer != null) {
if (log.isDebug()) logDebug("Closing output stream");
data.writer.close();
if (log.isDebug()) logDebug("Closed output stream");
}
data.writer = null;
if (data.cmdProc != null) {
if (log.isDebug()) logDebug("Ending running external command");
int procStatus = data.cmdProc.waitFor();
// close the streams
// otherwise you get "Too many open files, java.io.IOException" after a lot of iterations
try {
data.cmdProc.getErrorStream().close();
data.cmdProc.getOutputStream().close();
data.cmdProc.getInputStream().close();
} catch (IOException e) {
if (log.isDetailed()) logDetailed("Warning: Error closing streams: " + e.getMessage());
}
data.cmdProc = null;
if (log.isBasic() && procStatus != 0) logBasic("Command exit status: " + procStatus);
} else {
if (log.isDebug()) logDebug("Closing normal file ...");
if (FILE_COMPRESSION_TYPE_ZIP.equals(meta.getFileCompression())) {
data.zip.closeEntry();
data.zip.finish();
data.zip.close();
} else if (FILE_COMPRESSION_TYPE_GZIP.equals(meta.getFileCompression())) {
data.gzip.finish();
}
if (data.fos != null) {
data.fos.close();
data.fos = null;
}
}
retval = true;
} catch (Exception e) {
logError("Exception trying to close file: " + e.toString());
setErrors(1);
retval = false;
}
return retval;
}
private boolean writeHeader() {
boolean retval = false;
RowMetaInterface r = data.outputRowMeta;
try {
// If we have fields specified: list them in this order!
if (meta.getOutputFields() != null && meta.getOutputFields().length > 0) {
for (int i = 0; i < meta.getOutputFields().length; i++) {
String fieldName = meta.getOutputFields()[i].getName();
ValueMetaInterface v = r.searchValueMeta(fieldName);
if (i > 0 && data.binarySeparator.length > 0) {
data.writer.write(data.binarySeparator);
}
if (meta.isEnclosureForced()
&& data.binaryEnclosure.length > 0
&& v != null
&& v.isString()) {
data.writer.write(data.binaryEnclosure);
}
data.writer.write(getBinaryString(fieldName));
if (meta.isEnclosureForced()
&& data.binaryEnclosure.length > 0
&& v != null
&& v.isString()) {
data.writer.write(data.binaryEnclosure);
}
}
data.writer.write(data.binaryNewline);
} else if (r != null) // Just put all field names in the header/footer
{
for (int i = 0; i < r.size(); i++) {
if (i > 0 && data.binarySeparator.length > 0) {
data.writer.write(data.binarySeparator);
}
ValueMetaInterface v = r.getValueMeta(i);
if (meta.isEnclosureForced() && data.binaryEnclosure.length > 0 && v.isString()) {
data.writer.write(data.binaryEnclosure);
}
data.writer.write(getBinaryString(v.getName()));
if (meta.isEnclosureForced() && data.binaryEnclosure.length > 0 && v.isString()) {
data.writer.write(data.binaryEnclosure);
}
}
data.writer.write(data.binaryNewline);
} else {
data.writer.write(getBinaryString("no rows selected" + Const.CR));
}
} catch (Exception e) {
logError("Error writing header line: " + e.toString());
logError(Const.getStackTracker(e));
retval = true;
}
incrementLinesOutput();
return retval;
}
/**
* This method should only be used when you have a filename in the input stream.
*
* @param filename the filename to set the data.writer field for
* @throws KettleException
*/
private void setDataWriterForFilename(String filename) throws KettleException {
// First handle the writers themselves.
// If we didn't have a writer yet, we create one.
// Basically we open a new file
//
data.writer = data.fileWriterMap.get(filename);
if (data.writer == null) {
openNewFile(filename);
data.fileWriterMap.put(filename, data.writer);
// If it's the first time we open it and we have a header, we write a header...
//
if (!meta.isFileAppended() && meta.isHeaderEnabled()) {
if (writeHeader()) {
incrementLinesOutput();
}
}
}
}
private byte[] getBinaryString(String string) throws KettleStepException {
try {
if (data.hasEncoding) {
return string.getBytes(meta.getEncoding());
} else {
return string.getBytes();
}
} catch (Exception e) {
throw new KettleStepException(e);
}
}
private void writeRowToFile(RowMetaInterface rowMeta, Object[] r) throws KettleStepException {
try {
if (meta.getOutputFields() == null || meta.getOutputFields().length == 0) {
/*
* Write all values in stream to text file.
*/
for (int i = 0; i < rowMeta.size(); i++) {
if (i > 0 && data.binarySeparator.length > 0) {
data.writer.write(data.binarySeparator);
}
ValueMetaInterface v = rowMeta.getValueMeta(i);
Object valueData = r[i];
// no special null value default was specified since no fields are specified at all
// As such, we pass null
//
writeField(v, valueData, null);
}
data.writer.write(data.binaryNewline);
} else {
/*
* Only write the fields specified!
*/
for (int i = 0; i < meta.getOutputFields().length; i++) {
if (i > 0 && data.binarySeparator.length > 0) data.writer.write(data.binarySeparator);
ValueMetaInterface v = rowMeta.getValueMeta(data.fieldnrs[i]);
Object valueData = r[data.fieldnrs[i]];
writeField(v, valueData, data.binaryNullValue[i]);
}
data.writer.write(data.binaryNewline);
}
incrementLinesOutput();
// flush every 4k lines
// if (linesOutput>0 && (linesOutput&0xFFF)==0) data.writer.flush();
} catch (Exception e) {
throw new KettleStepException("Error writing line", e);
}
}
private boolean writeEndedLine() {
boolean retval = false;
try {
String sLine = meta.getEndedLine();
if (sLine != null) {
if (sLine.trim().length() > 0) {
data.writer.write(getBinaryString(sLine));
incrementLinesOutput();
}
}
} catch (Exception e) {
logError("Error writing ended tag line: " + e.toString());
logError(Const.getStackTracker(e));
retval = true;
}
return retval;
}
private void initBinaryDataFields() throws KettleException {
try {
data.hasEncoding = !Const.isEmpty(meta.getEncoding());
data.binarySeparator = new byte[] {};
data.binaryEnclosure = new byte[] {};
data.binaryNewline = new byte[] {};
if (data.hasEncoding) {
if (!Const.isEmpty(meta.getSeparator()))
data.binarySeparator =
environmentSubstitute(meta.getSeparator()).getBytes(meta.getEncoding());
if (!Const.isEmpty(meta.getEnclosure()))
data.binaryEnclosure =
environmentSubstitute(meta.getEnclosure()).getBytes(meta.getEncoding());
if (!Const.isEmpty(meta.getNewline()))
data.binaryNewline = meta.getNewline().getBytes(meta.getEncoding());
} else {
if (!Const.isEmpty(meta.getSeparator()))
data.binarySeparator = environmentSubstitute(meta.getSeparator()).getBytes();
if (!Const.isEmpty(meta.getEnclosure()))
data.binaryEnclosure = environmentSubstitute(meta.getEnclosure()).getBytes();
if (!Const.isEmpty(meta.getNewline()))
data.binaryNewline = environmentSubstitute(meta.getNewline()).getBytes();
}
data.binaryNullValue = new byte[meta.getOutputFields().length][];
for (int i = 0; i < meta.getOutputFields().length; i++) {
data.binaryNullValue[i] = null;
String nullString = meta.getOutputFields()[i].getNullString();
if (!Const.isEmpty(nullString)) {
if (data.hasEncoding) {
data.binaryNullValue[i] = nullString.getBytes(meta.getEncoding());
} else {
data.binaryNullValue[i] = nullString.getBytes();
}
}
}
} catch (Exception e) {
throw new KettleException("Unexpected error while encoding binary fields", e);
}
}
public synchronized boolean processRow(StepMetaInterface smi, StepDataInterface sdi)
throws KettleException {
meta = (TextFileOutputMeta) smi;
data = (TextFileOutputData) sdi;
boolean result = true;
boolean bEndedLineWrote = false;
Object[] r = getRow(); // This also waits for a row to be finished.
if (r != null && first) {
first = false;
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
// if file name in field is enabled then set field name and open file
//
if (meta.isFileNameInField()) {
// find and set index of file name field in input stream
//
data.fileNameFieldIndex = getInputRowMeta().indexOfValue(meta.getFileNameField());
// set the file name for this row
//
if (data.fileNameFieldIndex < 0) {
throw new KettleStepException(
Messages.getString(
"TextFileOutput.Exception.FileNameFieldNotFound",
meta.getFileNameField())); // $NON-NLS-1$
}
data.fileNameMeta = getInputRowMeta().getValueMeta(data.fileNameFieldIndex);
data.fileName = data.fileNameMeta.getString(r[data.fileNameFieldIndex]);
setDataWriterForFilename(data.fileName);
} else if (meta.isDoNotOpenNewFileInit() && !meta.isFileNameInField()) {
// Open a new file here
//
openNewFile(meta.getFileName());
data.oneFileOpened = true;
initBinaryDataFields();
}
if (!meta.isFileAppended()
&& (meta.isHeaderEnabled()
|| meta.isFooterEnabled())) // See if we have to write a header-line)
{
if (!meta.isFileNameInField() && meta.isHeaderEnabled() && data.outputRowMeta != null) {
writeHeader();
}
}
data.fieldnrs = new int[meta.getOutputFields().length];
for (int i = 0; i < meta.getOutputFields().length; i++) {
data.fieldnrs[i] = data.outputRowMeta.indexOfValue(meta.getOutputFields()[i].getName());
if (data.fieldnrs[i] < 0) {
throw new KettleStepException(
"Field ["
+ meta.getOutputFields()[i].getName()
+ "] couldn't be found in the input stream!");
}
}
}
if ((r == null && data.outputRowMeta != null && meta.isFooterEnabled())
|| (r != null
&& getLinesOutput() > 0
&& meta.getSplitEvery() > 0
&& ((getLinesOutput() + 1) % meta.getSplitEvery()) == 0)) {
if (data.outputRowMeta != null) {
if (meta.isFooterEnabled()) {
writeHeader();
}
}
if (r == null) {
// add tag to last line if needed
writeEndedLine();
bEndedLineWrote = true;
}
// Done with this part or with everything.
closeFile();
// Not finished: open another file...
if (r != null) {
openNewFile(meta.getFileName());
if (meta.isHeaderEnabled() && data.outputRowMeta != null)
if (writeHeader()) incrementLinesOutput();
}
}
if (r == null) // no more input to be expected...
{
if (false == bEndedLineWrote) {
// add tag to last line if needed
writeEndedLine();
bEndedLineWrote = true;
}
setOutputDone();
return false;
}
// First handle the file name in field
// Write a header line as well if needed
//
if (meta.isFileNameInField()) {
String baseFilename = data.fileNameMeta.getString(r[data.fileNameFieldIndex]);
setDataWriterForFilename(baseFilename);
}
writeRowToFile(data.outputRowMeta, r);
putRow(data.outputRowMeta, r); // in case we want it to go further...
if (checkFeedback(getLinesOutput())) logBasic("linenr " + getLinesOutput());
return result;
}
public void openNewFile(String baseFilename) throws KettleException {
data.writer = null;
ResultFile resultFile = null;
String filename = buildFilename(environmentSubstitute(baseFilename), true);
try {
if (meta.isFileAsCommand()) {
if (log.isDebug()) logDebug("Spawning external process");
if (data.cmdProc != null) {
logError("Previous command not correctly terminated");
setErrors(1);
}
String cmdstr = environmentSubstitute(meta.getFileName());
if (Const.getOS().equals("Windows 95")) {
cmdstr = "command.com /C " + cmdstr;
} else {
if (Const.getOS().startsWith("Windows")) {
cmdstr = "cmd.exe /C " + cmdstr;
}
}
if (log.isDetailed()) logDetailed("Starting: " + cmdstr);
Runtime r = Runtime.getRuntime();
data.cmdProc = r.exec(cmdstr, EnvUtil.getEnvironmentVariablesForRuntimeExec());
data.writer = data.cmdProc.getOutputStream();
StreamLogger stdoutLogger = new StreamLogger(data.cmdProc.getInputStream(), "(stdout)");
StreamLogger stderrLogger = new StreamLogger(data.cmdProc.getErrorStream(), "(stderr)");
new Thread(stdoutLogger).start();
new Thread(stderrLogger).start();
} else {
// Add this to the result file names...
resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL,
KettleVFS.getFileObject(filename),
getTransMeta().getName(),
getStepname());
resultFile.setComment("This file was created with a text file output step");
addResultFile(resultFile);
OutputStream outputStream;
if (!Const.isEmpty(meta.getFileCompression())
&& !meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_NONE)) {
if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_ZIP)) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in zipped mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
data.zip = new ZipOutputStream(data.fos);
File entry = new File(filename);
ZipEntry zipentry = new ZipEntry(entry.getName());
zipentry.setComment("Compressed by Kettle");
data.zip.putNextEntry(zipentry);
outputStream = data.zip;
} else if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_GZIP)) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in gzipped mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
data.gzip = new GZIPOutputStream(data.fos);
outputStream = data.gzip;
} else {
throw new KettleFileException("No compression method specified!");
}
} else {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in nocompress mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
outputStream = data.fos;
}
if (!Const.isEmpty(meta.getEncoding())) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in encoding: " + meta.getEncoding());
data.writer = new BufferedOutputStream(outputStream, 5000);
} else {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in default encoding");
data.writer = new BufferedOutputStream(outputStream, 5000);
}
if (log.isDetailed()) logDetailed("Opened new file with name [" + filename + "]");
}
} catch (Exception e) {
throw new KettleException("Error opening new file : " + e.toString());
}
// System.out.println("end of newFile(), splitnr="+splitnr);
data.splitnr++;
if (resultFile != null && meta.isAddToResultFiles()) {
// Add this to the result file names...
addResultFile(resultFile);
}
}
public String buildFilename(
String filename,
String extension,
VariableSpace space,
int stepnr,
String partnr,
int splitnr,
boolean ziparchive,
TextFileOutputMeta meta) {
SimpleDateFormat daf = new SimpleDateFormat();
// Replace possible environment variables...
String retval = space.environmentSubstitute(filename);
String realextension = space.environmentSubstitute(extension);
if (meta.isFileAsCommand()) {
return retval;
}
Date now = new Date();
if (meta.isSpecifyingFormat() && !Const.isEmpty(meta.getDateTimeFormat())) {
daf.applyPattern(meta.getDateTimeFormat());
String dt = daf.format(now);
retval += dt;
} else {
if (meta.isDateInFilename()) {
daf.applyPattern("yyyMMdd");
String d = daf.format(now);
retval += "_" + d;
}
if (meta.isTimeInFilename()) {
daf.applyPattern("HHmmss");
String t = daf.format(now);
retval += "_" + t;
}
}
if (meta.isStepNrInFilename()) {
retval += "_" + stepnr;
}
if (meta.isPartNrInFilename()) {
retval += "_" + partnr;
}
if (meta.getSplitEvery() > 0) {
retval += "_" + splitnr;
}
if (meta.getFileCompression().equals("Zip")) {
if (ziparchive) {
retval += ".zip";
} else {
if (realextension != null && realextension.length() != 0) {
retval += "." + realextension;
}
}
} else {
if (realextension != null && realextension.length() != 0) {
retval += "." + realextension;
}
if (meta.getFileCompression().equals("GZip")) {
retval += ".gz";
}
}
return retval;
}