/**
* Finds the tab for the transformation that matches the metadata provided (either the file must
* be the same or the repository id).
*
* @param trans Transformation metadata to look for
* @return Tab with transformation open whose metadata matches {@code trans} or {@code null} if no
* tab exists.
* @throws KettleFileException If there is a problem loading the file object for an open
* transformation with an invalid a filename.
*/
public TabMapEntry findTabForTransformation(TransMeta trans) throws KettleFileException {
// File for the transformation we're looking for. It will be loaded upon first request.
FileObject transFile = null;
for (TabMapEntry entry : tabMap) {
if (entry == null || entry.getTabItem().isDisposed()) {
continue;
}
if (trans.getFilename() != null && entry.getFilename() != null) {
// If the entry has a file name it is the same as trans iff. they originated from the same
// files
FileObject entryFile = KettleVFS.getFileObject(entry.getFilename());
if (transFile == null) {
transFile = KettleVFS.getFileObject(trans.getFilename());
}
if (entryFile.equals(transFile)) {
return entry;
}
} else if (trans.getObjectId() != null && entry.getObject() != null) {
EngineMetaInterface meta = entry.getObject().getMeta();
if (meta != null && trans.getObjectId().equals(meta.getObjectId())) {
// If the transformation has an object id and the entry shares the same id they are the
// same
return entry;
}
}
}
// No tabs for the transformation exist and are not disposed
return null;
}
/**
* Create the command line for a psql process depending on the meta information supplied.
*
* @param meta The meta data to create the command line from
* @param password Use the real password or not
* @return The string to execute.
* @throws KettleException Upon any exception
*/
public String createCommandLine(GPLoadMeta meta, boolean password) throws KettleException {
StringBuffer sb = new StringBuffer(300);
if (meta.getGploadPath() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getGploadPath()), getTransMeta());
String psqlexec = KettleVFS.getFilename(fileObject);
// sb.append('\'').append(psqlexec).append('\'');
sb.append(psqlexec);
} catch (Exception ex) {
throw new KettleException("Error retrieving sqlldr string", ex);
}
} else {
throw new KettleException("No psql application specified");
}
if (meta.getControlFile() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getControlFile()), getTransMeta());
sb.append(" -f ");
// sb.append('\'').append(KettleVFS.getFilename(fileObject)).append('\'');
sb.append(KettleVFS.getFilename(fileObject));
} catch (Exception ex) {
throw new KettleException("Error retrieving controlfile string", ex);
}
} else {
throw new KettleException("No control file specified");
}
if (meta.getLogFile() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getLogFile()), getTransMeta());
sb.append(" -l ");
sb.append('\'').append(KettleVFS.getFilename(fileObject)).append('\'');
} catch (Exception ex) {
throw new KettleException("Error retrieving logfile string", ex);
}
}
// hostname, port and so on are passed through the control file
//
return sb.toString();
}
private boolean CreateFolder(String filefolder) {
FileObject folder = null;
try {
folder = KettleVFS.getFileObject(filefolder, this);
if (!folder.exists()) {
if (createtargetfolder) {
folder.createFolder();
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(PKG, "JobSSH2GET.Log.FolderCreated", folder.toString()));
}
} else {
return false;
}
}
return true;
} catch (Exception e) {
logError(
BaseMessages.getString(PKG, "JobSSH2GET.Log.CanNotCreateFolder", folder.toString()), e);
} finally {
if (folder != null) {
try {
folder.close();
} catch (Exception ex) {
/* Ignore */
}
}
}
return false;
}
private boolean OpenFile() throws Exception {
data.oneFileOpened = true;
String realFilename = environmentSubstitute(meta.getFilename());
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "AccessOutput.log.WritingToFile", realFilename));
}
FileObject fileObject = KettleVFS.getFileObject(realFilename, getTransMeta());
File file = FileUtils.toFile(fileObject.getURL());
// First open or create the access file
if (!file.exists()) {
if (meta.isFileCreated()) {
data.db = Database.create(file);
} else {
logError(
BaseMessages.getString(PKG, "AccessOutput.InitError.FileDoesNotExist", realFilename));
return false;
}
} else {
data.db = Database.open(file);
}
// Add the filename to the result object...
//
if (meta.isAddToResultFiles()) {
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL, fileObject, getTransMeta().getName(), toString());
resultFile.setComment("This file was created with an access output step");
addResultFile(resultFile);
}
return true;
}
/**
* Since the exported transformation that runs this will reside in a ZIP file, we can't reference
* files relatively. So what this does is turn the name of the base path into an absolute path.
*
* @param space the variable space to use
* @param definitions
* @param resourceNamingInterface
* @param repository The repository to optionally load other resources from (to be converted to
* XML)
* @param metaStore the metaStore in which non-kettle metadata could reside.
* @return the filename of the exported resource
*/
public String exportResources(
VariableSpace space,
Map<String, ResourceDefinition> definitions,
ResourceNamingInterface resourceNamingInterface,
Repository repository,
IMetaStore metaStore)
throws KettleException {
try {
// The object that we're modifying here is a copy of the original!
// So let's change the filename from relative to absolute by grabbing the file object...
// In case the name of the file comes from previous steps, forget about this!
//
if (!fileNameInField) {
if (!Const.isEmpty(fileName)) {
FileObject fileObject =
KettleVFS.getFileObject(space.environmentSubstitute(fileName), space);
fileName = resourceNamingInterface.nameResource(fileObject, space, true);
}
}
return null;
} catch (Exception e) {
throw new KettleException(e);
}
}
@Test
public void testZeroSizeFile() throws Exception {
ByteArrayOutputStream log = new ByteArrayOutputStream();
helper.redirectLog(log, LogLevel.BASIC);
try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.json");
LocaleChange enUs = new LocaleChange(Locale.US); ) {
fileObj.createFile();
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
JsonInputMeta meta = createSimpleMeta("in file", price);
meta.setIsAFile(true);
meta.setRemoveSourceField(true);
meta.setIgnoreEmptyFile(false);
JsonInput jsonInput =
createJsonInput(
"in file", meta, new Object[][] {new Object[] {BASE_RAM_DIR + "test.json"}});
processRows(jsonInput, 1);
String logMsgs = log.toString();
assertTrue(logMsgs, logMsgs.contains("is empty!"));
} finally {
deleteFiles();
}
}
private boolean deleteOrMoveFiles(FileObject file, String destinationFolder)
throws KettleException {
try {
boolean retval = false;
// Delete the file if this is needed!
//
if (afterFtpPut.equals("delete_file")) {
file.delete();
retval = true;
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DeletedFile", file.toString()));
}
} else if (afterFtpPut.equals("move_file")) {
// Move File
FileObject destination = null;
FileObject source = null;
try {
destination =
KettleVFS.getFileObject(
destinationFolder + Const.FILE_SEPARATOR + file.getName().getBaseName(), this);
file.moveTo(destination);
retval = true;
} catch (Exception e) {
logError(
BaseMessages.getString(
PKG,
"JobSSH2PUT.Cant_Move_File.Label",
file.toString(),
destinationFolder,
e.getMessage()));
} finally {
if (destination != null) {
try {
destination.close();
} catch (Exception ex) {
/* Ignore */
}
}
if (source != null) {
try {
source.close();
} catch (Exception ex) {
/* Ignore */
}
}
}
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.MovedFile", file.toString(), ftpDirectory));
}
}
return retval;
} catch (Exception e) {
throw new KettleException(e);
}
}
private void ProcessFile(String filename, String wildcard) {
FileObject filefolder = null;
String realFilefoldername = environmentSubstitute(filename);
String realwilcard = environmentSubstitute(wildcard);
try {
filefolder = KettleVFS.getFileObject(realFilefoldername);
FileObject[] files = new FileObject[] {filefolder};
if (filefolder.exists()) {
// the file or folder exists
if (filefolder.getType() == FileType.FOLDER) {
// It's a folder
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryCheckFilesLocked.ProcessingFolder", realFilefoldername));
}
// Retrieve all files
files = filefolder.findFiles(new TextFileSelector(filefolder.toString(), realwilcard));
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobEntryCheckFilesLocked.TotalFilesToCheck",
String.valueOf(files.length)));
}
} else {
// It's a file
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryCheckFilesLocked.ProcessingFile", realFilefoldername));
}
}
// Check files locked
checkFilesLocked(files);
} else {
// We can not find thsi file
logBasic(
BaseMessages.getString(
PKG, "JobEntryCheckFilesLocked.FileNotExist", realFilefoldername));
}
} catch (Exception e) {
logError(
BaseMessages.getString(
PKG, "JobEntryCheckFilesLocked.CouldNotProcess", realFilefoldername, e.getMessage()));
} finally {
if (filefolder != null) {
try {
filefolder.close();
} catch (IOException ex) {
// Ignore
}
}
}
}
@Test
public void testFileList() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input1 = getBasicTestJson();
final String input2 = "{ \"store\": { \"book\": [ { \"price\": 9.99 } ] } }";
try (FileObject fileObj1 = KettleVFS.getFileObject(BASE_RAM_DIR + "test1.json");
FileObject fileObj2 = KettleVFS.getFileObject(BASE_RAM_DIR + "test2.json")) {
try (OutputStream out = fileObj1.getContent().getOutputStream()) {
out.write(input1.getBytes());
}
try (OutputStream out = fileObj2.getContent().getOutputStream()) {
out.write(input2.getBytes());
}
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
List<FileObject> fileList = Arrays.asList(fileObj1, fileObj2);
JsonInputMeta meta = createFileListMeta(fileList);
meta.setInputFields(new JsonInputField[] {price});
meta.setIncludeRowNumber(true);
meta.setRowNumberField("rownbr");
meta.setShortFileNameField("fname");
JsonInput jsonInput = createJsonInput(meta);
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {8.95d, 1L, "test1.json"},
new Object[] {12.99d, 2L, "test1.json"},
new Object[] {8.99d, 3L, "test1.json"},
new Object[] {22.99d, 4L, "test1.json"},
new Object[] {9.99d, 5L, "test2.json"});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
disposeJsonInput(jsonInput);
assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
public void dispose(StepMetaInterface smi, StepDataInterface sdi) {
meta = (GPLoadMeta) smi;
data = (GPLoadData) sdi;
super.dispose(smi, sdi);
if (!preview && meta.isEraseFiles()) {
// Erase the created cfg/dat files if requested. We don't erase
// the rest of the files because it would be "stupid" to erase them
// right after creation. If you don't want them, don't fill them in.
FileObject fileObject = null;
String method = meta.getLoadMethod();
if ( // GPLoadMeta.METHOD_AUTO_CONCURRENT.equals(method) ||
GPLoadMeta.METHOD_AUTO_END.equals(method)) {
/*
if ( meta.getControlFile() != null )
{
try
{
fileObject = KettleVFS.getFileObject(environmentSubstitute(meta.getControlFile()), getTransMeta());
fileObject.delete();
fileObject.close();
}
catch ( Exception ex )
{
logError("Error deleting control file \'" + KettleVFS.getFilename(fileObject) + "\': " + ex.getMessage());
}
}*/
}
if (GPLoadMeta.METHOD_AUTO_END.equals(method)) {
// In concurrent mode the data is written to the control file.
if (meta.getDataFile() != null) {
try {
fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getDataFile()), getTransMeta());
fileObject.delete();
fileObject.close();
} catch (Exception ex) {
logError(
"Error deleting data file \'"
+ KettleVFS.getFilename(fileObject)
+ "\': "
+ ex.getMessage(),
ex);
}
}
}
if (GPLoadMeta.METHOD_MANUAL.equals(method)) {
logBasic("Deletion of files is not compatible with \'manual load method\'");
}
}
}
/**
* Log only lines belonging to the specified log channel ID or one of it's children
* (grandchildren) to the specified file.
*
* @param logChannelId
* @param filename
* @param append
* @throws KettleException
*/
public FileLoggingEventListener(String logChannelId, String filename, boolean append)
throws KettleException {
this.logChannelId = logChannelId;
this.filename = filename;
this.layout = new KettleLogLayout(true);
this.exception = null;
file = KettleVFS.getFileObject(filename);
outputStream = null;
try {
outputStream = KettleVFS.getOutputStream(file, append);
} catch (Exception e) {
throw new KettleException(
"Unable to create a logging event listener to write to file '" + filename + "'", e);
}
}
/**
* Check existence of a local file
*
* @param filename
* @return true, if file exists
*/
public boolean FileExists(String filename) {
FileObject file = null;
try {
file = KettleVFS.getFileObject(filename, this);
if (!file.exists()) {
return false;
} else {
if (file.getType() == FileType.FILE) {
return true;
} else {
return false;
}
}
} catch (Exception e) {
return false;
}
}
@Test
public void testZipFileInput() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input = getBasicTestJson();
try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.zip")) {
fileObj.createFile();
try (OutputStream out = fileObj.getContent().getOutputStream()) {
try (ZipOutputStream zipOut = new ZipOutputStream(out)) {
ZipEntry jsonFile = new ZipEntry("test.json");
zipOut.putNextEntry(jsonFile);
zipOut.write(input.getBytes());
zipOut.closeEntry();
zipOut.flush();
}
}
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
JsonInputMeta meta = createSimpleMeta("in file", price);
meta.setIsAFile(true);
meta.setRemoveSourceField(true);
JsonInput jsonInput =
createJsonInput(
"in file",
meta,
new Object[][] {new Object[] {"zip:" + BASE_RAM_DIR + "test.zip!/test.json"}});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {8.95d},
new Object[] {12.99d},
new Object[] {8.99d},
new Object[] {22.99d});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
Assert.assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
/**
* Init Helper Class with connection settings
*
* @param serverIP IP address of remote server
* @param serverPort port of remote server
* @param userName username of remote server
* @param privateKeyFilename filename of private key
* @param passPhrase passphrase
* @throws KettleJobException
*/
public SFTPClient(
InetAddress serverIP,
int serverPort,
String userName,
String privateKeyFilename,
String passPhrase)
throws KettleJobException {
if (serverIP == null || serverPort < 0 || userName == null || userName.equals("")) {
throw new KettleJobException(
"For a SFTP connection server name and username must be set and server port must be greater than zero.");
}
this.serverIP = serverIP;
this.serverPort = serverPort;
this.userName = userName;
JSch jsch = new JSch();
try {
if (!Const.isEmpty(privateKeyFilename)) {
// We need to use private key authentication
this.prvkey = privateKeyFilename;
byte[] passphrasebytes = new byte[0];
if (!Const.isEmpty(passPhrase)) {
// Set passphrase
this.passphrase = passPhrase;
passphrasebytes = GetPrivateKeyPassPhrase().getBytes();
}
jsch.addIdentity(
getUserName(),
FileUtil.getContent(KettleVFS.getFileObject(prvkey)), // byte[] privateKey
null, // byte[] publicKey
passphrasebytes); // byte[] passPhrase
}
s = jsch.getSession(userName, serverIP.getHostAddress(), serverPort);
} catch (IOException e) {
throw new KettleJobException(e);
} catch (KettleFileException e) {
throw new KettleJobException(e);
} catch (JSchException e) {
throw new KettleJobException(e);
}
}
private List<FileObject> getFiles(String localfolder) throws KettleFileException {
try {
List<FileObject> myFileList = new ArrayList<FileObject>();
// Get all the files in the local directory...
FileObject localFiles = KettleVFS.getFileObject(localfolder, this);
FileObject[] children = localFiles.getChildren();
if (children != null) {
for (int i = 0; i < children.length; i++) {
// Get filename of file or directory
if (children[i].getType().equals(FileType.FILE)) {
myFileList.add(children[i]);
}
} // end for
}
return myFileList;
} catch (IOException e) {
throw new KettleFileException(e);
}
}
/**
* Since the exported transformation that runs this will reside in a ZIP file, we can't reference
* files relatively. So what this does is turn the name of files into absolute paths OR it simply
* includes the resource in the ZIP file. For now, we'll simply turn it into an absolute path and
* pray that the file is on a shared drive or something like that.
*
* <p>TODO: create options to configure this behavior
*/
public String exportResources(
VariableSpace space,
Map<String, ResourceDefinition> definitions,
ResourceNamingInterface resourceNamingInterface,
Repository repository)
throws KettleException {
try {
// The object that we're modifying here is a copy of the original!
// So let's change the filename from relative to absolute by grabbing the file object...
// In case the name of the file comes from previous steps, forget about this!
//
if (Const.isEmpty(filenameField)) {
// From : ${Internal.Transformation.Filename.Directory}/../foo/bar.csv
// To : /home/matt/test/files/foo/bar.csv
//
FileObject fileObject = KettleVFS.getFileObject(space.environmentSubstitute(filename));
// If the file doesn't exist, forget about this effort too!
//
if (fileObject.exists()) {
// Convert to an absolute path...
//
filename =
resourceNamingInterface.nameResource(
fileObject.getName().getBaseName(),
fileObject.getParent().getName().getPath(),
space.toString(),
FileNamingType.DATA_FILE);
return filename;
}
}
return null;
} catch (Exception e) {
throw new KettleException(e); // $NON-NLS-1$
}
}
public Result execute(Result previousResult, int nr, Repository rep, Job parentJob) {
LogWriter log = LogWriter.getInstance();
Result result = previousResult;
result.setResult(false);
if (filename != null) {
String realFilename = getRealFilename();
try {
FileObject file = KettleVFS.getFileObject(realFilename);
if (file.exists() && file.isReadable()) {
log.logDetailed(
toString(),
Messages.getString("JobEntryFileExists.File_Exists", realFilename)); // $NON-NLS-1$
result.setResult(true);
} else {
log.logDetailed(
toString(),
Messages.getString(
"JobEntryFileExists.File_Does_Not_Exist", realFilename)); // $NON-NLS-1$
}
} catch (IOException e) {
result.setNrErrors(1);
log.logError(
toString(),
Messages.getString(
"JobEntryFileExists.ERROR_0004_IO_Exception", e.toString())); // $NON-NLS-1$
}
} else {
result.setNrErrors(1);
log.logError(
toString(),
Messages.getString("JobEntryFileExists.ERROR_0005_No_Filename_Defined")); // $NON-NLS-1$
}
return result;
}
public void openNewFile(String baseFilename) throws KettleException {
data.writer = null;
ResultFile resultFile = null;
String filename = buildFilename(environmentSubstitute(baseFilename), true);
try {
if (meta.isFileAsCommand()) {
if (log.isDebug()) logDebug("Spawning external process");
if (data.cmdProc != null) {
logError("Previous command not correctly terminated");
setErrors(1);
}
String cmdstr = environmentSubstitute(meta.getFileName());
if (Const.getOS().equals("Windows 95")) {
cmdstr = "command.com /C " + cmdstr;
} else {
if (Const.getOS().startsWith("Windows")) {
cmdstr = "cmd.exe /C " + cmdstr;
}
}
if (log.isDetailed()) logDetailed("Starting: " + cmdstr);
Runtime r = Runtime.getRuntime();
data.cmdProc = r.exec(cmdstr, EnvUtil.getEnvironmentVariablesForRuntimeExec());
data.writer = data.cmdProc.getOutputStream();
StreamLogger stdoutLogger = new StreamLogger(data.cmdProc.getInputStream(), "(stdout)");
StreamLogger stderrLogger = new StreamLogger(data.cmdProc.getErrorStream(), "(stderr)");
new Thread(stdoutLogger).start();
new Thread(stderrLogger).start();
} else {
// Add this to the result file names...
resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL,
KettleVFS.getFileObject(filename),
getTransMeta().getName(),
getStepname());
resultFile.setComment("This file was created with a text file output step");
addResultFile(resultFile);
OutputStream outputStream;
if (!Const.isEmpty(meta.getFileCompression())
&& !meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_NONE)) {
if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_ZIP)) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in zipped mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
data.zip = new ZipOutputStream(data.fos);
File entry = new File(filename);
ZipEntry zipentry = new ZipEntry(entry.getName());
zipentry.setComment("Compressed by Kettle");
data.zip.putNextEntry(zipentry);
outputStream = data.zip;
} else if (meta.getFileCompression().equals(FILE_COMPRESSION_TYPE_GZIP)) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in gzipped mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
data.gzip = new GZIPOutputStream(data.fos);
outputStream = data.gzip;
} else {
throw new KettleFileException("No compression method specified!");
}
} else {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in nocompress mode");
if (checkPreviouslyOpened(filename)) {
data.fos = KettleVFS.getOutputStream(filename, true);
} else {
data.fos = KettleVFS.getOutputStream(filename, meta.isFileAppended());
}
outputStream = data.fos;
}
if (!Const.isEmpty(meta.getEncoding())) {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in encoding: " + meta.getEncoding());
data.writer = new BufferedOutputStream(outputStream, 5000);
} else {
if (log.isDetailed())
log.logDetailed(toString(), "Opening output stream in default encoding");
data.writer = new BufferedOutputStream(outputStream, 5000);
}
if (log.isDetailed()) logDetailed("Opened new file with name [" + filename + "]");
}
} catch (Exception e) {
throw new KettleException("Error opening new file : " + e.toString());
}
// System.out.println("end of newFile(), splitnr="+splitnr);
data.splitnr++;
if (resultFile != null && meta.isAddToResultFiles()) {
// Add this to the result file names...
addResultFile(resultFile);
}
}
private boolean ReadNextString() {
try {
data.readrow = getRow(); // Grab another row ...
if (data.readrow == null) {
// finished processing!
if (log.isDetailed())
logDetailed(BaseMessages.getString(PKG, "JsonInput.Log.FinishedProcessing"));
return false;
}
if (first) {
first = false;
data.inputRowMeta = getInputRowMeta();
data.outputRowMeta = data.inputRowMeta.clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
// Get total previous fields
data.totalpreviousfields = data.inputRowMeta.size();
// Create convert meta-data objects that will contain Date & Number formatters
data.convertRowMeta = data.outputRowMeta.clone();
for (int i = 0; i < data.convertRowMeta.size(); i++)
data.convertRowMeta.getValueMeta(i).setType(ValueMetaInterface.TYPE_STRING);
// For String to <type> conversions, we allocate a conversion meta data row as well...
//
data.convertRowMeta = data.outputRowMeta.clone();
for (int i = 0; i < data.convertRowMeta.size(); i++) {
data.convertRowMeta.getValueMeta(i).setType(ValueMetaInterface.TYPE_STRING);
}
// Check if source field is provided
if (Const.isEmpty(meta.getFieldValue())) {
logError(BaseMessages.getString(PKG, "JsonInput.Log.NoField"));
throw new KettleException(BaseMessages.getString(PKG, "JsonInput.Log.NoField"));
}
// cache the position of the field
if (data.indexSourceField < 0) {
data.indexSourceField = getInputRowMeta().indexOfValue(meta.getFieldValue());
if (data.indexSourceField < 0) {
// The field is unreachable !
logError(
BaseMessages.getString(
PKG,
"JsonInput.Log.ErrorFindingField",
meta.getFieldValue())); // $NON-NLS-1$ //$NON-NLS-2$
throw new KettleException(
BaseMessages.getString(
PKG,
"JsonInput.Exception.CouldnotFindField",
meta.getFieldValue())); // $NON-NLS-1$ //$NON-NLS-2$
}
}
}
// get source field value
String fieldValue = getInputRowMeta().getString(data.readrow, data.indexSourceField);
if (log.isDetailed())
logDetailed(
BaseMessages.getString(
PKG, "JsonInput.Log.SourceValue", meta.getFieldValue(), fieldValue));
if (meta.getIsAFile()) {
// source is a file.
data.file = KettleVFS.getFileObject(fieldValue, getTransMeta());
if (meta.isIgnoreEmptyFile() && data.file.getContent().getSize() == 0) {
// log only basic as a warning (was before logError)
logBasic(
BaseMessages.getString(PKG, "JsonInput.Error.FileSizeZero", data.file.getName()));
ReadNextString();
}
} else {
// read string
data.stringToParse = fieldValue;
}
readFileOrString();
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JsonInput.Log.UnexpectedError", e.toString()));
stopAll();
logError(Const.getStackTracker(e));
setErrors(1);
return false;
}
return true;
}
public boolean openNewFile() {
boolean retval = false;
data.writer = null;
try {
FileObject file = KettleVFS.getFileObject(buildFilename(true), getTransMeta());
if (meta.isAddToResultFiles()) {
// Add this to the result file names...
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL, file, getTransMeta().getName(), getStepname());
resultFile.setComment("This file was created with a xml output step"); // $NON-NLS-1$
addResultFile(resultFile);
}
OutputStream outputStream;
if (meta.isZipped()) {
OutputStream fos = KettleVFS.getOutputStream(file, false);
data.zip = new ZipOutputStream(fos);
File entry = new File(buildFilename(false));
ZipEntry zipentry = new ZipEntry(entry.getName());
zipentry.setComment("Compressed by Kettle"); // $NON-NLS-1$
data.zip.putNextEntry(zipentry);
outputStream = data.zip;
} else {
OutputStream fos = KettleVFS.getOutputStream(file, false);
outputStream = fos;
}
if (meta.getEncoding() != null && meta.getEncoding().length() > 0) {
logBasic("Opening output stream in encoding: " + meta.getEncoding()); // $NON-NLS-1$
data.writer = new OutputStreamWriter(outputStream, meta.getEncoding());
data.writer.write(XMLHandler.getXMLHeader(meta.getEncoding()).toCharArray());
} else {
logBasic(
"Opening output stream in default encoding : " + Const.XML_ENCODING); // $NON-NLS-1$
data.writer = new OutputStreamWriter(outputStream);
data.writer.write(XMLHandler.getXMLHeader(Const.XML_ENCODING).toCharArray());
}
// Add the name space if defined
StringBuffer nameSpace = new StringBuffer();
if ((meta.getNameSpace() != null) && (!"".equals(meta.getNameSpace()))) { // $NON-NLS-1$
nameSpace.append(" xmlns=\""); // $NON-NLS-1$
nameSpace.append(meta.getNameSpace());
nameSpace.append("\""); // $NON-NLS-1$
}
// OK, write the header & the parent element:
data.writer.write(
("<" + meta.getMainElement() + nameSpace.toString() + ">" + Const.CR)
.toCharArray()); //$NON-NLS-1$//$NON-NLS-2$
retval = true;
} catch (Exception e) {
logError("Error opening new file : " + e.toString()); // $NON-NLS-1$
}
// System.out.println("end of newFile(), splitnr="+splitnr);
data.splitnr++;
return retval;
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (AutoDocMeta) smi;
data = (AutoDocData) sdi;
Object[] row = getRow();
if (row == null) {
if (data.filenames.isEmpty()) {
// Nothing to see here, move along!
//
setOutputDone();
return false;
}
// End of the line, create the documentation...
//
FileObject targetFile =
KettleVFS.getFileObject(environmentSubstitute(meta.getTargetFilename()));
String targetFilename = KettleVFS.getFilename(targetFile);
// Create the report builder
//
KettleReportBuilder kettleReportBuilder =
new KettleReportBuilder(this, data.filenames, KettleVFS.getFilename(targetFile), meta);
try {
// Try to get the Classic Reporting Engine to boot inside of the plugin class loader...
//
if (ClassicEngineBoot.getInstance().isBootDone() == false) {
ObjectUtilities.setClassLoader(getClass().getClassLoader());
ObjectUtilities.setClassLoaderSource(ObjectUtilities.CLASS_CONTEXT);
LibLoaderBoot.getInstance().start();
LibFontBoot.getInstance().start();
ClassicEngineBoot.getInstance().start();
}
// Do the reporting thing...
//
kettleReportBuilder.createReport();
kettleReportBuilder.render();
Object[] outputRowData = RowDataUtil.allocateRowData(data.outputRowMeta.size());
int outputIndex = 0;
outputRowData[outputIndex++] = targetFilename;
// Pass along the data to the next steps...
//
putRow(data.outputRowMeta, outputRowData);
// Add the target file to the result file list
//
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL, targetFile, getTransMeta().getName(), toString());
resultFile.setComment("This file was generated by the 'Auto Documentation Output' step");
addResultFile(resultFile);
} catch (Exception e) {
throw new KettleException(
BaseMessages.getString(PKG, "AutoDoc.Exception.UnableToRenderReport"), e);
}
setOutputDone();
return false;
}
if (first) {
first = false;
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
// Get the filename field index...
//
String filenameField = environmentSubstitute(meta.getFilenameField());
data.fileNameFieldIndex = getInputRowMeta().indexOfValue(filenameField);
if (data.fileNameFieldIndex < 0) {
throw new KettleException(
BaseMessages.getString(PKG, "AutoDoc.Exception.FilenameFieldNotFound", filenameField));
}
// Get the file type field index...
//
String fileTypeField = environmentSubstitute(meta.getFileTypeField());
data.fileTypeFieldIndex = getInputRowMeta().indexOfValue(fileTypeField);
if (data.fileTypeFieldIndex < 0) {
throw new KettleException(
BaseMessages.getString(PKG, "AutoDoc.Exception.FileTypeFieldNotFound", fileTypeField));
}
data.repository = getTrans().getRepository();
if (data.repository != null) {
data.tree = data.repository.loadRepositoryDirectoryTree();
}
// Initialize the repository information handlers (images, metadata, loading, etc)
//
TransformationInformation.init(getTrans().getRepository());
JobInformation.init(getTrans().getRepository());
}
// One more transformation or job to place in the documentation.
//
String fileName = getInputRowMeta().getString(row, data.fileNameFieldIndex);
String fileType = getInputRowMeta().getString(row, data.fileTypeFieldIndex);
RepositoryObjectType objectType;
if ("Transformation".equalsIgnoreCase(fileType)) {
objectType = RepositoryObjectType.TRANSFORMATION;
} else if ("Job".equalsIgnoreCase(fileType)) {
objectType = RepositoryObjectType.JOB;
} else {
throw new KettleException(
BaseMessages.getString(PKG, "AutoDoc.Exception.UnknownFileTypeValue", fileType));
}
ReportSubjectLocation location = null;
if (getTrans().getRepository() == null) {
switch (objectType) {
case TRANSFORMATION:
location = new ReportSubjectLocation(fileName, null, null, objectType);
break;
case JOB:
location = new ReportSubjectLocation(fileName, null, null, objectType);
break;
default:
break;
}
} else {
int lastSlashIndex = fileName.lastIndexOf(RepositoryDirectory.DIRECTORY_SEPARATOR);
if (lastSlashIndex < 0) {
fileName = RepositoryDirectory.DIRECTORY_SEPARATOR + fileName;
lastSlashIndex = 0;
}
String directoryName = fileName.substring(0, lastSlashIndex + 1);
String objectName = fileName.substring(lastSlashIndex + 1);
RepositoryDirectoryInterface directory = data.tree.findDirectory(directoryName);
if (directory == null) {
throw new KettleException(
BaseMessages.getString(
PKG, "AutoDoc.Exception.RepositoryDirectoryNotFound", directoryName));
}
location = new ReportSubjectLocation(null, directory, objectName, objectType);
}
if (location == null) {
throw new KettleException(
BaseMessages.getString(
PKG, "AutoDoc.Exception.UnableToDetermineLocation", fileName, fileType));
}
if (meta.getOutputType() != OutputType.METADATA) {
// Add the file location to the list for later processing in one output report
//
data.filenames.add(location);
} else {
// Load the metadata from the transformation / job...
// Output it in one row for each input row
//
Object[] outputRow = RowDataUtil.resizeArray(row, data.outputRowMeta.size());
int outputIndex = getInputRowMeta().size();
List<AreaOwner> imageAreaList = null;
switch (location.getObjectType()) {
case TRANSFORMATION:
TransformationInformation ti = TransformationInformation.getInstance();
TransMeta transMeta = ti.getTransMeta(location);
imageAreaList = ti.getImageAreaList(location);
// TransMeta
outputRow[outputIndex++] = transMeta;
break;
case JOB:
JobInformation ji = JobInformation.getInstance();
JobMeta jobMeta = ji.getJobMeta(location);
imageAreaList = ji.getImageAreaList(location);
// TransMeta
outputRow[outputIndex++] = jobMeta;
break;
default:
break;
}
// Name
if (meta.isIncludingName()) {
outputRow[outputIndex++] = KettleFileTableModel.getName(location);
}
// Description
if (meta.isIncludingDescription()) {
outputRow[outputIndex++] = KettleFileTableModel.getDescription(location);
}
// Extended Description
if (meta.isIncludingExtendedDescription()) {
outputRow[outputIndex++] = KettleFileTableModel.getExtendedDescription(location);
}
// created
if (meta.isIncludingCreated()) {
outputRow[outputIndex++] = KettleFileTableModel.getCreation(location);
}
// modified
if (meta.isIncludingModified()) {
outputRow[outputIndex++] = KettleFileTableModel.getModification(location);
}
// image
if (meta.isIncludingImage()) {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try {
BufferedImage image = KettleFileTableModel.getImage(location);
ImageIO.write(image, "png", outputStream);
outputRow[outputIndex++] = outputStream.toByteArray();
} catch (Exception e) {
throw new KettleException("Unable to serialize image to PNG", e);
} finally {
try {
outputStream.close();
} catch (IOException e) {
throw new KettleException("Unable to serialize image to PNG", e);
}
}
}
if (meta.isIncludingLoggingConfiguration()) {
outputRow[outputIndex++] = KettleFileTableModel.getLogging(location);
}
if (meta.isIncludingLastExecutionResult()) {
outputRow[outputIndex++] = KettleFileTableModel.getLogging(location);
}
if (meta.isIncludingImageAreaList()) {
outputRow[outputIndex++] = imageAreaList;
}
putRow(data.outputRowMeta, outputRow);
}
return true;
}
public Result execute(Result previousResult, int nr) {
String LimitNbrLignes = "";
String ListOfColumn = "*";
String strHighPriority = "";
String OutDumpText = "";
String OptionEnclosed = "";
String FieldSeparator = "";
String LinesTerminated = "";
Result result = previousResult;
result.setResult(false);
// Let's check the filename ...
if (filename != null) {
// User has specified a file, We can continue ...
String realFilename = getRealFilename();
File file = new File(realFilename);
if (file.exists() && iffileexists == 2) {
// the file exists and user want to Fail
result.setResult(false);
result.setNrErrors(1);
logError(
BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
+ realFilename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));
} else if (file.exists() && iffileexists == 1) {
// the file exists and user want to do nothing
result.setResult(true);
if (log.isDetailed())
logDetailed(
BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
+ realFilename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));
} else {
if (file.exists() && iffileexists == 0) {
// File exists and user want to renamme it with unique name
// Format Date
// Try to clean filename (without wildcard)
String wildcard =
realFilename.substring(realFilename.length() - 4, realFilename.length());
if (wildcard.substring(0, 1).equals(".")) {
// Find wildcard
realFilename =
realFilename.substring(0, realFilename.length() - 4)
+ "_"
+ StringUtil.getFormattedDateTimeNow(true)
+ wildcard;
} else {
// did not find wildcard
realFilename = realFilename + "_" + StringUtil.getFormattedDateTimeNow(true);
}
logDebug(
BaseMessages.getString(PKG, "JobMysqlBulkFile.FileNameChange1.Label")
+ realFilename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.FileNameChange1.Label"));
}
// User has specified an existing file, We can continue ...
if (log.isDetailed())
logDetailed(
BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists1.Label")
+ realFilename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.FileExists2.Label"));
if (connection != null) {
// User has specified a connection, We can continue ...
Database db = new Database(this, connection);
db.shareVariablesWith(this);
try {
db.connect(parentJob.getTransactionId(), null);
// Get schemaname
String realSchemaname = environmentSubstitute(schemaname);
// Get tablename
String realTablename = environmentSubstitute(tablename);
if (db.checkTableExists(realTablename)) {
// The table existe, We can continue ...
if (log.isDetailed())
logDetailed(
BaseMessages.getString(PKG, "JobMysqlBulkFile.TableExists1.Label")
+ realTablename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.TableExists2.Label"));
// Add schemaname (Most the time Schemaname.Tablename)
if (schemaname != null) {
realTablename = realSchemaname + "." + realTablename;
}
// Set the Limit lines
if (Const.toInt(getRealLimitlines(), 0) > 0) {
LimitNbrLignes = "LIMIT " + getRealLimitlines();
}
// Set list of Column, if null get all columns (*)
if (getRealListColumn() != null) {
ListOfColumn = MysqlString(getRealListColumn());
}
// Fields separator
if (getRealSeparator() != null && outdumpvalue == 0) {
FieldSeparator =
"FIELDS TERMINATED BY '" + Const.replace(getRealSeparator(), "'", "''") + "'";
}
// Lines Terminated by
if (getRealLineterminated() != null && outdumpvalue == 0) {
LinesTerminated =
"LINES TERMINATED BY '"
+ Const.replace(getRealLineterminated(), "'", "''")
+ "'";
}
// High Priority ?
if (isHighPriority()) {
strHighPriority = "HIGH_PRIORITY";
}
if (getRealEnclosed() != null && outdumpvalue == 0) {
if (isOptionEnclosed()) {
OptionEnclosed = "OPTIONALLY ";
}
OptionEnclosed =
OptionEnclosed
+ "ENCLOSED BY '"
+ Const.replace(getRealEnclosed(), "'", "''")
+ "'";
}
// OutFile or Dumpfile
if (outdumpvalue == 0) {
OutDumpText = "INTO OUTFILE";
} else {
OutDumpText = "INTO DUMPFILE";
}
String FILEBulkFile =
"SELECT "
+ strHighPriority
+ " "
+ ListOfColumn
+ " "
+ OutDumpText
+ " '"
+ realFilename
+ "' "
+ FieldSeparator
+ " "
+ OptionEnclosed
+ " "
+ LinesTerminated
+ " FROM "
+ realTablename
+ " "
+ LimitNbrLignes
+ " LOCK IN SHARE MODE";
try {
if (log.isDetailed()) logDetailed(FILEBulkFile);
// Run the SQL
PreparedStatement ps = db.prepareSQL(FILEBulkFile);
ps.execute();
// Everything is OK...we can disconnect now
db.disconnect();
if (isAddFileToResult()) {
// Add filename to output files
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL,
KettleVFS.getFileObject(realFilename, this),
parentJob.getJobname(),
toString());
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
}
result.setResult(true);
} catch (SQLException je) {
db.disconnect();
result.setNrErrors(1);
logError(
BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label")
+ " "
+ je.getMessage());
} catch (KettleFileException e) {
logError(
BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label") + e.getMessage());
result.setNrErrors(1);
}
} else {
// Of course, the table should have been created already before the bulk load
// operation
db.disconnect();
result.setNrErrors(1);
if (log.isDetailed())
logDetailed(
BaseMessages.getString(PKG, "JobMysqlBulkFile.TableNotExists1.Label")
+ realTablename
+ BaseMessages.getString(PKG, "JobMysqlBulkFile.TableNotExists2.Label"));
}
} catch (KettleDatabaseException dbe) {
db.disconnect();
result.setNrErrors(1);
logError(
BaseMessages.getString(PKG, "JobMysqlBulkFile.Error.Label")
+ " "
+ dbe.getMessage());
}
} else {
// No database connection is defined
result.setNrErrors(1);
logError(BaseMessages.getString(PKG, "JobMysqlBulkFile.Nodatabase.Label"));
}
}
} else {
// No file was specified
result.setNrErrors(1);
logError(BaseMessages.getString(PKG, "JobMysqlBulkFile.Nofilename.Label"));
}
return result;
}
protected void pickFileVFS() {
FileDialog dialog = new FileDialog(shell, SWT.OPEN);
dialog.setFilterExtensions(Const.STRING_TRANS_FILTER_EXT);
dialog.setFilterNames(Const.getTransformationFilterNames());
String prevName = jobMeta.environmentSubstitute(wPath.getText());
String parentFolder = null;
try {
parentFolder =
KettleVFS.getFilename(
KettleVFS.getFileObject(jobMeta.environmentSubstitute(jobMeta.getFilename()))
.getParent());
} catch (Exception e) {
// not that important
}
if (!Utils.isEmpty(prevName)) {
try {
if (KettleVFS.fileExists(prevName)) {
dialog.setFilterPath(
KettleVFS.getFilename(KettleVFS.getFileObject(prevName).getParent()));
} else {
if (!prevName.endsWith(".ktr")) {
prevName = getEntryName(Const.trim(wPath.getText()) + ".ktr");
}
if (KettleVFS.fileExists(prevName)) {
specificationMethod = ObjectLocationSpecificationMethod.FILENAME;
wPath.setText(prevName);
return;
} else {
// File specified doesn't exist. Ask if we should create the file...
//
MessageBox mb = new MessageBox(shell, SWT.YES | SWT.NO | SWT.ICON_QUESTION);
mb.setMessage(
BaseMessages.getString(
PKG, "JobTrans.Dialog.CreateTransformationQuestion.Message"));
mb.setText(
BaseMessages.getString(
PKG, "JobTrans.Dialog.CreateTransformationQuestion.Title")); // Sorry!
int answer = mb.open();
if (answer == SWT.YES) {
Spoon spoon = Spoon.getInstance();
spoon.newTransFile();
TransMeta transMeta = spoon.getActiveTransformation();
transMeta.initializeVariablesFrom(jobEntry);
transMeta.setFilename(jobMeta.environmentSubstitute(prevName));
wPath.setText(prevName);
specificationMethod = ObjectLocationSpecificationMethod.FILENAME;
spoon.saveFile();
return;
}
}
}
} catch (Exception e) {
dialog.setFilterPath(parentFolder);
}
} else if (!Utils.isEmpty(parentFolder)) {
dialog.setFilterPath(parentFolder);
}
String fname = dialog.open();
if (fname != null) {
File file = new File(fname);
String name = file.getName();
String parentFolderSelection = file.getParentFile().toString();
if (!Utils.isEmpty(parentFolder) && parentFolder.equals(parentFolderSelection)) {
wPath.setText(getEntryName(name));
} else {
wPath.setText(fname);
}
}
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
// starttime (in seconds)
long timeStart = System.currentTimeMillis() / 1000;
if (filename != null) {
FileObject fileObject = null;
String realFilename = getRealFilename();
try {
fileObject = KettleVFS.getFileObject(realFilename, this);
long iMaximumTimeout =
Const.toInt(getRealMaximumTimeout(), Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0));
long iCycleTime =
Const.toInt(getRealCheckCycleTime(), Const.toInt(DEFAULT_CHECK_CYCLE_TIME, 0));
//
// Sanity check on some values, and complain on insanity
//
if (iMaximumTimeout < 0) {
iMaximumTimeout = Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0);
if (log.isBasic()) logBasic("Maximum timeout invalid, reset to " + iMaximumTimeout);
}
if (iCycleTime < 1) {
// If lower than 1 set to the default
iCycleTime = Const.toInt(DEFAULT_CHECK_CYCLE_TIME, 1);
if (log.isBasic()) logBasic("Check cycle time invalid, reset to " + iCycleTime);
}
if (iMaximumTimeout == 0) {
if (log.isBasic()) logBasic("Waiting indefinitely for file [" + realFilename + "]");
} else {
if (log.isBasic())
logBasic("Waiting " + iMaximumTimeout + " seconds for file [" + realFilename + "]");
}
boolean continueLoop = true;
while (continueLoop && !parentJob.isStopped()) {
fileObject = KettleVFS.getFileObject(realFilename, this);
if (fileObject.exists()) {
// file exists, we're happy to exit
if (log.isBasic()) logBasic("Detected file [" + realFilename + "] within timeout");
result.setResult(true);
continueLoop = false;
// add filename to result filenames
if (addFilenameToResult && fileObject.getType() == FileType.FILE) {
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL, fileObject, parentJob.getJobname(), toString());
resultFile.setComment(BaseMessages.getString(PKG, "JobWaitForFile.FilenameAdded"));
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
}
} else {
long now = System.currentTimeMillis() / 1000;
if ((iMaximumTimeout > 0) && (now > (timeStart + iMaximumTimeout))) {
continueLoop = false;
// file doesn't exist after timeout, either true or false
if (isSuccessOnTimeout()) {
if (log.isBasic())
logBasic("Didn't detect file [" + realFilename + "] before timeout, success");
result.setResult(true);
} else {
if (log.isBasic())
logBasic("Didn't detect file [" + realFilename + "] before timeout, failure");
result.setResult(false);
}
}
// sleep algorithm
long sleepTime = 0;
if (iMaximumTimeout == 0) {
sleepTime = iCycleTime;
} else {
if ((now + iCycleTime) < (timeStart + iMaximumTimeout)) {
sleepTime = iCycleTime;
} else {
sleepTime = iCycleTime - ((now + iCycleTime) - (timeStart + iMaximumTimeout));
}
}
try {
if (sleepTime > 0) {
if (log.isDetailed()) {
logDetailed(
"Sleeping "
+ sleepTime
+ " seconds before next check for file ["
+ realFilename
+ "]");
}
Thread.sleep(sleepTime * 1000);
}
} catch (InterruptedException e) {
// something strange happened
result.setResult(false);
continueLoop = false;
}
}
}
if (!parentJob.isStopped() && fileObject.exists() && isFileSizeCheck()) {
long oldSize = -1;
long newSize = fileObject.getContent().getSize();
if (log.isDetailed())
logDetailed("File [" + realFilename + "] is " + newSize + " bytes long");
if (log.isBasic())
logBasic(
"Waiting until file ["
+ realFilename
+ "] stops growing for "
+ iCycleTime
+ " seconds");
while (oldSize != newSize && !parentJob.isStopped()) {
try {
if (log.isDetailed()) {
logDetailed(
"Sleeping "
+ iCycleTime
+ " seconds, waiting for file ["
+ realFilename
+ "] to stop growing");
}
Thread.sleep(iCycleTime * 1000);
} catch (InterruptedException e) {
// something strange happened
result.setResult(false);
continueLoop = false;
}
oldSize = newSize;
newSize = fileObject.getContent().getSize();
if (log.isDetailed()) {
logDetailed("File [" + realFilename + "] is " + newSize + " bytes long");
}
}
if (log.isBasic())
logBasic("Stopped waiting for file [" + realFilename + "] to stop growing");
}
if (parentJob.isStopped()) {
result.setResult(false);
}
} catch (Exception e) {
logBasic("Exception while waiting for file [" + realFilename + "] to stop growing", e);
} finally {
if (fileObject != null) {
try {
fileObject.close();
} catch (Exception e) {
}
}
}
} else {
logError("No filename is defined.");
}
return result;
}
/**
* Resolve file name.
*
* @param fileName the filename to resolve. may contain Kettle Environment variables.
* @return the data file name.
* @throws KettleException the kettle exception
*/
@SuppressWarnings("unused")
private String resolveFileName(final String fileName) throws KettleException {
final FileObject fileObject = KettleVFS.getFileObject(parent.environmentSubstitute(fileName));
return KettleVFS.getFilename(fileObject);
}
private boolean processOneXMLFile(
String xmlfilename, String xslfilename, String outputfilename, Result result, Job parentJob) {
boolean retval = false;
FileObject xmlfile = null;
FileObject xslfile = null;
FileObject outputfile = null;
try {
xmlfile = KettleVFS.getFileObject(xmlfilename, this);
xslfile = KettleVFS.getFileObject(xslfilename, this);
outputfile = KettleVFS.getFileObject(outputfilename, this);
if (xmlfile.exists() && xslfile.exists()) {
if (outputfile.exists() && iffileexists == 2) {
// Output file exists
// User want to fail
logError(
BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
+ outputfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
return retval;
} else if (outputfile.exists() && iffileexists == 1) {
// Do nothing
if (log.isDebug()) {
logDebug(
BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
+ outputfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
}
retval = true;
return retval;
} else {
if (outputfile.exists() && iffileexists == 0) {
// the output file exists and user want to create new one with unique name
// Format Date
// Try to clean filename (without wildcard)
String wildcard =
outputfilename.substring(outputfilename.length() - 4, outputfilename.length());
if (wildcard.substring(0, 1).equals(".")) {
// Find wildcard
outputfilename =
outputfilename.substring(0, outputfilename.length() - 4)
+ "_"
+ StringUtil.getFormattedDateTimeNow(true)
+ wildcard;
} else {
// did not find wildcard
outputfilename = outputfilename + "_" + StringUtil.getFormattedDateTimeNow(true);
}
if (log.isDebug()) {
logDebug(
BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists1.Label")
+ outputfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileExists2.Label"));
logDebug(
BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileNameChange1.Label")
+ outputfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.OuputFileNameChange2.Label"));
}
}
// Create transformer factory
TransformerFactory factory = TransformerFactory.newInstance();
if (xsltfactory.equals(FACTORY_SAXON)) {
// Set the TransformerFactory to the SAXON implementation.
factory = new net.sf.saxon.TransformerFactoryImpl();
}
if (log.isDetailed()) {
log.logDetailed(
BaseMessages.getString(PKG, "JobEntryXSL.Log.TransformerFactoryInfos"),
BaseMessages.getString(
PKG, "JobEntryXSL.Log.TransformerFactory", factory.getClass().getName()));
}
InputStream xslInputStream = KettleVFS.getInputStream(xslfile);
InputStream xmlInputStream = KettleVFS.getInputStream(xmlfile);
OutputStream os = null;
try {
// Use the factory to create a template containing the xsl file
Templates template = factory.newTemplates(new StreamSource(xslInputStream));
// Use the template to create a transformer
Transformer xformer = template.newTransformer();
if (log.isDetailed()) {
log.logDetailed(
BaseMessages.getString(PKG, "JobEntryXSL.Log.TransformerClassInfos"),
BaseMessages.getString(
PKG, "JobEntryXSL.Log.TransformerClass", xformer.getClass().getName()));
}
// Do we need to set output properties?
if (setOutputProperties) {
xformer.setOutputProperties(outputProperties);
}
// Do we need to pass parameters?
if (useParameters) {
for (int i = 0; i < nrParams; i++) {
xformer.setParameter(nameOfParams[i], valueOfParams[i]);
}
}
// Prepare the input and output files
Source source = new StreamSource(xmlInputStream);
os = KettleVFS.getOutputStream(outputfile, false);
StreamResult resultat = new StreamResult(os);
// Apply the xsl file to the source file and write the result to the output file
xformer.transform(source, resultat);
if (isAddFileToResult()) {
// Add output filename to output files
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL,
KettleVFS.getFileObject(outputfilename, this),
parentJob.getJobname(),
toString());
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
}
// Everything is OK
retval = true;
} finally {
try {
xslInputStream.close();
} catch (IOException ignored) {
// ignore IO Exception on close
}
try {
xmlInputStream.close();
} catch (IOException ignored) {
// ignore IO Exception on close
}
try {
if (os != null) {
os.close();
}
} catch (IOException ignored) {
// ignore IO Exception on close
}
}
}
} else {
if (!xmlfile.exists()) {
logError(
BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist1.Label")
+ xmlfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist2.Label"));
}
if (!xslfile.exists()) {
logError(
BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist1.Label")
+ xmlfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.FileDoesNotExist2.Label"));
}
}
} catch (Exception e) {
logError(
BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLST.Label")
+ BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXML1.Label")
+ xmlfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXML2.Label")
+ BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXSL1.Label")
+ xslfilename
+ BaseMessages.getString(PKG, "JobEntryXSLT.ErrorXLSTXSL2.Label")
+ e.getMessage());
} finally {
try {
if (xmlfile != null) {
xmlfile.close();
}
if (xslfile != null) {
xslfile.close();
}
if (outputfile != null) {
outputfile.close();
}
} catch (IOException e) {
logError("Unable to close file", e);
}
}
return retval;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
try {
// Get real variable value
String realServerName = environmentSubstitute(serverName);
int realServerPort = Const.toInt(environmentSubstitute(serverPort), 22);
String realUserName = environmentSubstitute(userName);
String realServerPassword =
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password));
// Proxy Host
String realProxyHost = environmentSubstitute(httpproxyhost);
int realProxyPort = Const.toInt(environmentSubstitute(httpproxyport), 22);
String realproxyUserName = environmentSubstitute(httpproxyusername);
String realProxyPassword =
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(httpProxyPassword));
// Key file
String realKeyFilename = environmentSubstitute(keyFilename);
String relKeyFilepass = environmentSubstitute(keyFilePass);
// Source files
String realLocalDirectory = environmentSubstitute(localDirectory);
String realwildcard = environmentSubstitute(wildcard);
// Remote destination
String realftpDirectory = environmentSubstitute(ftpDirectory);
// Destination folder (Move to)
String realDestinationFolder = environmentSubstitute(destinationfolder);
try {
// Remote source
realftpDirectory = FTPUtils.normalizePath(realftpDirectory);
// Destination folder (Move to)
realDestinationFolder = FTPUtils.normalizePath(realDestinationFolder);
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.CanNotNormalizePath", e.getMessage()));
result.setNrErrors(1);
return result;
}
// Check for mandatory fields
boolean mandatoryok = true;
if (Const.isEmpty(realServerName)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.ServernameMissing"));
}
if (usehttpproxy) {
if (Const.isEmpty(realProxyHost)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.HttpProxyhostMissing"));
}
}
if (publicpublickey) {
if (Const.isEmpty(realKeyFilename)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileMissing"));
} else {
// Let's check if folder exists...
if (!KettleVFS.fileExists(realKeyFilename, this)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileNotExist"));
}
}
}
if (Const.isEmpty(realLocalDirectory)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.LocalFolderMissing"));
}
if (afterFtpPut.equals("move_file")) {
if (Const.isEmpty(realDestinationFolder)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderMissing"));
} else {
FileObject folder = null;
try {
folder = KettleVFS.getFileObject(realDestinationFolder, this);
// Let's check if folder exists...
if (!folder.exists()) {
// Do we need to create it?
if (createDestinationFolder) {
folder.createFolder();
} else {
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.DestinatFolderNotExist", realDestinationFolder));
}
}
} catch (Exception e) {
throw new KettleException(e);
} finally {
if (folder != null) {
try {
folder.close();
folder = null;
} catch (Exception e) {
/* Ignore */
}
}
}
}
}
if (mandatoryok) {
Connection conn = null;
SFTPv3Client client = null;
boolean good = true;
int nbfilestoput = 0;
int nbput = 0;
int nbrerror = 0;
try {
// Create a connection instance
conn =
getConnection(
realServerName,
realServerPort,
realProxyHost,
realProxyPort,
realproxyUserName,
realProxyPassword);
if (timeout > 0) {
// Use timeout
// Cache Host Key
if (cachehostkey) {
conn.connect(new SimpleVerifier(database), 0, timeout * 1000);
} else {
conn.connect(null, 0, timeout * 1000);
}
} else {
// Cache Host Key
if (cachehostkey) {
conn.connect(new SimpleVerifier(database));
} else {
conn.connect();
}
}
// Authenticate
boolean isAuthenticated = false;
if (publicpublickey) {
String keyContent =
KettleVFS.getTextFileContent(realKeyFilename, this, Const.XML_ENCODING);
isAuthenticated =
conn.authenticateWithPublicKey(
realUserName, keyContent.toCharArray(), relKeyFilepass);
} else {
isAuthenticated = conn.authenticateWithPassword(realUserName, realServerPassword);
}
// LET'S CHECK AUTHENTICATION ...
if (isAuthenticated == false) {
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.AuthenticationFailed"));
} else {
if (log.isBasic()) {
logBasic(
BaseMessages.getString(PKG, "JobSSH2PUT.Log.Connected", serverName, userName));
}
client = new SFTPv3Client(conn);
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.ProtocolVersion", "" + client.getProtocolVersion()));
}
// Check if remote directory exists
if (!Const.isEmpty(realftpDirectory)) {
if (!sshDirectoryExists(client, realftpDirectory)) {
good = false;
if (createRemoteFolder) {
good = CreateRemoteFolder(client, realftpDirectory);
if (good) {
logBasic(BaseMessages.getString(PKG, "JobSSH2PUT.Log.RemoteDirectoryCreated"));
}
} else {
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.RemoteDirectoryNotExist", realftpDirectory));
}
} else if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.RemoteDirectoryExist", realftpDirectory));
}
}
if (good) {
// Get files list from local folder (source)
List<FileObject> myFileList = getFiles(realLocalDirectory);
// Prepare Pattern for wildcard
Pattern pattern = null;
if (!Const.isEmpty(realwildcard)) {
pattern = Pattern.compile(realwildcard);
}
// Let's put files now ...
// Get the files in the list
for (int i = 0; i < myFileList.size() && !parentJob.isStopped(); i++) {
FileObject myFile = myFileList.get(i);
String localFilename = myFile.toString();
String remoteFilename = myFile.getName().getBaseName();
boolean getIt = true;
// First see if the file matches the regular expression!
if (pattern != null) {
Matcher matcher = pattern.matcher(remoteFilename);
getIt = matcher.matches();
}
// do we have a target directory?
if (!Const.isEmpty(realftpDirectory)) {
remoteFilename = realftpDirectory + FTPUtils.FILE_SEPARATOR + remoteFilename;
}
if (onlyGettingNewFiles) {
// We get only new files
// ie not exist on the remote server
getIt = !sshFileExists(client, remoteFilename);
}
if (getIt) {
nbfilestoput++;
boolean putok = putFile(myFile, remoteFilename, client);
if (!putok) {
nbrerror++;
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Error.CanNotPutFile", localFilename));
} else {
nbput++;
}
if (putok && !afterFtpPut.equals("do_nothing")) {
deleteOrMoveFiles(myFile, realDestinationFolder);
}
}
}
/** ****************************** RESULT ******************* */
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd1"));
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Result.TotalFiles", "" + nbfilestoput));
logDetailed(
BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.TotalFilesPut", "" + nbput));
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Result.TotalFilesError", "" + nbrerror));
logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd2"));
}
if (nbrerror == 0) {
result.setResult(true);
/** ****************************** RESULT ******************* */
}
}
}
} catch (Exception e) {
result.setNrErrors(nbrerror);
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.ErrorFTP", e.getMessage()));
} finally {
if (conn != null) {
conn.close();
}
if (client != null) {
client.close();
}
}
}
} catch (Exception e) {
result.setResult(false);
result.setNrErrors(1L);
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.UnexpectedError"), e);
}
return result;
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (XsltMeta) smi;
data = (XsltData) sdi;
Object[] row = getRow();
if (row == null) { // no more input to be expected...
setOutputDone();
return false;
}
if (first) {
first = false;
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
// Check if The result field is given
if (Const.isEmpty(meta.getResultfieldname())) {
// Result Field is missing !
logError(BaseMessages.getString(PKG, "Xslt.Log.ErrorResultFieldMissing"));
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.ErrorResultFieldMissing"));
}
// Check if The XML field is given
if (Const.isEmpty(meta.getFieldname())) {
// Result Field is missing !
logError(BaseMessages.getString(PKG, "Xslt.Exception.ErrorXMLFieldMissing"));
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.ErrorXMLFieldMissing"));
}
// Try to get XML Field index
data.fieldposition = getInputRowMeta().indexOfValue(meta.getFieldname());
// Let's check the Field
if (data.fieldposition < 0) {
// The field is unreachable !
logError(
BaseMessages.getString(PKG, "Xslt.Log.ErrorFindingField")
+ "["
+ meta.getFieldname()
+ "]");
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.CouldnotFindField", meta.getFieldname()));
}
// Check if the XSL Filename is contained in a column
if (meta.useXSLField()) {
if (Const.isEmpty(meta.getXSLFileField())) {
// The field is missing
// Result field is missing !
logError(BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLFileFieldMissing"));
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.ErrorXSLFileFieldMissing"));
}
// Try to get Field index
data.fielxslfiledposition = getInputRowMeta().indexOfValue(meta.getXSLFileField());
// Let's check the Field
if (data.fielxslfiledposition < 0) {
// The field is unreachable !
logError(
BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLFileFieldFinding")
+ "["
+ meta.getXSLFileField()
+ "]");
throw new KettleStepException(
BaseMessages.getString(
PKG, "Xslt.Exception.ErrorXSLFileFieldFinding", meta.getXSLFileField()));
}
} else {
if (Const.isEmpty(meta.getXslFilename())) {
logError(BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLFile"));
throw new KettleStepException(BaseMessages.getString(PKG, "Xslt.Exception.ErrorXSLFile"));
}
// Check if XSL File exists!
data.xslfilename = environmentSubstitute(meta.getXslFilename());
FileObject file = null;
try {
file = KettleVFS.getFileObject(data.xslfilename);
if (!file.exists()) {
logError(
BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLFileNotExists", data.xslfilename));
throw new KettleStepException(
BaseMessages.getString(
PKG, "Xslt.Exception.ErrorXSLFileNotExists", data.xslfilename));
}
if (file.getType() != FileType.FILE) {
logError(BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLNotAFile", data.xslfilename));
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.ErrorXSLNotAFile", data.xslfilename));
}
} catch (Exception e) {
throw new KettleStepException(e);
} finally {
try {
if (file != null) {
file.close();
}
} catch (Exception e) {
/* Ignore */
}
}
}
// Check output parameters
int nrOutputProps =
meta.getOutputPropertyName() == null ? 0 : meta.getOutputPropertyName().length;
if (nrOutputProps > 0) {
data.outputProperties = new Properties();
for (int i = 0; i < nrOutputProps; i++) {
data.outputProperties.put(
meta.getOutputPropertyName()[i],
environmentSubstitute(meta.getOutputPropertyValue()[i]));
}
data.setOutputProperties = true;
}
// Check parameters
data.nrParams = meta.getParameterField() == null ? 0 : meta.getParameterField().length;
if (data.nrParams > 0) {
data.indexOfParams = new int[data.nrParams];
data.nameOfParams = new String[data.nrParams];
for (int i = 0; i < data.nrParams; i++) {
String name = environmentSubstitute(meta.getParameterName()[i]);
String field = environmentSubstitute(meta.getParameterField()[i]);
if (Const.isEmpty(field)) {
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.ParameterFieldMissing", name, i));
}
data.indexOfParams[i] = getInputRowMeta().indexOfValue(field);
if (data.indexOfParams[i] < 0) {
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.ParameterFieldNotFound", name));
}
data.nameOfParams[i] = name;
}
data.useParameters = true;
}
data.factory = TransformerFactory.newInstance();
if (meta.getXSLFactory().equals("SAXON")) {
// Set the TransformerFactory to the SAXON implementation.
data.factory = new net.sf.saxon.TransformerFactoryImpl();
}
} // end if first
// Get the field value
String xmlValue = getInputRowMeta().getString(row, data.fieldposition);
if (meta.useXSLField()) {
// Get the value
data.xslfilename = getInputRowMeta().getString(row, data.fielxslfiledposition);
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "Xslt.Log.XslfileNameFromFied", data.xslfilename, meta.getXSLFileField()));
}
}
try {
if (log.isDetailed()) {
if (meta.isXSLFieldIsAFile()) {
logDetailed(BaseMessages.getString(PKG, "Xslt.Log.Filexsl") + data.xslfilename);
} else {
logDetailed(BaseMessages.getString(PKG, "Xslt.Log.XslStream", data.xslfilename));
}
}
// Get the template from the cache
Transformer transformer = data.getTemplate(data.xslfilename, data.xslIsAfile);
// Do we need to set output properties?
if (data.setOutputProperties) {
transformer.setOutputProperties(data.outputProperties);
}
// Do we need to pass parameters?
if (data.useParameters) {
for (int i = 0; i < data.nrParams; i++) {
transformer.setParameter(data.nameOfParams[i], row[data.indexOfParams[i]]);
}
}
Source source = new StreamSource(new StringReader(xmlValue));
// Prepare output stream
StreamResult result = new StreamResult(new StringWriter());
// transform xml source
transformer.transform(source, result);
String xmlString = result.getWriter().toString();
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "Xslt.Log.FileResult"));
logDetailed(xmlString);
}
Object[] outputRowData = RowDataUtil.addValueData(row, getInputRowMeta().size(), xmlString);
if (log.isRowLevel()) {
logRowlevel(
BaseMessages.getString(PKG, "Xslt.Log.ReadRow")
+ " "
+ getInputRowMeta().getString(row));
}
// add new values to the row.
putRow(data.outputRowMeta, outputRowData); // copy row to output rowset(s);
} catch (Exception e) {
String errorMessage = e.getMessage();
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
DefaultErrorHandler.printLocation(pw, e);
pw.close();
errorMessage = sw.toString() + "\n" + errorMessage;
if (getStepMeta().isDoingErrorHandling()) {
// Simply add this row to the error row
putError(getInputRowMeta(), row, 1, errorMessage, meta.getResultfieldname(), "XSLT01");
} else {
logError(BaseMessages.getString(PKG, "Xslt.ErrorProcesing" + " : " + errorMessage));
throw new KettleStepException(BaseMessages.getString(PKG, "Xslt.ErrorProcesing"), e);
}
}
return true;
}
private boolean openNextFile() {
try {
if (meta.getIsInFields()) {
data.readrow = getRow(); // Grab another row ...
if (data.readrow == null) // finished processing!
{
if (isDetailed())
logDetailed(BaseMessages.getString(PKG, "LoadFileInput.Log.FinishedProcessing"));
return false;
}
if (first) {
first = false;
data.inputRowMeta = getInputRowMeta();
data.outputRowMeta = data.inputRowMeta.clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
// Create convert meta-data objects that will contain Date & Number formatters
//
data.convertRowMeta = data.outputRowMeta.cloneToType(ValueMetaInterface.TYPE_STRING);
if (meta.getIsInFields()) {
// Check is filename field is provided
if (Const.isEmpty(meta.getDynamicFilenameField())) {
logError(BaseMessages.getString(PKG, "LoadFileInput.Log.NoField"));
throw new KettleException(BaseMessages.getString(PKG, "LoadFileInput.Log.NoField"));
}
// cache the position of the field
if (data.indexOfFilenameField < 0) {
data.indexOfFilenameField =
data.inputRowMeta.indexOfValue(meta.getDynamicFilenameField());
if (data.indexOfFilenameField < 0) {
// The field is unreachable !
logError(
BaseMessages.getString(PKG, "LoadFileInput.Log.ErrorFindingField")
+ "["
+ meta.getDynamicFilenameField()
+ "]"); //$NON-NLS-1$ //$NON-NLS-2$
throw new KettleException(
BaseMessages.getString(
PKG,
"LoadFileInput.Exception.CouldnotFindField",
meta.getDynamicFilenameField())); // $NON-NLS-1$ //$NON-NLS-2$
}
}
// Get the number of previous fields
data.totalpreviousfields = data.inputRowMeta.size();
}
} // end if first
// get field value
String Fieldvalue = data.inputRowMeta.getString(data.readrow, data.indexOfFilenameField);
if (isDetailed())
logDetailed(
BaseMessages.getString(
PKG, "LoadFileInput.Log.Stream", meta.getDynamicFilenameField(), Fieldvalue));
FileObject file = null;
try {
// Source is a file.
data.file = KettleVFS.getFileObject(Fieldvalue);
} catch (Exception e) {
throw new KettleException(e);
} finally {
try {
if (file != null) file.close();
} catch (Exception e) {
}
}
} else {
if (data.filenr >= data.files.nrOfFiles()) // finished processing!
{
if (isDetailed())
logDetailed(BaseMessages.getString(PKG, "LoadFileInput.Log.FinishedProcessing"));
return false;
}
// Is this the last file?
data.last_file = (data.filenr == data.files.nrOfFiles() - 1);
data.file = (FileObject) data.files.getFile(data.filenr);
}
// Check if file is empty
data.fileSize = data.file.getContent().getSize();
// Move file pointer ahead!
data.filenr++;
if (meta.isIgnoreEmptyFile() && data.fileSize == 0) {
logError(
BaseMessages.getString(
PKG, "LoadFileInput.Error.FileSizeZero", "" + data.file.getName()));
openNextFile();
} else {
if (isDetailed())
logDetailed(
BaseMessages.getString(PKG, "LoadFileInput.Log.OpeningFile", data.file.toString()));
data.filename = KettleVFS.getFilename(data.file);
// Add additional fields?
if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
data.shortFilename = data.file.getName().getBaseName();
}
if (meta.getPathField() != null && meta.getPathField().length() > 0) {
data.path = KettleVFS.getFilename(data.file.getParent());
}
if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
data.hidden = data.file.isHidden();
}
if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
data.extension = data.file.getName().getExtension();
}
if (meta.getLastModificationDateField() != null
&& meta.getLastModificationDateField().length() > 0) {
data.lastModificationDateTime = new Date(data.file.getContent().getLastModifiedTime());
}
if (meta.getUriField() != null && meta.getUriField().length() > 0) {
data.uriName = data.file.getName().getURI();
}
if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
data.rootUriName = data.file.getName().getRootURI();
}
// get File content
getFileContent();
addFileToResultFilesname(data.file);
if (isDetailed()) {
logDetailed(
BaseMessages.getString(PKG, "LoadFileInput.Log.FileOpened", data.file.toString()));
}
}
} catch (Exception e) {
logError(
BaseMessages.getString(
PKG,
"LoadFileInput.Log.UnableToOpenFile",
"" + data.filenr,
data.file.toString(),
e.toString()));
stopAll();
setErrors(1);
return false;
}
return true;
}
public boolean validate() {
boolean retval = false;
FileObject xmlfile = null;
FileObject DTDfile = null;
ByteArrayInputStream ba = null;
try {
if (xmlfilename != null
&& ((getDTDFilename() != null && !isInternDTD()) || (isInternDTD()))) {
xmlfile = KettleVFS.getFileObject(getXMLFilename());
if (xmlfile.exists()) {
URL xmlFile = new File(KettleVFS.getFilename(xmlfile)).toURI().toURL();
StringBuffer xmlStringbuffer = new StringBuffer("");
BufferedReader xmlBufferedReader = null;
InputStreamReader is = null;
try {
// open XML File
is = new InputStreamReader(xmlFile.openStream());
xmlBufferedReader = new BufferedReader(is);
char[] buffertXML = new char[1024];
int LenXML = -1;
while ((LenXML = xmlBufferedReader.read(buffertXML)) != -1) {
xmlStringbuffer.append(buffertXML, 0, LenXML);
}
} finally {
if (is != null) {
is.close();
}
if (xmlBufferedReader != null) {
xmlBufferedReader.close();
}
}
// Prepare parsing ...
DocumentBuilderFactory DocBuilderFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder DocBuilder = DocBuilderFactory.newDocumentBuilder();
// Let's try to get XML document encoding
DocBuilderFactory.setValidating(false);
ba = new ByteArrayInputStream(xmlStringbuffer.toString().getBytes("UTF-8"));
Document xmlDocDTD = DocBuilder.parse(ba);
if (ba != null) {
ba.close();
}
String encoding = null;
if (xmlDocDTD.getXmlEncoding() == null) {
encoding = "UTF-8";
} else {
encoding = xmlDocDTD.getXmlEncoding();
}
int xmlStartDTD = xmlStringbuffer.indexOf("<!DOCTYPE");
if (isInternDTD()) {
// DTD find in the XML document
if (xmlStartDTD != -1) {
log.logBasic(
BaseMessages.getString(
PKG, "JobEntryDTDValidator.ERRORDTDFound.Label", getXMLFilename()));
} else {
setErrorMessage(
BaseMessages.getString(
PKG, "JobEntryDTDValidator.ERRORDTDNotFound.Label", getXMLFilename()));
}
} else {
// DTD in external document
// If we find an intern declaration, we remove it
DTDfile = KettleVFS.getFileObject(getDTDFilename());
if (DTDfile.exists()) {
if (xmlStartDTD != -1) {
int EndDTD = xmlStringbuffer.indexOf(">", xmlStartDTD);
// String DocTypeDTD = xmlStringbuffer.substring(xmlStartDTD, EndDTD + 1);
xmlStringbuffer.replace(xmlStartDTD, EndDTD + 1, "");
}
String xmlRootnodeDTD = xmlDocDTD.getDocumentElement().getNodeName();
String RefDTD =
"<?xml version='"
+ xmlDocDTD.getXmlVersion()
+ "' encoding='"
+ encoding
+ "'?>\n<!DOCTYPE "
+ xmlRootnodeDTD
+ " SYSTEM '"
+ KettleVFS.getFilename(DTDfile)
+ "'>\n";
int xmloffsetDTD = xmlStringbuffer.indexOf("<" + xmlRootnodeDTD);
xmlStringbuffer.replace(0, xmloffsetDTD, RefDTD);
} else {
log.logError(
BaseMessages.getString(PKG, "JobEntryDTDValidator.ERRORDTDFileNotExists.Subject"),
BaseMessages.getString(
PKG, "JobEntryDTDValidator.ERRORDTDFileNotExists.Msg", getDTDFilename()));
}
}
if (!(isInternDTD() && xmlStartDTD == -1 || (!isInternDTD() && !DTDfile.exists()))) {
// Let's parse now ...
MyErrorHandler error = new MyErrorHandler();
DocBuilderFactory.setValidating(true);
DocBuilder = DocBuilderFactory.newDocumentBuilder();
DocBuilder.setErrorHandler(error);
ba = new ByteArrayInputStream(xmlStringbuffer.toString().getBytes(encoding));
xmlDocDTD = DocBuilder.parse(ba);
if (error.errorMessage == null) {
log.logBasic(
BaseMessages.getString(PKG, "JobEntryDTDValidator.DTDValidatorOK.Subject"),
BaseMessages.getString(
PKG, "JobEntryDTDValidator.DTDValidatorOK.Label", getXMLFilename()));
// Everything is OK
retval = true;
} else {
// Invalid DTD
setNrErrors(error.nrErrors);
setErrorMessage(
BaseMessages.getString(
PKG,
"JobEntryDTDValidator.DTDValidatorKO",
getXMLFilename(),
error.nrErrors,
error.errorMessage));
}
}
} else {
if (!xmlfile.exists()) {
setErrorMessage(
BaseMessages.getString(
PKG, "JobEntryDTDValidator.FileDoesNotExist.Label", getXMLFilename()));
}
}
} else {
setErrorMessage(BaseMessages.getString(PKG, "JobEntryDTDValidator.AllFilesNotNull.Label"));
}
} catch (Exception e) {
setErrorMessage(
BaseMessages.getString(
PKG,
"JobEntryDTDValidator.ErrorDTDValidator.Label",
getXMLFilename(),
getDTDFilename(),
e.getMessage()));
} finally {
try {
if (xmlfile != null) {
xmlfile.close();
}
if (DTDfile != null) {
DTDfile.close();
}
if (ba != null) {
ba.close();
}
} catch (IOException e) {
// Ignore close errors
}
}
return retval;
}
