/**
* Finds the tab for the transformation that matches the metadata provided (either the file must
* be the same or the repository id).
*
* @param trans Transformation metadata to look for
* @return Tab with transformation open whose metadata matches {@code trans} or {@code null} if no
* tab exists.
* @throws KettleFileException If there is a problem loading the file object for an open
* transformation with an invalid a filename.
*/
public TabMapEntry findTabForTransformation(TransMeta trans) throws KettleFileException {
// File for the transformation we're looking for. It will be loaded upon first request.
FileObject transFile = null;
for (TabMapEntry entry : tabMap) {
if (entry == null || entry.getTabItem().isDisposed()) {
continue;
}
if (trans.getFilename() != null && entry.getFilename() != null) {
// If the entry has a file name it is the same as trans iff. they originated from the same
// files
FileObject entryFile = KettleVFS.getFileObject(entry.getFilename());
if (transFile == null) {
transFile = KettleVFS.getFileObject(trans.getFilename());
}
if (entryFile.equals(transFile)) {
return entry;
}
} else if (trans.getObjectId() != null && entry.getObject() != null) {
EngineMetaInterface meta = entry.getObject().getMeta();
if (meta != null && trans.getObjectId().equals(meta.getObjectId())) {
// If the transformation has an object id and the entry shares the same id they are the
// same
return entry;
}
}
}
// No tabs for the transformation exist and are not disposed
return null;
}
public void dispose(StepMetaInterface smi, StepDataInterface sdi) {
meta = (GPLoadMeta) smi;
data = (GPLoadData) sdi;
super.dispose(smi, sdi);
if (!preview && meta.isEraseFiles()) {
// Erase the created cfg/dat files if requested. We don't erase
// the rest of the files because it would be "stupid" to erase them
// right after creation. If you don't want them, don't fill them in.
FileObject fileObject = null;
String method = meta.getLoadMethod();
if ( // GPLoadMeta.METHOD_AUTO_CONCURRENT.equals(method) ||
GPLoadMeta.METHOD_AUTO_END.equals(method)) {
/*
if ( meta.getControlFile() != null )
{
try
{
fileObject = KettleVFS.getFileObject(environmentSubstitute(meta.getControlFile()), getTransMeta());
fileObject.delete();
fileObject.close();
}
catch ( Exception ex )
{
logError("Error deleting control file \'" + KettleVFS.getFilename(fileObject) + "\': " + ex.getMessage());
}
}*/
}
if (GPLoadMeta.METHOD_AUTO_END.equals(method)) {
// In concurrent mode the data is written to the control file.
if (meta.getDataFile() != null) {
try {
fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getDataFile()), getTransMeta());
fileObject.delete();
fileObject.close();
} catch (Exception ex) {
logError(
"Error deleting data file \'"
+ KettleVFS.getFilename(fileObject)
+ "\': "
+ ex.getMessage(),
ex);
}
}
}
if (GPLoadMeta.METHOD_MANUAL.equals(method)) {
logBasic("Deletion of files is not compatible with \'manual load method\'");
}
}
}
protected void registerXmlPlugins() throws KettlePluginException {
for (PluginFolderInterface folder : pluginFolders) {
if (folder.isPluginXmlFolder()) {
List<FileObject> pluginXmlFiles = findPluginXmlFiles(folder.getFolder());
for (FileObject file : pluginXmlFiles) {
try {
Document document = XMLHandler.loadXMLFile(file);
Node pluginNode = XMLHandler.getSubNode(document, "plugin");
if (pluginNode != null) {
registerPluginFromXmlResource(
pluginNode,
KettleVFS.getFilename(file.getParent()),
this.getClass(),
false,
file.getParent().getURL());
}
} catch (Exception e) {
// We want to report this plugin.xml error, perhaps an XML typo or something like
// that...
//
log.logError(
"Error found while reading step plugin.xml file: " + file.getName().toString(), e);
}
}
}
}
}
/**
* Since the exported transformation that runs this will reside in a ZIP file, we can't reference
* files relatively. So what this does is turn the name of the base path into an absolute path.
*
* @param space the variable space to use
* @param definitions
* @param resourceNamingInterface
* @param repository The repository to optionally load other resources from (to be converted to
* XML)
* @param metaStore the metaStore in which non-kettle metadata could reside.
* @return the filename of the exported resource
*/
public String exportResources(
VariableSpace space,
Map<String, ResourceDefinition> definitions,
ResourceNamingInterface resourceNamingInterface,
Repository repository,
IMetaStore metaStore)
throws KettleException {
try {
// The object that we're modifying here is a copy of the original!
// So let's change the filename from relative to absolute by grabbing the file object...
// In case the name of the file comes from previous steps, forget about this!
//
if (!fileNameInField) {
if (!Const.isEmpty(fileName)) {
FileObject fileObject =
KettleVFS.getFileObject(space.environmentSubstitute(fileName), space);
fileName = resourceNamingInterface.nameResource(fileObject, space, true);
}
}
return null;
} catch (Exception e) {
throw new KettleException(e);
}
}
@Test
public void testZeroSizeFile() throws Exception {
ByteArrayOutputStream log = new ByteArrayOutputStream();
helper.redirectLog(log, LogLevel.BASIC);
try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.json");
LocaleChange enUs = new LocaleChange(Locale.US); ) {
fileObj.createFile();
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
JsonInputMeta meta = createSimpleMeta("in file", price);
meta.setIsAFile(true);
meta.setRemoveSourceField(true);
meta.setIgnoreEmptyFile(false);
JsonInput jsonInput =
createJsonInput(
"in file", meta, new Object[][] {new Object[] {BASE_RAM_DIR + "test.json"}});
processRows(jsonInput, 1);
String logMsgs = log.toString();
assertTrue(logMsgs, logMsgs.contains("is empty!"));
} finally {
deleteFiles();
}
}
private boolean OpenFile() throws Exception {
data.oneFileOpened = true;
String realFilename = environmentSubstitute(meta.getFilename());
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "AccessOutput.log.WritingToFile", realFilename));
}
FileObject fileObject = KettleVFS.getFileObject(realFilename, getTransMeta());
File file = FileUtils.toFile(fileObject.getURL());
// First open or create the access file
if (!file.exists()) {
if (meta.isFileCreated()) {
data.db = Database.create(file);
} else {
logError(
BaseMessages.getString(PKG, "AccessOutput.InitError.FileDoesNotExist", realFilename));
return false;
}
} else {
data.db = Database.open(file);
}
// Add the filename to the result object...
//
if (meta.isAddToResultFiles()) {
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL, fileObject, getTransMeta().getName(), toString());
resultFile.setComment("This file was created with an access output step");
addResultFile(resultFile);
}
return true;
}
private boolean CreateFolder(String filefolder) {
FileObject folder = null;
try {
folder = KettleVFS.getFileObject(filefolder, this);
if (!folder.exists()) {
if (createtargetfolder) {
folder.createFolder();
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(PKG, "JobSSH2GET.Log.FolderCreated", folder.toString()));
}
} else {
return false;
}
}
return true;
} catch (Exception e) {
logError(
BaseMessages.getString(PKG, "JobSSH2GET.Log.CanNotCreateFolder", folder.toString()), e);
} finally {
if (folder != null) {
try {
folder.close();
} catch (Exception ex) {
/* Ignore */
}
}
}
return false;
}
/**
* Log only lines belonging to the specified log channel ID or one of it's children
* (grandchildren) to the specified file.
*
* @param logChannelId
* @param filename
* @param append
* @throws KettleException
*/
public FileLoggingEventListener(String logChannelId, String filename, boolean append)
throws KettleException {
this.logChannelId = logChannelId;
this.filename = filename;
this.layout = new KettleLogLayout(true);
this.exception = null;
file = KettleVFS.getFileObject(filename);
outputStream = null;
try {
outputStream = KettleVFS.getOutputStream(file, append);
} catch (Exception e) {
throw new KettleException(
"Unable to create a logging event listener to write to file '" + filename + "'", e);
}
}
private void ProcessFile(String filename, String wildcard) {
FileObject filefolder = null;
String realFilefoldername = environmentSubstitute(filename);
String realwilcard = environmentSubstitute(wildcard);
try {
filefolder = KettleVFS.getFileObject(realFilefoldername);
FileObject[] files = new FileObject[] {filefolder};
if (filefolder.exists()) {
// the file or folder exists
if (filefolder.getType() == FileType.FOLDER) {
// It's a folder
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryCheckFilesLocked.ProcessingFolder", realFilefoldername));
}
// Retrieve all files
files = filefolder.findFiles(new TextFileSelector(filefolder.toString(), realwilcard));
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobEntryCheckFilesLocked.TotalFilesToCheck",
String.valueOf(files.length)));
}
} else {
// It's a file
if (isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryCheckFilesLocked.ProcessingFile", realFilefoldername));
}
}
// Check files locked
checkFilesLocked(files);
} else {
// We can not find thsi file
logBasic(
BaseMessages.getString(
PKG, "JobEntryCheckFilesLocked.FileNotExist", realFilefoldername));
}
} catch (Exception e) {
logError(
BaseMessages.getString(
PKG, "JobEntryCheckFilesLocked.CouldNotProcess", realFilefoldername, e.getMessage()));
} finally {
if (filefolder != null) {
try {
filefolder.close();
} catch (IOException ex) {
// Ignore
}
}
}
}
private boolean deleteOrMoveFiles(FileObject file, String destinationFolder)
throws KettleException {
try {
boolean retval = false;
// Delete the file if this is needed!
//
if (afterFtpPut.equals("delete_file")) {
file.delete();
retval = true;
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DeletedFile", file.toString()));
}
} else if (afterFtpPut.equals("move_file")) {
// Move File
FileObject destination = null;
FileObject source = null;
try {
destination =
KettleVFS.getFileObject(
destinationFolder + Const.FILE_SEPARATOR + file.getName().getBaseName(), this);
file.moveTo(destination);
retval = true;
} catch (Exception e) {
logError(
BaseMessages.getString(
PKG,
"JobSSH2PUT.Cant_Move_File.Label",
file.toString(),
destinationFolder,
e.getMessage()));
} finally {
if (destination != null) {
try {
destination.close();
} catch (Exception ex) {
/* Ignore */
}
}
if (source != null) {
try {
source.close();
} catch (Exception ex) {
/* Ignore */
}
}
}
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.MovedFile", file.toString(), ftpDirectory));
}
}
return retval;
} catch (Exception e) {
throw new KettleException(e);
}
}
@Test
public void testFileList() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input1 = getBasicTestJson();
final String input2 = "{ \"store\": { \"book\": [ { \"price\": 9.99 } ] } }";
try (FileObject fileObj1 = KettleVFS.getFileObject(BASE_RAM_DIR + "test1.json");
FileObject fileObj2 = KettleVFS.getFileObject(BASE_RAM_DIR + "test2.json")) {
try (OutputStream out = fileObj1.getContent().getOutputStream()) {
out.write(input1.getBytes());
}
try (OutputStream out = fileObj2.getContent().getOutputStream()) {
out.write(input2.getBytes());
}
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
List<FileObject> fileList = Arrays.asList(fileObj1, fileObj2);
JsonInputMeta meta = createFileListMeta(fileList);
meta.setInputFields(new JsonInputField[] {price});
meta.setIncludeRowNumber(true);
meta.setRowNumberField("rownbr");
meta.setShortFileNameField("fname");
JsonInput jsonInput = createJsonInput(meta);
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {8.95d, 1L, "test1.json"},
new Object[] {12.99d, 2L, "test1.json"},
new Object[] {8.99d, 3L, "test1.json"},
new Object[] {22.99d, 4L, "test1.json"},
new Object[] {9.99d, 5L, "test2.json"});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
disposeJsonInput(jsonInput);
assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
private void readFileOrString() throws Exception {
if (data.file != null) {
data.filename = KettleVFS.getFilename(data.file);
// Add additional fields?
if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
data.shortFilename = data.file.getName().getBaseName();
}
if (meta.getPathField() != null && meta.getPathField().length() > 0) {
data.path = KettleVFS.getFilename(data.file.getParent());
}
if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
data.hidden = data.file.isHidden();
}
if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
data.extension = data.file.getName().getExtension();
}
if (meta.getLastModificationDateField() != null
&& meta.getLastModificationDateField().length() > 0) {
data.lastModificationDateTime = new Date(data.file.getContent().getLastModifiedTime());
}
if (meta.getUriField() != null && meta.getUriField().length() > 0) {
data.uriName = data.file.getName().getURI();
}
if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
data.rootUriName = data.file.getName().getRootURI();
}
// Check if file is empty
long fileSize = data.file.getContent().getSize();
if (meta.getSizeField() != null && meta.getSizeField().length() > 0) {
data.size = fileSize;
}
// Move file pointer ahead!
data.filenr++;
if (log.isDetailed())
logDetailed(BaseMessages.getString(PKG, "JsonInput.Log.OpeningFile", data.file.toString()));
addFileToResultFilesname(data.file);
}
parseJson();
}
public OdfWorkbook(String filename, String encoding) throws KettleException {
this.filename = filename;
this.encoding = encoding;
try {
document = OdfSpreadsheetDocument.loadDocument(KettleVFS.getInputStream(filename));
} catch (Exception e) {
throw new KettleException(e);
}
}
private void checkFilesLocked(FileObject[] files) throws KettleException {
for (int i = 0; i < files.length && !oneFileLocked; i++) {
FileObject file = files[i];
String filename = KettleVFS.getFilename(file);
LockFile locked = new LockFile(filename);
if (locked.isLocked()) {
oneFileLocked = true;
logError(BaseMessages.getString(PKG, "JobCheckFilesLocked.Log.FileLocked", filename));
} else {
if (isDetailed()) {
logDetailed(
BaseMessages.getString(PKG, "JobCheckFilesLocked.Log.FileNotLocked", filename));
}
}
}
}
/**
* Create a new log channel file writer
*
* @param logChannelId The log channel (+children) to write to the log file
* @param logFile The logging file to write to
* @param appending set to true if you want to append to an existing file
* @param pollingInterval The polling interval in milliseconds.
* @throws KettleException in case the specified log file can't be created.
*/
public LogChannelFileWriter(
String logChannelId, FileObject logFile, boolean appending, int pollingInterval)
throws KettleException {
this.logChannelId = logChannelId;
this.logFile = logFile;
this.appending = appending;
this.pollingInterval = pollingInterval;
active = new AtomicBoolean(false);
lastBufferLineNr = KettleLogStore.getLastBufferLineNr();
try {
logFileOutputStream = KettleVFS.getOutputStream(logFile, appending);
} catch (IOException e) {
throw new KettleException(
"There was an error while trying to open file '" + logFile + "' for writing", e);
}
}
public void put(FileObject fileObject, String remoteFile) throws KettleJobException {
int mode = ChannelSftp.OVERWRITE;
InputStream inputStream = null;
try {
inputStream = KettleVFS.getInputStream(fileObject);
c.put(inputStream, remoteFile, null, mode);
} catch (Exception e) {
throw new KettleJobException(e);
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
throw new KettleJobException(e);
}
}
}
}
/**
* Check existence of a local file
*
* @param filename
* @return true, if file exists
*/
public boolean FileExists(String filename) {
FileObject file = null;
try {
file = KettleVFS.getFileObject(filename, this);
if (!file.exists()) {
return false;
} else {
if (file.getType() == FileType.FILE) {
return true;
} else {
return false;
}
}
} catch (Exception e) {
return false;
}
}
public void get(FileObject localFile, String remoteFile) throws KettleJobException {
OutputStream localStream = null;
try {
localStream = KettleVFS.getOutputStream(localFile, false);
c.get(remoteFile, localStream);
} catch (SftpException e) {
throw new KettleJobException(e);
} catch (IOException e) {
throw new KettleJobException(e);
} finally {
if (localStream != null) {
try {
localStream.close();
} catch (IOException ignore) {
// Ignore any IOException, as we're trying to close the stream anyways
}
}
}
}
/**
* Init Helper Class with connection settings
*
* @param serverIP IP address of remote server
* @param serverPort port of remote server
* @param userName username of remote server
* @param privateKeyFilename filename of private key
* @param passPhrase passphrase
* @throws KettleJobException
*/
public SFTPClient(
InetAddress serverIP,
int serverPort,
String userName,
String privateKeyFilename,
String passPhrase)
throws KettleJobException {
if (serverIP == null || serverPort < 0 || userName == null || userName.equals("")) {
throw new KettleJobException(
"For a SFTP connection server name and username must be set and server port must be greater than zero.");
}
this.serverIP = serverIP;
this.serverPort = serverPort;
this.userName = userName;
JSch jsch = new JSch();
try {
if (!Const.isEmpty(privateKeyFilename)) {
// We need to use private key authentication
this.prvkey = privateKeyFilename;
byte[] passphrasebytes = new byte[0];
if (!Const.isEmpty(passPhrase)) {
// Set passphrase
this.passphrase = passPhrase;
passphrasebytes = GetPrivateKeyPassPhrase().getBytes();
}
jsch.addIdentity(
getUserName(),
FileUtil.getContent(KettleVFS.getFileObject(prvkey)), // byte[] privateKey
null, // byte[] publicKey
passphrasebytes); // byte[] passPhrase
}
s = jsch.getSession(userName, serverIP.getHostAddress(), serverPort);
} catch (IOException e) {
throw new KettleJobException(e);
} catch (KettleFileException e) {
throw new KettleJobException(e);
} catch (JSchException e) {
throw new KettleJobException(e);
}
}
@Test
public void testZipFileInput() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input = getBasicTestJson();
try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.zip")) {
fileObj.createFile();
try (OutputStream out = fileObj.getContent().getOutputStream()) {
try (ZipOutputStream zipOut = new ZipOutputStream(out)) {
ZipEntry jsonFile = new ZipEntry("test.json");
zipOut.putNextEntry(jsonFile);
zipOut.write(input.getBytes());
zipOut.closeEntry();
zipOut.flush();
}
}
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
JsonInputMeta meta = createSimpleMeta("in file", price);
meta.setIsAFile(true);
meta.setRemoveSourceField(true);
JsonInput jsonInput =
createJsonInput(
"in file",
meta,
new Object[][] {new Object[] {"zip:" + BASE_RAM_DIR + "test.zip!/test.json"}});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {8.95d},
new Object[] {12.99d},
new Object[] {8.99d},
new Object[] {22.99d});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
Assert.assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
/**
* Create the command line for a psql process depending on the meta information supplied.
*
* @param meta The meta data to create the command line from
* @param password Use the real password or not
* @return The string to execute.
* @throws KettleException Upon any exception
*/
public String createCommandLine(GPLoadMeta meta, boolean password) throws KettleException {
StringBuffer sb = new StringBuffer(300);
if (meta.getGploadPath() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getGploadPath()), getTransMeta());
String psqlexec = KettleVFS.getFilename(fileObject);
// sb.append('\'').append(psqlexec).append('\'');
sb.append(psqlexec);
} catch (Exception ex) {
throw new KettleException("Error retrieving sqlldr string", ex);
}
} else {
throw new KettleException("No psql application specified");
}
if (meta.getControlFile() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getControlFile()), getTransMeta());
sb.append(" -f ");
// sb.append('\'').append(KettleVFS.getFilename(fileObject)).append('\'');
sb.append(KettleVFS.getFilename(fileObject));
} catch (Exception ex) {
throw new KettleException("Error retrieving controlfile string", ex);
}
} else {
throw new KettleException("No control file specified");
}
if (meta.getLogFile() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getLogFile()), getTransMeta());
sb.append(" -l ");
sb.append('\'').append(KettleVFS.getFilename(fileObject)).append('\'');
} catch (Exception ex) {
throw new KettleException("Error retrieving logfile string", ex);
}
}
// hostname, port and so on are passed through the control file
//
return sb.toString();
}
/**
* Read a text file.
*
* @param vfsFilename the filename or URL to read from
* @param charSetName the character set of the string (UTF-8, ISO8859-1, etc)
* @return The content of the file as a String
* @throws IOException
*/
public static String getTextFileContent(String vfsFilename, String encoding)
throws KettleException {
InputStream inputStream = null;
InputStreamReader reader = null;
String retval = null;
try {
inputStream = KettleVFS.getInputStream(vfsFilename);
if (!Const.isEmpty(encoding)) {
reader = new InputStreamReader(new BufferedInputStream(inputStream), encoding);
} else {
reader = new InputStreamReader(new BufferedInputStream(inputStream));
}
int c;
StringBuffer stringBuffer = new StringBuffer();
while ((c = reader.read()) != -1) stringBuffer.append((char) c);
retval = stringBuffer.toString();
} catch (Exception e) {
throw new KettleException(
BaseMessages.getString(
PKG, "LoadFileInput.Error.GettingFileContent", vfsFilename, e.toString()));
} finally {
if (reader != null)
try {
reader.close();
} catch (Exception e) {
}
;
if (inputStream != null)
try {
inputStream.close();
} catch (Exception e) {
}
;
}
return retval;
}
private List<FileObject> getFiles(String localfolder) throws KettleFileException {
try {
List<FileObject> myFileList = new ArrayList<FileObject>();
// Get all the files in the local directory...
FileObject localFiles = KettleVFS.getFileObject(localfolder, this);
FileObject[] children = localFiles.getChildren();
if (children != null) {
for (int i = 0; i < children.length; i++) {
// Get filename of file or directory
if (children[i].getType().equals(FileType.FILE)) {
myFileList.add(children[i]);
}
} // end for
}
return myFileList;
} catch (IOException e) {
throw new KettleFileException(e);
}
}
private FileObject createTemporaryShellFile(FileObject tempFile, String fileContent)
throws Exception {
// Create a unique new temporary filename in the working directory, put the script in there
// Set the permissions to execute and then run it...
//
if (tempFile != null && fileContent != null) {
try {
// flag indicates if current OS is Windows or not
boolean isWindows = Const.isWindows();
if (!isWindows) {
fileContent = replaceWinEOL(fileContent);
}
tempFile.createFile();
OutputStream outputStream = tempFile.getContent().getOutputStream();
outputStream.write(fileContent.getBytes());
outputStream.close();
if (!isWindows) {
String tempFilename = KettleVFS.getFilename(tempFile);
// Now we have to make this file executable...
// On Unix-like systems this is done using the command "/bin/chmod +x filename"
//
ProcessBuilder procBuilder = new ProcessBuilder("chmod", "+x", tempFilename);
Process proc = procBuilder.start();
// Eat/log stderr/stdout all messages in a different thread...
StreamLogger errorLogger =
new StreamLogger(log, proc.getErrorStream(), toString() + " (stderr)");
StreamLogger outputLogger =
new StreamLogger(log, proc.getInputStream(), toString() + " (stdout)");
new Thread(errorLogger).start();
new Thread(outputLogger).start();
proc.waitFor();
}
} catch (Exception e) {
throw new Exception("Unable to create temporary file to execute script", e);
}
}
return tempFile;
}
/**
* Since the exported transformation that runs this will reside in a ZIP file, we can't reference
* files relatively. So what this does is turn the name of files into absolute paths OR it simply
* includes the resource in the ZIP file. For now, we'll simply turn it into an absolute path and
* pray that the file is on a shared drive or something like that.
*
* <p>TODO: create options to configure this behavior
*/
public String exportResources(
VariableSpace space,
Map<String, ResourceDefinition> definitions,
ResourceNamingInterface resourceNamingInterface,
Repository repository)
throws KettleException {
try {
// The object that we're modifying here is a copy of the original!
// So let's change the filename from relative to absolute by grabbing the file object...
// In case the name of the file comes from previous steps, forget about this!
//
if (Const.isEmpty(filenameField)) {
// From : ${Internal.Transformation.Filename.Directory}/../foo/bar.csv
// To : /home/matt/test/files/foo/bar.csv
//
FileObject fileObject = KettleVFS.getFileObject(space.environmentSubstitute(filename));
// If the file doesn't exist, forget about this effort too!
//
if (fileObject.exists()) {
// Convert to an absolute path...
//
filename =
resourceNamingInterface.nameResource(
fileObject.getName().getBaseName(),
fileObject.getParent().getName().getPath(),
space.toString(),
FileNamingType.DATA_FILE);
return filename;
}
}
return null;
} catch (Exception e) {
throw new KettleException(e); // $NON-NLS-1$
}
}
public Result execute(Result previousResult, int nr, Repository rep, Job parentJob) {
LogWriter log = LogWriter.getInstance();
Result result = previousResult;
result.setResult(false);
if (filename != null) {
String realFilename = getRealFilename();
try {
FileObject file = KettleVFS.getFileObject(realFilename);
if (file.exists() && file.isReadable()) {
log.logDetailed(
toString(),
Messages.getString("JobEntryFileExists.File_Exists", realFilename)); // $NON-NLS-1$
result.setResult(true);
} else {
log.logDetailed(
toString(),
Messages.getString(
"JobEntryFileExists.File_Does_Not_Exist", realFilename)); // $NON-NLS-1$
}
} catch (IOException e) {
result.setNrErrors(1);
log.logError(
toString(),
Messages.getString(
"JobEntryFileExists.ERROR_0004_IO_Exception", e.toString())); // $NON-NLS-1$
}
} else {
result.setNrErrors(1);
log.logError(
toString(),
Messages.getString("JobEntryFileExists.ERROR_0005_No_Filename_Defined")); // $NON-NLS-1$
}
return result;
}
private boolean openNextFile() {
try {
if (meta.getIsInFields()) {
data.readrow = getRow(); // Grab another row ...
if (data.readrow == null) // finished processing!
{
if (isDetailed())
logDetailed(BaseMessages.getString(PKG, "LoadFileInput.Log.FinishedProcessing"));
return false;
}
if (first) {
first = false;
data.inputRowMeta = getInputRowMeta();
data.outputRowMeta = data.inputRowMeta.clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
// Create convert meta-data objects that will contain Date & Number formatters
//
data.convertRowMeta = data.outputRowMeta.cloneToType(ValueMetaInterface.TYPE_STRING);
if (meta.getIsInFields()) {
// Check is filename field is provided
if (Const.isEmpty(meta.getDynamicFilenameField())) {
logError(BaseMessages.getString(PKG, "LoadFileInput.Log.NoField"));
throw new KettleException(BaseMessages.getString(PKG, "LoadFileInput.Log.NoField"));
}
// cache the position of the field
if (data.indexOfFilenameField < 0) {
data.indexOfFilenameField =
data.inputRowMeta.indexOfValue(meta.getDynamicFilenameField());
if (data.indexOfFilenameField < 0) {
// The field is unreachable !
logError(
BaseMessages.getString(PKG, "LoadFileInput.Log.ErrorFindingField")
+ "["
+ meta.getDynamicFilenameField()
+ "]"); //$NON-NLS-1$ //$NON-NLS-2$
throw new KettleException(
BaseMessages.getString(
PKG,
"LoadFileInput.Exception.CouldnotFindField",
meta.getDynamicFilenameField())); // $NON-NLS-1$ //$NON-NLS-2$
}
}
// Get the number of previous fields
data.totalpreviousfields = data.inputRowMeta.size();
}
} // end if first
// get field value
String Fieldvalue = data.inputRowMeta.getString(data.readrow, data.indexOfFilenameField);
if (isDetailed())
logDetailed(
BaseMessages.getString(
PKG, "LoadFileInput.Log.Stream", meta.getDynamicFilenameField(), Fieldvalue));
FileObject file = null;
try {
// Source is a file.
data.file = KettleVFS.getFileObject(Fieldvalue);
} catch (Exception e) {
throw new KettleException(e);
} finally {
try {
if (file != null) file.close();
} catch (Exception e) {
}
}
} else {
if (data.filenr >= data.files.nrOfFiles()) // finished processing!
{
if (isDetailed())
logDetailed(BaseMessages.getString(PKG, "LoadFileInput.Log.FinishedProcessing"));
return false;
}
// Is this the last file?
data.last_file = (data.filenr == data.files.nrOfFiles() - 1);
data.file = (FileObject) data.files.getFile(data.filenr);
}
// Check if file is empty
data.fileSize = data.file.getContent().getSize();
// Move file pointer ahead!
data.filenr++;
if (meta.isIgnoreEmptyFile() && data.fileSize == 0) {
logError(
BaseMessages.getString(
PKG, "LoadFileInput.Error.FileSizeZero", "" + data.file.getName()));
openNextFile();
} else {
if (isDetailed())
logDetailed(
BaseMessages.getString(PKG, "LoadFileInput.Log.OpeningFile", data.file.toString()));
data.filename = KettleVFS.getFilename(data.file);
// Add additional fields?
if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
data.shortFilename = data.file.getName().getBaseName();
}
if (meta.getPathField() != null && meta.getPathField().length() > 0) {
data.path = KettleVFS.getFilename(data.file.getParent());
}
if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
data.hidden = data.file.isHidden();
}
if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
data.extension = data.file.getName().getExtension();
}
if (meta.getLastModificationDateField() != null
&& meta.getLastModificationDateField().length() > 0) {
data.lastModificationDateTime = new Date(data.file.getContent().getLastModifiedTime());
}
if (meta.getUriField() != null && meta.getUriField().length() > 0) {
data.uriName = data.file.getName().getURI();
}
if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
data.rootUriName = data.file.getName().getRootURI();
}
// get File content
getFileContent();
addFileToResultFilesname(data.file);
if (isDetailed()) {
logDetailed(
BaseMessages.getString(PKG, "LoadFileInput.Log.FileOpened", data.file.toString()));
}
}
} catch (Exception e) {
logError(
BaseMessages.getString(
PKG,
"LoadFileInput.Log.UnableToOpenFile",
"" + data.filenr,
data.file.toString(),
e.toString()));
stopAll();
setErrors(1);
return false;
}
return true;
}
protected void pickFileVFS() {
FileDialog dialog = new FileDialog(shell, SWT.OPEN);
dialog.setFilterExtensions(Const.STRING_TRANS_FILTER_EXT);
dialog.setFilterNames(Const.getTransformationFilterNames());
String prevName = jobMeta.environmentSubstitute(wPath.getText());
String parentFolder = null;
try {
parentFolder =
KettleVFS.getFilename(
KettleVFS.getFileObject(jobMeta.environmentSubstitute(jobMeta.getFilename()))
.getParent());
} catch (Exception e) {
// not that important
}
if (!Utils.isEmpty(prevName)) {
try {
if (KettleVFS.fileExists(prevName)) {
dialog.setFilterPath(
KettleVFS.getFilename(KettleVFS.getFileObject(prevName).getParent()));
} else {
if (!prevName.endsWith(".ktr")) {
prevName = getEntryName(Const.trim(wPath.getText()) + ".ktr");
}
if (KettleVFS.fileExists(prevName)) {
specificationMethod = ObjectLocationSpecificationMethod.FILENAME;
wPath.setText(prevName);
return;
} else {
// File specified doesn't exist. Ask if we should create the file...
//
MessageBox mb = new MessageBox(shell, SWT.YES | SWT.NO | SWT.ICON_QUESTION);
mb.setMessage(
BaseMessages.getString(
PKG, "JobTrans.Dialog.CreateTransformationQuestion.Message"));
mb.setText(
BaseMessages.getString(
PKG, "JobTrans.Dialog.CreateTransformationQuestion.Title")); // Sorry!
int answer = mb.open();
if (answer == SWT.YES) {
Spoon spoon = Spoon.getInstance();
spoon.newTransFile();
TransMeta transMeta = spoon.getActiveTransformation();
transMeta.initializeVariablesFrom(jobEntry);
transMeta.setFilename(jobMeta.environmentSubstitute(prevName));
wPath.setText(prevName);
specificationMethod = ObjectLocationSpecificationMethod.FILENAME;
spoon.saveFile();
return;
}
}
}
} catch (Exception e) {
dialog.setFilterPath(parentFolder);
}
} else if (!Utils.isEmpty(parentFolder)) {
dialog.setFilterPath(parentFolder);
}
String fname = dialog.open();
if (fname != null) {
File file = new File(fname);
String name = file.getName();
String parentFolderSelection = file.getParentFile().toString();
if (!Utils.isEmpty(parentFolder) && parentFolder.equals(parentFolderSelection)) {
wPath.setText(getEntryName(name));
} else {
wPath.setText(fname);
}
}
}
private boolean putFile(FileObject localFile, String remotefilename, SFTPv3Client sftpClient) {
long filesize = -1;
InputStream in = null;
BufferedInputStream inBuf = null;
SFTPv3FileHandle sftpFileHandle = null;
boolean retval = false;
try {
// Put file in the folder
sftpFileHandle = sftpClient.createFileTruncate(remotefilename);
// Associate a file input stream for the current local file
in = KettleVFS.getInputStream(localFile);
inBuf = new BufferedInputStream(in);
byte[] buf = new byte[2048];
long offset = 0;
long length = localFile.getContent().getSize();
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobSSH2PUT.Log.SendingFile",
localFile.toString(),
"" + length,
remotefilename));
}
// Write to remote file
while (true) {
int len = in.read(buf, 0, buf.length);
if (len <= 0) {
break;
}
sftpClient.write(sftpFileHandle, offset, buf, 0, len);
offset += len;
}
// Get File size
filesize = getFileSize(sftpClient, remotefilename);
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.FileOnRemoteHost", remotefilename, "" + filesize));
}
retval = true;
} catch (Exception e) {
// We failed to put files
logError(
BaseMessages.getString(PKG, "JobSSH2PUT.Log.ErrorCopyingFile", localFile.toString())
+ ":"
+ e.getMessage());
} finally {
if (in != null) {
try {
in.close();
in = null;
} catch (Exception ex) {
// Ignore errors
}
}
if (inBuf != null) {
try {
inBuf.close();
inBuf = null;
} catch (Exception ex) {
// Ignore errors
}
}
if (sftpFileHandle != null) {
try {
sftpClient.closeFile(sftpFileHandle);
sftpFileHandle = null;
} catch (Exception ex) {
// Ignore errors
}
}
}
return retval;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
try {
// Get real variable value
String realServerName = environmentSubstitute(serverName);
int realServerPort = Const.toInt(environmentSubstitute(serverPort), 22);
String realUserName = environmentSubstitute(userName);
String realServerPassword =
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password));
// Proxy Host
String realProxyHost = environmentSubstitute(httpproxyhost);
int realProxyPort = Const.toInt(environmentSubstitute(httpproxyport), 22);
String realproxyUserName = environmentSubstitute(httpproxyusername);
String realProxyPassword =
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(httpProxyPassword));
// Key file
String realKeyFilename = environmentSubstitute(keyFilename);
String relKeyFilepass = environmentSubstitute(keyFilePass);
// Source files
String realLocalDirectory = environmentSubstitute(localDirectory);
String realwildcard = environmentSubstitute(wildcard);
// Remote destination
String realftpDirectory = environmentSubstitute(ftpDirectory);
// Destination folder (Move to)
String realDestinationFolder = environmentSubstitute(destinationfolder);
try {
// Remote source
realftpDirectory = FTPUtils.normalizePath(realftpDirectory);
// Destination folder (Move to)
realDestinationFolder = FTPUtils.normalizePath(realDestinationFolder);
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.CanNotNormalizePath", e.getMessage()));
result.setNrErrors(1);
return result;
}
// Check for mandatory fields
boolean mandatoryok = true;
if (Const.isEmpty(realServerName)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.ServernameMissing"));
}
if (usehttpproxy) {
if (Const.isEmpty(realProxyHost)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.HttpProxyhostMissing"));
}
}
if (publicpublickey) {
if (Const.isEmpty(realKeyFilename)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileMissing"));
} else {
// Let's check if folder exists...
if (!KettleVFS.fileExists(realKeyFilename, this)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileNotExist"));
}
}
}
if (Const.isEmpty(realLocalDirectory)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.LocalFolderMissing"));
}
if (afterFtpPut.equals("move_file")) {
if (Const.isEmpty(realDestinationFolder)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderMissing"));
} else {
FileObject folder = null;
try {
folder = KettleVFS.getFileObject(realDestinationFolder, this);
// Let's check if folder exists...
if (!folder.exists()) {
// Do we need to create it?
if (createDestinationFolder) {
folder.createFolder();
} else {
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.DestinatFolderNotExist", realDestinationFolder));
}
}
} catch (Exception e) {
throw new KettleException(e);
} finally {
if (folder != null) {
try {
folder.close();
folder = null;
} catch (Exception e) {
/* Ignore */
}
}
}
}
}
if (mandatoryok) {
Connection conn = null;
SFTPv3Client client = null;
boolean good = true;
int nbfilestoput = 0;
int nbput = 0;
int nbrerror = 0;
try {
// Create a connection instance
conn =
getConnection(
realServerName,
realServerPort,
realProxyHost,
realProxyPort,
realproxyUserName,
realProxyPassword);
if (timeout > 0) {
// Use timeout
// Cache Host Key
if (cachehostkey) {
conn.connect(new SimpleVerifier(database), 0, timeout * 1000);
} else {
conn.connect(null, 0, timeout * 1000);
}
} else {
// Cache Host Key
if (cachehostkey) {
conn.connect(new SimpleVerifier(database));
} else {
conn.connect();
}
}
// Authenticate
boolean isAuthenticated = false;
if (publicpublickey) {
String keyContent =
KettleVFS.getTextFileContent(realKeyFilename, this, Const.XML_ENCODING);
isAuthenticated =
conn.authenticateWithPublicKey(
realUserName, keyContent.toCharArray(), relKeyFilepass);
} else {
isAuthenticated = conn.authenticateWithPassword(realUserName, realServerPassword);
}
// LET'S CHECK AUTHENTICATION ...
if (isAuthenticated == false) {
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.AuthenticationFailed"));
} else {
if (log.isBasic()) {
logBasic(
BaseMessages.getString(PKG, "JobSSH2PUT.Log.Connected", serverName, userName));
}
client = new SFTPv3Client(conn);
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.ProtocolVersion", "" + client.getProtocolVersion()));
}
// Check if remote directory exists
if (!Const.isEmpty(realftpDirectory)) {
if (!sshDirectoryExists(client, realftpDirectory)) {
good = false;
if (createRemoteFolder) {
good = CreateRemoteFolder(client, realftpDirectory);
if (good) {
logBasic(BaseMessages.getString(PKG, "JobSSH2PUT.Log.RemoteDirectoryCreated"));
}
} else {
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.RemoteDirectoryNotExist", realftpDirectory));
}
} else if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.RemoteDirectoryExist", realftpDirectory));
}
}
if (good) {
// Get files list from local folder (source)
List<FileObject> myFileList = getFiles(realLocalDirectory);
// Prepare Pattern for wildcard
Pattern pattern = null;
if (!Const.isEmpty(realwildcard)) {
pattern = Pattern.compile(realwildcard);
}
// Let's put files now ...
// Get the files in the list
for (int i = 0; i < myFileList.size() && !parentJob.isStopped(); i++) {
FileObject myFile = myFileList.get(i);
String localFilename = myFile.toString();
String remoteFilename = myFile.getName().getBaseName();
boolean getIt = true;
// First see if the file matches the regular expression!
if (pattern != null) {
Matcher matcher = pattern.matcher(remoteFilename);
getIt = matcher.matches();
}
// do we have a target directory?
if (!Const.isEmpty(realftpDirectory)) {
remoteFilename = realftpDirectory + FTPUtils.FILE_SEPARATOR + remoteFilename;
}
if (onlyGettingNewFiles) {
// We get only new files
// ie not exist on the remote server
getIt = !sshFileExists(client, remoteFilename);
}
if (getIt) {
nbfilestoput++;
boolean putok = putFile(myFile, remoteFilename, client);
if (!putok) {
nbrerror++;
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Error.CanNotPutFile", localFilename));
} else {
nbput++;
}
if (putok && !afterFtpPut.equals("do_nothing")) {
deleteOrMoveFiles(myFile, realDestinationFolder);
}
}
}
/** ****************************** RESULT ******************* */
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd1"));
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Result.TotalFiles", "" + nbfilestoput));
logDetailed(
BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.TotalFilesPut", "" + nbput));
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Result.TotalFilesError", "" + nbrerror));
logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd2"));
}
if (nbrerror == 0) {
result.setResult(true);
/** ****************************** RESULT ******************* */
}
}
}
} catch (Exception e) {
result.setNrErrors(nbrerror);
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.ErrorFTP", e.getMessage()));
} finally {
if (conn != null) {
conn.close();
}
if (client != null) {
client.close();
}
}
}
} catch (Exception e) {
result.setResult(false);
result.setNrErrors(1L);
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.UnexpectedError"), e);
}
return result;
}