private void initializeCurrent(FileObject file, String filePart) throws KettleException {
try {
FileObject lineFile =
AbstractFileErrorHandler.getReplayFilename(
lineNumberDirectory,
file.getName().getBaseName(),
replayDate,
lineNumberExtension,
filePart);
if (lineFile.exists()) {
currentLineNumberFile =
new FilePlayListReplayLineNumberFile(lineFile, encoding, file, filePart);
} else {
currentLineNumberFile = new FilePlayListReplayFile(file, filePart);
}
FileObject errorFile =
AbstractFileErrorHandler.getReplayFilename(
errorDirectory,
file.getName().getURI(),
replayDate,
errorExtension,
AbstractFileErrorHandler.NO_PARTS);
if (errorFile.exists()) {
currentErrorFile = new FilePlayListReplayErrorFile(errorFile, file);
} else {
currentErrorFile = new FilePlayListReplayFile(file, AbstractFileErrorHandler.NO_PARTS);
}
} catch (IOException e) {
throw new KettleException(e);
}
}
/**
* Tests closing all FileSystems
*
* @throws Exception
*/
@Ignore("vfs close file system doesn't seem to work properly")
@Test
public void testCloseFileSystems() throws Exception {
logger.info("*************** testCloseFileSystems");
String[] validUrls = server.getVFSRootURLs();
ArrayList<FileObject> fos = new ArrayList<FileObject>();
for (String validUrl : validUrls) {
FileObject mounted = VFSMountManagerHelper.mount(validUrl);
Assert.assertTrue(mounted.exists());
fos.add(mounted);
}
VFSMountManagerHelper.closeFileSystems(Arrays.asList(validUrls));
boolean onlyExceptions = true;
for (FileObject closedFo : fos) {
try {
FileObject toto = closedFo.resolveFile("toto");
toto.createFile();
onlyExceptions = false;
logger.error(toto.getURL() + " exists : " + toto.exists());
} catch (FileSystemException e) {
// this should occur
}
}
Assert.assertTrue("Only Exceptions received", onlyExceptions);
}
private boolean CreateFolder(String filefolder) {
FileObject folder = null;
try {
folder = KettleVFS.getFileObject(filefolder, this);
if (!folder.exists()) {
if (createtargetfolder) {
folder.createFolder();
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(PKG, "JobSSH2GET.Log.FolderCreated", folder.toString()));
}
} else {
return false;
}
}
return true;
} catch (Exception e) {
logError(
BaseMessages.getString(PKG, "JobSSH2GET.Log.CanNotCreateFolder", folder.toString()), e);
} finally {
if (folder != null) {
try {
folder.close();
} catch (Exception ex) {
/* Ignore */
}
}
}
return false;
}
public Boolean fileExists(ServerDetailsDTO conDetails, String filePath) {
FileObject fileObject = null;
StandardFileSystemManager manager = null;
try {
manager = new StandardFileSystemManager();
manager.init();
UserAuthenticator auth =
new StaticUserAuthenticator(null, conDetails.getUserName(), conDetails.getPassword());
FileSystemOptions opts = new FileSystemOptions();
DefaultFileSystemConfigBuilder.getInstance().setUserAuthenticator(opts, auth);
SftpFileSystemConfigBuilder.getInstance().setStrictHostKeyChecking(opts, "no");
String fileUri = buildUri(conDetails, filePath);
fileObject = manager.resolveFile(fileUri, opts);
return fileObject.exists() && fileObject.isReadable();
} catch (FileSystemException e) {
log.error("cannot access the server", e);
} finally {
try {
if (fileObject != null) {
((SftpFileSystem) fileObject.getFileSystem()).closeCommunicationLink();
FileSystem fs = null;
fs = fileObject.getFileSystem();
manager.closeFileSystem(fs);
}
} finally {
}
}
return null;
}
/**
* Parse a set of URLs from a comma-separated list of URLs. If the URL points to a directory all
* jar files within that directory will be returned as well.
*
* @param urlString Comma-separated list of URLs (relative or absolute)
* @return List of URLs resolved from {@code urlString}
*/
protected List<URL> parseURLs(FileObject root, String urlString) {
if (urlString == null || urlString.trim().isEmpty()) {
return Collections.emptyList();
}
String[] paths = urlString.split(",");
List<URL> urls = new ArrayList<URL>();
for (String path : paths) {
try {
FileObject file = root.resolveFile(path.trim());
if (!file.exists()) {
file = defaultFsm.resolveFile(path.trim());
}
if (FileType.FOLDER.equals(file.getType())) {
// Add directories with a trailing / so the URL ClassLoader interprets
// them as directories
urls.add(new URL(file.getURL().toExternalForm() + "/"));
// Also add all jars within this directory
urls.addAll(findJarsIn(file, 1, new HashSet<String>()));
} else {
urls.add(file.getURL());
}
} catch (Exception e) {
// Log invalid path
logger.error(BaseMessages.getString(PKG, "Error.InvalidClasspathEntry", path));
}
}
return urls;
}
public boolean isProcessingNeeded(FileObject file, long lineNr, String filePart)
throws KettleException {
try {
return errorFile.exists();
} catch (IOException e) {
throw new KettleException(e);
}
}
/**
* Tests mounting only one valid FileSystem
*
* @throws Exception
*/
@Test
public void testMountOk() throws Exception {
logger.info("*************** testMountOk");
String[] validUrls = server.getVFSRootURLs();
for (String validUrl : validUrls) {
FileObject mounted = VFSMountManagerHelper.mount(validUrl);
Assert.assertTrue(mounted.exists());
}
}
/** Does a 'touch' command. */
private void touch(final String[] cmd) throws Exception {
if (cmd.length < 2) {
throw new Exception("USAGE: touch <path>");
}
final FileObject file = mgr.resolveFile(cwd, cmd[1]);
if (!file.exists()) {
file.createFile();
}
file.getContent().setLastModifiedTime(System.currentTimeMillis());
}
/**
* Creates a temporary local copy of a file and its descendents.
*
* @param file The FileObject to replicate.
* @param selector The FileSelector.
* @return The replicated File.
* @throws FileSystemException if an error occurs.
*/
public File replicateFile(final FileObject file, final FileSelector selector)
throws FileSystemException {
if (!file.exists()) {
throw new FileSystemException("vfs.provider/replicate-missing-file.error", file.getName());
}
try {
return doReplicateFile(file, selector);
} catch (final Exception e) {
throw new FileSystemException("vfs.provider/replicate-file.error", file.getName(), e);
}
}
/** Does a 'cp' command. */
private void cp(final String[] cmd) throws Exception {
if (cmd.length < 3) {
throw new Exception("USAGE: cp <src> <dest>");
}
final FileObject src = mgr.resolveFile(cwd, cmd[1]);
FileObject dest = mgr.resolveFile(cwd, cmd[2]);
if (dest.exists() && dest.getType() == FileType.FOLDER) {
dest = dest.resolveFile(src.getName().getBaseName());
}
dest.copyFrom(src, Selectors.SELECT_ALL);
}
public List<String> getDirContent(
ServerDetailsDTO conDetails, String filePath, String... extentions) throws IOException {
FileObject fileObject = null;
StandardFileSystemManager manager = null;
try {
manager = new StandardFileSystemManager();
manager.init();
UserAuthenticator auth =
new StaticUserAuthenticator(null, conDetails.getUserName(), conDetails.getPassword());
FileSystemOptions opts = new FileSystemOptions();
DefaultFileSystemConfigBuilder.getInstance().setUserAuthenticator(opts, auth);
SftpFileSystemConfigBuilder.getInstance().setStrictHostKeyChecking(opts, "no");
String fileUri = buildUri(conDetails, filePath);
fileObject = manager.resolveFile(fileUri, opts);
if (!fileObject.exists()) {
return null;
}
List<String> fileNames = new ArrayList<String>();
FileObject[] children = fileObject.getChildren();
if (children != null) {
for (FileObject child : children) {
String fileName = child.getName().getBaseName();
if (extentions.length > 0) {
for (String extention : extentions) {
if (fileName.endsWith(extention)) {
fileNames.add(fileName);
}
}
} else {
fileNames.add(fileName);
}
}
}
return fileNames;
} finally {
try {
if (fileObject != null) {
((SftpFileSystem) fileObject.getFileSystem()).closeCommunicationLink();
FileSystem fs = null;
fs = fileObject.getFileSystem();
manager.closeFileSystem(fs);
}
} finally {
}
}
}
@Override
public void validatePlatform() throws InstallException {
super.validatePlatform();
try {
boolean valid = true;
FileObject catalinaJar = targetLibDir.resolveFile("catalina.jar");
if (catalinaJar == null
|| !catalinaJar.exists()
|| !catalinaJar.getType().getName().equals(FileType.FILE.getName())) {
valid = false;
getPrinter().printErrStatus("CatalinaHome", "Cannot find catalina");
}
FileObject serverXml = targetConfDir.resolveFile("server.xml");
if (serverXml == null
|| !serverXml.exists()
|| !serverXml.getType().getName().equals(FileType.FILE.getName())) {
valid = false;
getPrinter().printErrStatus("CatalinaHome", "Cannot find server.xml");
}
// TODO : Validate Version ?
if (!valid)
throw new InstallException(
"Target does not seem a " + getTargetPlatform().getDescription() + " install.");
} catch (IOException e) {
getPrinter().printErrStatus("CatalinaHome", e.getMessage());
throw new InstallException(e.getMessage(), e);
}
getPrinter().printOkStatus("CatalinaHome");
}
@Test
public void testLocalFileProvider() throws Exception {
FileObject fo = null;
try {
fo = manager.resolveFile(testFile.getCanonicalPath());
assertTrue(fo.exists());
final InputStream ios = fo.getContent().getInputStream();
final BufferedReader reader = new BufferedReader(new InputStreamReader(ios));
assertEquals("test", reader.readLine());
} finally {
if (fo != null) fo.close();
}
}
public void okPressed() {
if (fileDialogMode == VFS_DIALOG_SAVEAS && "".equals(fileNameText.getText())) { // $NON-NLS-1$
// do nothing, user did not enter a file name for saving
MessageBox messageDialog = new MessageBox(dialog, SWT.OK);
messageDialog.setText(Messages.getString("VfsFileChooserDialog.error")); // $NON-NLS-1$
messageDialog.setMessage(
Messages.getString("VfsFileChooserDialog.noFilenameEntered")); // $NON-NLS-1$
messageDialog.open();
return;
}
if (fileDialogMode == VFS_DIALOG_SAVEAS) {
try {
FileObject toBeSavedFile =
vfsBrowser.getSelectedFileObject().resolveFile(fileNameText.getText());
if (toBeSavedFile.exists()) {
MessageBox messageDialog = new MessageBox(dialog, SWT.YES | SWT.NO);
messageDialog.setText(
Messages.getString("VfsFileChooserDialog.fileExists")); // $NON-NLS-1$
messageDialog.setMessage(
Messages.getString("VfsFileChooserDialog.fileExistsOverwrite")); // $NON-NLS-1$
int flag = messageDialog.open();
if (flag == SWT.NO) {
return;
}
}
} catch (FileSystemException e) {
e.printStackTrace();
}
}
if (fileDialogMode == VFS_DIALOG_SAVEAS) {
enteredFileName = fileNameText.getText();
}
try {
if (fileDialogMode == VFS_DIALOG_OPEN_FILE
&& vfsBrowser.getSelectedFileObject().getType().equals(FileType.FOLDER)) {
// try to open this node, it is a directory
vfsBrowser.selectTreeItemByFileObject(vfsBrowser.getSelectedFileObject(), true);
return;
}
} catch (FileSystemException e) {
}
okPressed = true;
hideCustomPanelChildren();
dialog.dispose();
}
/**
* Does a 'cd' command. If the taget directory does not exist, a message is printed to <code>
* System.err</code>.
*/
private void cd(final String[] cmd) throws Exception {
final String path;
if (cmd.length > 1) {
path = cmd[1];
} else {
path = System.getProperty("user.home");
}
// Locate and validate the folder
FileObject tmp = mgr.resolveFile(cwd, path);
if (tmp.exists()) {
cwd = tmp;
} else {
System.out.println("Folder does not exist: " + tmp.getName());
}
System.out.println("Current folder is " + cwd.getName());
}
@Override
public boolean backupAgentConfigurations(boolean remove) {
try {
// backup portal-ext.properties
FileObject portalConfFile = targetConfDir.resolveFile("portal-ext.properties");
if (portalConfFile.exists()) {
// backup file in the same folder it is installed
backupFile(portalConfFile, portalConfFile.getParent());
if (remove) {
portalConfFile.delete();
}
}
} catch (Exception e) {
getPrinter().printErrStatus("BackupAgentConfigurations", e.getMessage());
return false;
}
return true;
}
/**
* Check existence of a local file
*
* @param filename
* @return true, if file exists
*/
public boolean FileExists(String filename) {
FileObject file = null;
try {
file = KettleVFS.getFileObject(filename, this);
if (!file.exists()) {
return false;
} else {
if (file.getType() == FileType.FILE) {
return true;
} else {
return false;
}
}
} catch (Exception e) {
return false;
}
}
@Override
public void installConfiguration(JOSSOArtifact artifact, boolean replace)
throws InstallException {
try {
FileObject srcFile = getFileSystemManager().resolveFile(artifact.getLocation());
String name = srcFile.getName().getBaseName();
if (name.equals("portal-log4j-ext.xml") || name.equals("log4j.dtd")) {
FileObject metaInfDir = targetConfDir.resolveFile("META-INF/");
if (!metaInfDir.exists()) metaInfDir.createFolder();
installFile(srcFile, metaInfDir, replace);
} else {
installFile(srcFile, this.targetConfDir, replace);
}
} catch (IOException e) {
throw new InstallException(e.getMessage(), e);
}
}
/**
* Attempt to find any Hadoop configuration as a direct descendant of the provided directory.
*
* @param baseDir Directory to look for Hadoop configurations in
* @throws ConfigurationException
*/
private void findHadoopConfigurations(
FileObject baseDir, ActiveHadoopConfigurationLocator activeLocator)
throws ConfigurationException {
configurations = new HashMap<String, HadoopConfiguration>();
try {
if (!baseDir.exists()) {
throw new ConfigurationException(
BaseMessages.getString(
PKG, "Error.HadoopConfigurationDirectoryDoesNotExist", baseDir.getURL()));
}
for (FileObject f :
baseDir.findFiles(
new FileSelector() {
@Override
public boolean includeFile(FileSelectInfo info) throws Exception {
return info.getDepth() == 1 && FileType.FOLDER.equals(info.getFile().getType());
}
@Override
public boolean traverseDescendents(FileSelectInfo info) throws Exception {
return info.getDepth() == 0;
}
})) {
// Only load the specified configuration (ID should match the basename, we allow
// case-insensitivity)
if (f.getName().getBaseName().equalsIgnoreCase(activeLocator.getActiveConfigurationId())) {
HadoopConfiguration config = loadHadoopConfiguration(f);
if (config != null) {
configurations.put(config.getIdentifier(), config);
}
}
}
} catch (FileSystemException ex) {
throw new ConfigurationException(
BaseMessages.getString(
PKG, "Error.UnableToLoadConfigurations", baseDir.getName().getFriendlyURI()),
ex);
}
}
@Override
public boolean backupAgentConfigurations(boolean remove) {
try {
super.backupAgentConfigurations(remove);
// backup jaas.conf
FileObject jaasConfigFile = targetConfDir.resolveFile("jaas.conf");
if (jaasConfigFile.exists()) {
// backup file in the same folder it is installed
backupFile(jaasConfigFile, jaasConfigFile.getParent());
if (remove) {
jaasConfigFile.delete();
}
}
// backup setenv.sh and setenv.bat
FileObject[] libs = targetBinDir.getChildren();
for (int i = 0; i < libs.length; i++) {
FileObject cfgFile = libs[i];
if (!cfgFile.getType().getName().equals(FileType.FILE.getName())) {
// ignore folders
continue;
}
if (cfgFile.getName().getBaseName().startsWith("setenv")
&& (cfgFile.getName().getBaseName().endsWith(".sh")
|| cfgFile.getName().getBaseName().endsWith(".bat"))) {
// backup files in the same folder they're installed in
backupFile(cfgFile, cfgFile.getParent());
if (remove) {
cfgFile.delete();
}
}
}
} catch (Exception e) {
getPrinter().printErrStatus("BackupAgentConfigurations", e.getMessage());
return false;
}
return true;
}
/**
* Since the exported transformation that runs this will reside in a ZIP file, we can't reference
* files relatively. So what this does is turn the name of files into absolute paths OR it simply
* includes the resource in the ZIP file. For now, we'll simply turn it into an absolute path and
* pray that the file is on a shared drive or something like that.
*
* <p>TODO: create options to configure this behavior
*/
public String exportResources(
VariableSpace space,
Map<String, ResourceDefinition> definitions,
ResourceNamingInterface resourceNamingInterface,
Repository repository)
throws KettleException {
try {
// The object that we're modifying here is a copy of the original!
// So let's change the filename from relative to absolute by grabbing the file object...
// In case the name of the file comes from previous steps, forget about this!
//
if (Const.isEmpty(filenameField)) {
// From : ${Internal.Transformation.Filename.Directory}/../foo/bar.csv
// To : /home/matt/test/files/foo/bar.csv
//
FileObject fileObject = KettleVFS.getFileObject(space.environmentSubstitute(filename));
// If the file doesn't exist, forget about this effort too!
//
if (fileObject.exists()) {
// Convert to an absolute path...
//
filename =
resourceNamingInterface.nameResource(
fileObject.getName().getBaseName(),
fileObject.getParent().getName().getPath(),
space.toString(),
FileNamingType.DATA_FILE);
return filename;
}
}
return null;
} catch (Exception e) {
throw new KettleException(e); // $NON-NLS-1$
}
}
public Result execute(Result previousResult, int nr, Repository rep, Job parentJob) {
LogWriter log = LogWriter.getInstance();
Result result = previousResult;
result.setResult(false);
if (filename != null) {
String realFilename = getRealFilename();
try {
FileObject file = KettleVFS.getFileObject(realFilename);
if (file.exists() && file.isReadable()) {
log.logDetailed(
toString(),
Messages.getString("JobEntryFileExists.File_Exists", realFilename)); // $NON-NLS-1$
result.setResult(true);
} else {
log.logDetailed(
toString(),
Messages.getString(
"JobEntryFileExists.File_Does_Not_Exist", realFilename)); // $NON-NLS-1$
}
} catch (IOException e) {
result.setNrErrors(1);
log.logError(
toString(),
Messages.getString(
"JobEntryFileExists.ERROR_0004_IO_Exception", e.toString())); // $NON-NLS-1$
}
} else {
result.setNrErrors(1);
log.logError(
toString(),
Messages.getString("JobEntryFileExists.ERROR_0005_No_Filename_Defined")); // $NON-NLS-1$
}
return result;
}
public void setPath(String path) {
FileSystemManager fileSystemManager;
try {
fileSystemManager = VFS.getManager();
FileObject fileObject;
fileObject = fileSystemManager.resolveFile(path);
if (fileObject == null) {
throw new IOException("File cannot be resolved: " + path);
}
if (!fileObject.exists()) {
throw new IOException("File does not exist: " + path);
}
repoURL = fileObject.getURL();
if (repoURL == null) {
throw new Exception("Cannot load connection repository from path: " + path);
} else {
load();
}
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public boolean updateAgentConfiguration(String idpHostName, String idpPort, String idpType) {
boolean updated;
updated =
super.updateAgentConfiguration(
idpHostName,
idpPort,
idpType); // To change body of overridden methods use File | Settings | File Templates.
try {
log.debug("targetJOSSOConfDir = " + targetJOSSOConfDir);
FileObject agentConfigFile = targetJOSSOConfDir.resolveFile("josso-agent-config.xml");
if (agentConfigFile.exists()) {
// Get a DOM document of the josso-agent-config.xml
Node configXmlDom = readContentAsDom(agentConfigFile);
String updateSchemaLocations =
"<xupdate:update select=\"//@xsi:schemaLocation\">"
+ "http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd "
+ " urn:org:josso:agent:liferay5 jar:"
+ targetLibDir
+ "/josso-liferay5-agent-"
+ getProperty("version")
+ ".jar!/josso-liferay5-agent.xsd"
+ " urn:org:josso:protocol:client jar:"
+ targetLibDir
+ "/josso-agents-bin-"
+ getProperty("version")
+ ".jar!/josso-protocol-client.xsd "
+ " urn:org:josso:agent:core jar:"
+ targetLibDir
+ "/josso-agents-bin-"
+ getProperty("version")
+ ".jar!/josso-agent.xsd"
+ ""
+ "</xupdate:update>";
String updateSchemaLocationQryStr =
XUpdateUtil.XUPDATE_START + updateSchemaLocations + XUpdateUtil.XUPDATE_END;
log.debug("XUPDATE QUERY: \n" + updateSchemaLocationQryStr);
XUpdateQuery updateSchemaLocationQry = new XUpdateQueryImpl();
updateSchemaLocationQry.setQString(updateSchemaLocationQryStr);
updateSchemaLocationQry.execute(configXmlDom);
getPrinter().printActionOkStatus("Configure", "Schema Locations", "");
// Write modifications to file
writeContentFromDom(configXmlDom, agentConfigFile);
getPrinter()
.printActionOkStatus(
"Save",
agentConfigFile.getName().getBaseName(),
agentConfigFile.getName().getFriendlyURI());
}
} catch (Exception e) {
log.error("Error injecting schema locations to agent configuration", e);
getPrinter().printErrStatus("UpdateAgentConfiguration", e.getMessage());
updated = false;
}
return updated;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
// starttime (in seconds)
long timeStart = System.currentTimeMillis() / 1000;
if (filename != null) {
FileObject fileObject = null;
String realFilename = getRealFilename();
try {
fileObject = KettleVFS.getFileObject(realFilename, this);
long iMaximumTimeout =
Const.toInt(getRealMaximumTimeout(), Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0));
long iCycleTime =
Const.toInt(getRealCheckCycleTime(), Const.toInt(DEFAULT_CHECK_CYCLE_TIME, 0));
//
// Sanity check on some values, and complain on insanity
//
if (iMaximumTimeout < 0) {
iMaximumTimeout = Const.toInt(DEFAULT_MAXIMUM_TIMEOUT, 0);
if (log.isBasic()) logBasic("Maximum timeout invalid, reset to " + iMaximumTimeout);
}
if (iCycleTime < 1) {
// If lower than 1 set to the default
iCycleTime = Const.toInt(DEFAULT_CHECK_CYCLE_TIME, 1);
if (log.isBasic()) logBasic("Check cycle time invalid, reset to " + iCycleTime);
}
if (iMaximumTimeout == 0) {
if (log.isBasic()) logBasic("Waiting indefinitely for file [" + realFilename + "]");
} else {
if (log.isBasic())
logBasic("Waiting " + iMaximumTimeout + " seconds for file [" + realFilename + "]");
}
boolean continueLoop = true;
while (continueLoop && !parentJob.isStopped()) {
fileObject = KettleVFS.getFileObject(realFilename, this);
if (fileObject.exists()) {
// file exists, we're happy to exit
if (log.isBasic()) logBasic("Detected file [" + realFilename + "] within timeout");
result.setResult(true);
continueLoop = false;
// add filename to result filenames
if (addFilenameToResult && fileObject.getType() == FileType.FILE) {
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL, fileObject, parentJob.getJobname(), toString());
resultFile.setComment(BaseMessages.getString(PKG, "JobWaitForFile.FilenameAdded"));
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
}
} else {
long now = System.currentTimeMillis() / 1000;
if ((iMaximumTimeout > 0) && (now > (timeStart + iMaximumTimeout))) {
continueLoop = false;
// file doesn't exist after timeout, either true or false
if (isSuccessOnTimeout()) {
if (log.isBasic())
logBasic("Didn't detect file [" + realFilename + "] before timeout, success");
result.setResult(true);
} else {
if (log.isBasic())
logBasic("Didn't detect file [" + realFilename + "] before timeout, failure");
result.setResult(false);
}
}
// sleep algorithm
long sleepTime = 0;
if (iMaximumTimeout == 0) {
sleepTime = iCycleTime;
} else {
if ((now + iCycleTime) < (timeStart + iMaximumTimeout)) {
sleepTime = iCycleTime;
} else {
sleepTime = iCycleTime - ((now + iCycleTime) - (timeStart + iMaximumTimeout));
}
}
try {
if (sleepTime > 0) {
if (log.isDetailed()) {
logDetailed(
"Sleeping "
+ sleepTime
+ " seconds before next check for file ["
+ realFilename
+ "]");
}
Thread.sleep(sleepTime * 1000);
}
} catch (InterruptedException e) {
// something strange happened
result.setResult(false);
continueLoop = false;
}
}
}
if (!parentJob.isStopped() && fileObject.exists() && isFileSizeCheck()) {
long oldSize = -1;
long newSize = fileObject.getContent().getSize();
if (log.isDetailed())
logDetailed("File [" + realFilename + "] is " + newSize + " bytes long");
if (log.isBasic())
logBasic(
"Waiting until file ["
+ realFilename
+ "] stops growing for "
+ iCycleTime
+ " seconds");
while (oldSize != newSize && !parentJob.isStopped()) {
try {
if (log.isDetailed()) {
logDetailed(
"Sleeping "
+ iCycleTime
+ " seconds, waiting for file ["
+ realFilename
+ "] to stop growing");
}
Thread.sleep(iCycleTime * 1000);
} catch (InterruptedException e) {
// something strange happened
result.setResult(false);
continueLoop = false;
}
oldSize = newSize;
newSize = fileObject.getContent().getSize();
if (log.isDetailed()) {
logDetailed("File [" + realFilename + "] is " + newSize + " bytes long");
}
}
if (log.isBasic())
logBasic("Stopped waiting for file [" + realFilename + "] to stop growing");
}
if (parentJob.isStopped()) {
result.setResult(false);
}
} catch (Exception e) {
logBasic("Exception while waiting for file [" + realFilename + "] to stop growing", e);
} finally {
if (fileObject != null) {
try {
fileObject.close();
} catch (Exception e) {
}
}
}
} else {
logError("No filename is defined.");
}
return result;
}
public Result execute(Result previousResult, int nr) {
Result result = previousResult;
result.setResult(false);
try {
// Get real variable value
String realServerName = environmentSubstitute(serverName);
int realServerPort = Const.toInt(environmentSubstitute(serverPort), 22);
String realUserName = environmentSubstitute(userName);
String realServerPassword =
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(password));
// Proxy Host
String realProxyHost = environmentSubstitute(httpproxyhost);
int realProxyPort = Const.toInt(environmentSubstitute(httpproxyport), 22);
String realproxyUserName = environmentSubstitute(httpproxyusername);
String realProxyPassword =
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(httpProxyPassword));
// Key file
String realKeyFilename = environmentSubstitute(keyFilename);
String relKeyFilepass = environmentSubstitute(keyFilePass);
// Source files
String realLocalDirectory = environmentSubstitute(localDirectory);
String realwildcard = environmentSubstitute(wildcard);
// Remote destination
String realftpDirectory = environmentSubstitute(ftpDirectory);
// Destination folder (Move to)
String realDestinationFolder = environmentSubstitute(destinationfolder);
try {
// Remote source
realftpDirectory = FTPUtils.normalizePath(realftpDirectory);
// Destination folder (Move to)
realDestinationFolder = FTPUtils.normalizePath(realDestinationFolder);
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.CanNotNormalizePath", e.getMessage()));
result.setNrErrors(1);
return result;
}
// Check for mandatory fields
boolean mandatoryok = true;
if (Const.isEmpty(realServerName)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.ServernameMissing"));
}
if (usehttpproxy) {
if (Const.isEmpty(realProxyHost)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.HttpProxyhostMissing"));
}
}
if (publicpublickey) {
if (Const.isEmpty(realKeyFilename)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileMissing"));
} else {
// Let's check if folder exists...
if (!KettleVFS.fileExists(realKeyFilename, this)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.KeyFileNotExist"));
}
}
}
if (Const.isEmpty(realLocalDirectory)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.LocalFolderMissing"));
}
if (afterFtpPut.equals("move_file")) {
if (Const.isEmpty(realDestinationFolder)) {
mandatoryok = false;
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.DestinatFolderMissing"));
} else {
FileObject folder = null;
try {
folder = KettleVFS.getFileObject(realDestinationFolder, this);
// Let's check if folder exists...
if (!folder.exists()) {
// Do we need to create it?
if (createDestinationFolder) {
folder.createFolder();
} else {
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.DestinatFolderNotExist", realDestinationFolder));
}
}
} catch (Exception e) {
throw new KettleException(e);
} finally {
if (folder != null) {
try {
folder.close();
folder = null;
} catch (Exception e) {
/* Ignore */
}
}
}
}
}
if (mandatoryok) {
Connection conn = null;
SFTPv3Client client = null;
boolean good = true;
int nbfilestoput = 0;
int nbput = 0;
int nbrerror = 0;
try {
// Create a connection instance
conn =
getConnection(
realServerName,
realServerPort,
realProxyHost,
realProxyPort,
realproxyUserName,
realProxyPassword);
if (timeout > 0) {
// Use timeout
// Cache Host Key
if (cachehostkey) {
conn.connect(new SimpleVerifier(database), 0, timeout * 1000);
} else {
conn.connect(null, 0, timeout * 1000);
}
} else {
// Cache Host Key
if (cachehostkey) {
conn.connect(new SimpleVerifier(database));
} else {
conn.connect();
}
}
// Authenticate
boolean isAuthenticated = false;
if (publicpublickey) {
String keyContent =
KettleVFS.getTextFileContent(realKeyFilename, this, Const.XML_ENCODING);
isAuthenticated =
conn.authenticateWithPublicKey(
realUserName, keyContent.toCharArray(), relKeyFilepass);
} else {
isAuthenticated = conn.authenticateWithPassword(realUserName, realServerPassword);
}
// LET'S CHECK AUTHENTICATION ...
if (isAuthenticated == false) {
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.AuthenticationFailed"));
} else {
if (log.isBasic()) {
logBasic(
BaseMessages.getString(PKG, "JobSSH2PUT.Log.Connected", serverName, userName));
}
client = new SFTPv3Client(conn);
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.ProtocolVersion", "" + client.getProtocolVersion()));
}
// Check if remote directory exists
if (!Const.isEmpty(realftpDirectory)) {
if (!sshDirectoryExists(client, realftpDirectory)) {
good = false;
if (createRemoteFolder) {
good = CreateRemoteFolder(client, realftpDirectory);
if (good) {
logBasic(BaseMessages.getString(PKG, "JobSSH2PUT.Log.RemoteDirectoryCreated"));
}
} else {
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.RemoteDirectoryNotExist", realftpDirectory));
}
} else if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.RemoteDirectoryExist", realftpDirectory));
}
}
if (good) {
// Get files list from local folder (source)
List<FileObject> myFileList = getFiles(realLocalDirectory);
// Prepare Pattern for wildcard
Pattern pattern = null;
if (!Const.isEmpty(realwildcard)) {
pattern = Pattern.compile(realwildcard);
}
// Let's put files now ...
// Get the files in the list
for (int i = 0; i < myFileList.size() && !parentJob.isStopped(); i++) {
FileObject myFile = myFileList.get(i);
String localFilename = myFile.toString();
String remoteFilename = myFile.getName().getBaseName();
boolean getIt = true;
// First see if the file matches the regular expression!
if (pattern != null) {
Matcher matcher = pattern.matcher(remoteFilename);
getIt = matcher.matches();
}
// do we have a target directory?
if (!Const.isEmpty(realftpDirectory)) {
remoteFilename = realftpDirectory + FTPUtils.FILE_SEPARATOR + remoteFilename;
}
if (onlyGettingNewFiles) {
// We get only new files
// ie not exist on the remote server
getIt = !sshFileExists(client, remoteFilename);
}
if (getIt) {
nbfilestoput++;
boolean putok = putFile(myFile, remoteFilename, client);
if (!putok) {
nbrerror++;
logError(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Error.CanNotPutFile", localFilename));
} else {
nbput++;
}
if (putok && !afterFtpPut.equals("do_nothing")) {
deleteOrMoveFiles(myFile, realDestinationFolder);
}
}
}
/** ****************************** RESULT ******************* */
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd1"));
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Result.TotalFiles", "" + nbfilestoput));
logDetailed(
BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.TotalFilesPut", "" + nbput));
logDetailed(
BaseMessages.getString(
PKG, "JobSSH2PUT.Log.Result.TotalFilesError", "" + nbrerror));
logDetailed(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Result.JobEntryEnd2"));
}
if (nbrerror == 0) {
result.setResult(true);
/** ****************************** RESULT ******************* */
}
}
}
} catch (Exception e) {
result.setNrErrors(nbrerror);
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.ErrorFTP", e.getMessage()));
} finally {
if (conn != null) {
conn.close();
}
if (client != null) {
client.close();
}
}
}
} catch (Exception e) {
result.setResult(false);
result.setNrErrors(1L);
logError(BaseMessages.getString(PKG, "JobSSH2PUT.Log.Error.UnexpectedError"), e);
}
return result;
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
meta = (XsltMeta) smi;
data = (XsltData) sdi;
Object[] row = getRow();
if (row == null) { // no more input to be expected...
setOutputDone();
return false;
}
if (first) {
first = false;
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);
// Check if The result field is given
if (Const.isEmpty(meta.getResultfieldname())) {
// Result Field is missing !
logError(BaseMessages.getString(PKG, "Xslt.Log.ErrorResultFieldMissing"));
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.ErrorResultFieldMissing"));
}
// Check if The XML field is given
if (Const.isEmpty(meta.getFieldname())) {
// Result Field is missing !
logError(BaseMessages.getString(PKG, "Xslt.Exception.ErrorXMLFieldMissing"));
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.ErrorXMLFieldMissing"));
}
// Try to get XML Field index
data.fieldposition = getInputRowMeta().indexOfValue(meta.getFieldname());
// Let's check the Field
if (data.fieldposition < 0) {
// The field is unreachable !
logError(
BaseMessages.getString(PKG, "Xslt.Log.ErrorFindingField")
+ "["
+ meta.getFieldname()
+ "]");
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.CouldnotFindField", meta.getFieldname()));
}
// Check if the XSL Filename is contained in a column
if (meta.useXSLField()) {
if (Const.isEmpty(meta.getXSLFileField())) {
// The field is missing
// Result field is missing !
logError(BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLFileFieldMissing"));
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.ErrorXSLFileFieldMissing"));
}
// Try to get Field index
data.fielxslfiledposition = getInputRowMeta().indexOfValue(meta.getXSLFileField());
// Let's check the Field
if (data.fielxslfiledposition < 0) {
// The field is unreachable !
logError(
BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLFileFieldFinding")
+ "["
+ meta.getXSLFileField()
+ "]");
throw new KettleStepException(
BaseMessages.getString(
PKG, "Xslt.Exception.ErrorXSLFileFieldFinding", meta.getXSLFileField()));
}
} else {
if (Const.isEmpty(meta.getXslFilename())) {
logError(BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLFile"));
throw new KettleStepException(BaseMessages.getString(PKG, "Xslt.Exception.ErrorXSLFile"));
}
// Check if XSL File exists!
data.xslfilename = environmentSubstitute(meta.getXslFilename());
FileObject file = null;
try {
file = KettleVFS.getFileObject(data.xslfilename);
if (!file.exists()) {
logError(
BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLFileNotExists", data.xslfilename));
throw new KettleStepException(
BaseMessages.getString(
PKG, "Xslt.Exception.ErrorXSLFileNotExists", data.xslfilename));
}
if (file.getType() != FileType.FILE) {
logError(BaseMessages.getString(PKG, "Xslt.Log.ErrorXSLNotAFile", data.xslfilename));
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.ErrorXSLNotAFile", data.xslfilename));
}
} catch (Exception e) {
throw new KettleStepException(e);
} finally {
try {
if (file != null) {
file.close();
}
} catch (Exception e) {
/* Ignore */
}
}
}
// Check output parameters
int nrOutputProps =
meta.getOutputPropertyName() == null ? 0 : meta.getOutputPropertyName().length;
if (nrOutputProps > 0) {
data.outputProperties = new Properties();
for (int i = 0; i < nrOutputProps; i++) {
data.outputProperties.put(
meta.getOutputPropertyName()[i],
environmentSubstitute(meta.getOutputPropertyValue()[i]));
}
data.setOutputProperties = true;
}
// Check parameters
data.nrParams = meta.getParameterField() == null ? 0 : meta.getParameterField().length;
if (data.nrParams > 0) {
data.indexOfParams = new int[data.nrParams];
data.nameOfParams = new String[data.nrParams];
for (int i = 0; i < data.nrParams; i++) {
String name = environmentSubstitute(meta.getParameterName()[i]);
String field = environmentSubstitute(meta.getParameterField()[i]);
if (Const.isEmpty(field)) {
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.ParameterFieldMissing", name, i));
}
data.indexOfParams[i] = getInputRowMeta().indexOfValue(field);
if (data.indexOfParams[i] < 0) {
throw new KettleStepException(
BaseMessages.getString(PKG, "Xslt.Exception.ParameterFieldNotFound", name));
}
data.nameOfParams[i] = name;
}
data.useParameters = true;
}
data.factory = TransformerFactory.newInstance();
if (meta.getXSLFactory().equals("SAXON")) {
// Set the TransformerFactory to the SAXON implementation.
data.factory = new net.sf.saxon.TransformerFactoryImpl();
}
} // end if first
// Get the field value
String xmlValue = getInputRowMeta().getString(row, data.fieldposition);
if (meta.useXSLField()) {
// Get the value
data.xslfilename = getInputRowMeta().getString(row, data.fielxslfiledposition);
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "Xslt.Log.XslfileNameFromFied", data.xslfilename, meta.getXSLFileField()));
}
}
try {
if (log.isDetailed()) {
if (meta.isXSLFieldIsAFile()) {
logDetailed(BaseMessages.getString(PKG, "Xslt.Log.Filexsl") + data.xslfilename);
} else {
logDetailed(BaseMessages.getString(PKG, "Xslt.Log.XslStream", data.xslfilename));
}
}
// Get the template from the cache
Transformer transformer = data.getTemplate(data.xslfilename, data.xslIsAfile);
// Do we need to set output properties?
if (data.setOutputProperties) {
transformer.setOutputProperties(data.outputProperties);
}
// Do we need to pass parameters?
if (data.useParameters) {
for (int i = 0; i < data.nrParams; i++) {
transformer.setParameter(data.nameOfParams[i], row[data.indexOfParams[i]]);
}
}
Source source = new StreamSource(new StringReader(xmlValue));
// Prepare output stream
StreamResult result = new StreamResult(new StringWriter());
// transform xml source
transformer.transform(source, result);
String xmlString = result.getWriter().toString();
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "Xslt.Log.FileResult"));
logDetailed(xmlString);
}
Object[] outputRowData = RowDataUtil.addValueData(row, getInputRowMeta().size(), xmlString);
if (log.isRowLevel()) {
logRowlevel(
BaseMessages.getString(PKG, "Xslt.Log.ReadRow")
+ " "
+ getInputRowMeta().getString(row));
}
// add new values to the row.
putRow(data.outputRowMeta, outputRowData); // copy row to output rowset(s);
} catch (Exception e) {
String errorMessage = e.getMessage();
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
DefaultErrorHandler.printLocation(pw, e);
pw.close();
errorMessage = sw.toString() + "\n" + errorMessage;
if (getStepMeta().isDoingErrorHandling()) {
// Simply add this row to the error row
putError(getInputRowMeta(), row, 1, errorMessage, meta.getResultfieldname(), "XSLT01");
} else {
logError(BaseMessages.getString(PKG, "Xslt.ErrorProcesing" + " : " + errorMessage));
throw new KettleStepException(BaseMessages.getString(PKG, "Xslt.ErrorProcesing"), e);
}
}
return true;
}
public String getFileContent(String fileName) throws Exception {
FileSystemManager fsManager = null;
FileObject sftpFile = null;
FileObject src = null; // used for cleanup in release()
String fileContent = null;
try {
System.out.println("SFTP download");
FileSystemOptions opts = null;
// app.initialize();
try {
fsManager = VFS.getManager();
} catch (FileSystemException ex) {
throw new RuntimeException("failed to get fsManager from VFS", ex);
}
UserAuthenticator auth = new StaticUserAuthenticator(null, this.user, this.password);
opts = new FileSystemOptions();
try {
DefaultFileSystemConfigBuilder.getInstance().setUserAuthenticator(opts, auth);
SftpFileSystemConfigBuilder.getInstance().setStrictHostKeyChecking(opts, "no");
} catch (FileSystemException ex) {
throw new RuntimeException("setUserAuthenticator failed", ex);
}
// app.process();
String startPath =
"sftp://"
+ this.user
+ ":"
+ this.password
+ "@"
+ this.host
+ ":22"
+ this.remoteDir
+ fileName;
// Set starting path on remote SFTP server.
try {
sftpFile = fsManager.resolveFile(startPath, opts);
System.out.println("SFTP connection successfully established to " + startPath);
} catch (FileSystemException ex) {
throw new RuntimeException("SFTP error parsing path " + this.remoteDir, ex);
}
if (sftpFile.exists()) {
FileContent fc = sftpFile.getContent();
StringWriter writer = new StringWriter();
IOUtils.copy(fc.getInputStream(), writer, encoding);
String theString = writer.toString();
// System.out.println(theString);
fileContent = theString;
}
} finally {
// app.release();
/** Release system resources, close connection to the filesystem. */
try {
FileSystem fs = null;
if (sftpFile != null) {
fs = sftpFile.getFileSystem(); // This works even if the src is closed.
fsManager.closeFileSystem(fs);
} // TODO if sftpFile != null
} catch (Exception e) {
System.out.println(e);
}
}
return fileContent;
}
public Object execute(final CommandContext context) throws Exception {
assert context != null;
IO io = context.getIo();
FileObject file = resolveFile(context, path);
io.println("URL: {}", file.getURL());
io.println("Name: {}", file.getName());
io.println("BaseName: {}", file.getName().getBaseName());
io.println("Extension: {}", file.getName().getExtension());
io.println("Path: {}", file.getName().getPath());
io.println("Scheme: {}", file.getName().getScheme());
io.println("URI: {}", file.getName().getURI());
io.println("Root URI: {}", file.getName().getRootURI());
io.println("Parent: {}", file.getName().getParent());
io.println("Type: {}", file.getType());
io.println("Exists: {}", file.exists());
io.println("Readable: {}", file.isReadable());
io.println("Writeable: {}", file.isWriteable());
io.println("Root path: {}", file.getFileSystem().getRoot().getName().getPath());
if (file.exists()) {
FileContent content = file.getContent();
FileContentInfo contentInfo = content.getContentInfo();
io.println("Content type: {}", contentInfo.getContentType());
io.println("Content encoding: {}", contentInfo.getContentEncoding());
try {
// noinspection unchecked
Map<String, Object> attrs = content.getAttributes();
if (attrs != null && !attrs.isEmpty()) {
io.println("Attributes:");
for (Map.Entry<String, Object> entry : attrs.entrySet()) {
io.println(" {}='{}'", entry.getKey(), entry.getValue());
}
}
} catch (FileSystemException e) {
io.println("File attributes are NOT supported");
}
try {
Certificate[] certs = content.getCertificates();
if (certs != null && certs.length != 0) {
io.println("Certificate:");
for (Certificate cert : certs) {
io.println(" {}", cert);
}
}
} catch (FileSystemException e) {
io.println("File certificates are NOT supported");
}
if (file.getType().equals(FileType.FILE)) {
io.println("Size: {} bytes", content.getSize());
} else if (file.getType().hasChildren() && file.isReadable()) {
FileObject[] children = file.getChildren();
io.println("Directory with {} files", children.length);
for (int iterChildren = 0; iterChildren < children.length; iterChildren++) {
io.println("#{}:{}", iterChildren, children[iterChildren].getName());
if (iterChildren > 5) {
break;
}
}
}
io.println(
"Last modified: {}",
DateFormat.getInstance().format(new Date(content.getLastModifiedTime())));
} else {
io.println("The file does not exist");
}
FileObjects.close(file);
return Result.SUCCESS;
}