/**
* Parse a set of URLs from a comma-separated list of URLs. If the URL points to a directory all
* jar files within that directory will be returned as well.
*
* @param urlString Comma-separated list of URLs (relative or absolute)
* @return List of URLs resolved from {@code urlString}
*/
protected List<URL> parseURLs(FileObject root, String urlString) {
if (urlString == null || urlString.trim().isEmpty()) {
return Collections.emptyList();
}
String[] paths = urlString.split(",");
List<URL> urls = new ArrayList<URL>();
for (String path : paths) {
try {
FileObject file = root.resolveFile(path.trim());
if (!file.exists()) {
file = defaultFsm.resolveFile(path.trim());
}
if (FileType.FOLDER.equals(file.getType())) {
// Add directories with a trailing / so the URL ClassLoader interprets
// them as directories
urls.add(new URL(file.getURL().toExternalForm() + "/"));
// Also add all jars within this directory
urls.addAll(findJarsIn(file, 1, new HashSet<String>()));
} else {
urls.add(file.getURL());
}
} catch (Exception e) {
// Log invalid path
logger.error(BaseMessages.getString(PKG, "Error.InvalidClasspathEntry", path));
}
}
return urls;
}
@Override
public void run() {
String sourceUrl = "NOT YET DEFINED";
String destUrl = "NOT YET DEFINED";
try {
sourceUrl = source.getURL().toString();
destUrl = dest.getURL().toString();
logger.debug("Copying files from " + source + " to " + dest);
dest.copyFrom(source, fileSelector);
logger.debug("Finished copying files from " + source + " to " + dest);
} catch (FileSystemException e) {
logger.error("An error occured while copying files from " + source + " to " + dest, e);
if (jobId != null) {
logger.info(
"Job "
+ jobId
+ " will be removed from the known job list. The system will not attempt again to retrieve data for this job. You could try to manually copy the data from the location "
+ sourceUrl);
for (ISchedulerEventListenerExtended l : eventListeners) {
l.pullDataFailed(jobId, sourceUrl, e);
}
removeAwaitedJob(jobId);
return;
}
} // catch
removeAwaitedJob(jobId);
// delete source data
String url = "NOT YET DEFINED";
for (FileObject fo : foldersToDelete) {
try {
url = fo.getURL().toString();
fo.delete(Selectors.SELECT_ALL);
fo.delete();
} catch (FileSystemException e) {
logger.warn(
"Could not delete temporary fioles at location "
+ url
+ " . although the copy of these files to "
+ destUrl
+ " has been successffully performed.");
}
}
for (ISchedulerEventListenerExtended l : eventListeners) {
l.pullDataFinished(jobId, destUrl);
}
}
/**
* Create a ClassLoader to load resources for a {@code HadoopConfiguration}.
*
* @param root Configuration root directory
* @param parent Parent class loader to delegate to if resources cannot be found in the
* configuration's directory or provided classpath
* @param classpathUrls Additional URLs to add to the class loader. These will be added before any
* internal resources.
* @param ignoredClasses Classes (or packages) that should not be loaded by the class loader
* @return A class loader capable of loading a Hadoop configuration located at {@code root}.
* @throws ConfigurationException Error creating a class loader for the Hadoop configuration
* located at {@code root}
*/
protected ClassLoader createConfigurationLoader(
FileObject root,
ClassLoader parent,
List<URL> classpathUrls,
ShimProperties configurationProperties,
String... ignoredClasses)
throws ConfigurationException {
try {
if (root == null || !FileType.FOLDER.equals(root.getType())) {
throw new IllegalArgumentException("root must be a folder: " + root);
}
// Find all jar files in the configuration, at most 2 folders deep
List<URL> jars =
findJarsIn(root, 3, configurationProperties.getConfigSet(SHIM_CLASSPATH_IGNORE));
// Add the root of the configuration
jars.add(0, new URL(root.getURL().toExternalForm() + "/"));
// Inject any overriding URLs before all other paths
if (classpathUrls != null) {
jars.addAll(0, classpathUrls);
}
return new HadoopConfigurationClassLoader(
jars.toArray(EMPTY_URL_ARRAY), parent, ignoredClasses);
} catch (Exception ex) {
throw new ConfigurationException(
BaseMessages.getString(PKG, "Error.CreatingClassLoader"), ex);
}
}
private List<URL> findJarsIn(FileObject path, final int maxdepth, final Set<String> paths)
throws FileSystemException {
FileObject[] jars =
path.findFiles(
new FileSelector() {
@Override
public boolean includeFile(FileSelectInfo info) throws Exception {
for (String path : paths) {
if (info.getFile().getURL().toString().endsWith(path)) {
return false;
}
}
return info.getFile().getName().getBaseName().endsWith(JAR_EXTENSION);
}
@Override
public boolean traverseDescendents(FileSelectInfo info) throws Exception {
for (String path : paths) {
if (info.getFile().getURL().toString().endsWith(path)) {
return false;
}
}
return info.getDepth() <= maxdepth;
}
});
List<URL> jarUrls = new ArrayList<URL>();
for (FileObject jar : jars) {
jarUrls.add(jar.getURL());
}
return jarUrls;
}
private boolean OpenFile() throws Exception {
data.oneFileOpened = true;
String realFilename = environmentSubstitute(meta.getFilename());
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "AccessOutput.log.WritingToFile", realFilename));
}
FileObject fileObject = KettleVFS.getFileObject(realFilename, getTransMeta());
File file = FileUtils.toFile(fileObject.getURL());
// First open or create the access file
if (!file.exists()) {
if (meta.isFileCreated()) {
data.db = Database.create(file);
} else {
logError(
BaseMessages.getString(PKG, "AccessOutput.InitError.FileDoesNotExist", realFilename));
return false;
}
} else {
data.db = Database.open(file);
}
// Add the filename to the result object...
//
if (meta.isAddToResultFiles()) {
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL, fileObject, getTransMeta().getName(), toString());
resultFile.setComment("This file was created with an access output step");
addResultFile(resultFile);
}
return true;
}
/**
* Tests closing all FileSystems
*
* @throws Exception
*/
@Ignore("vfs close file system doesn't seem to work properly")
@Test
public void testCloseFileSystems() throws Exception {
logger.info("*************** testCloseFileSystems");
String[] validUrls = server.getVFSRootURLs();
ArrayList<FileObject> fos = new ArrayList<FileObject>();
for (String validUrl : validUrls) {
FileObject mounted = VFSMountManagerHelper.mount(validUrl);
Assert.assertTrue(mounted.exists());
fos.add(mounted);
}
VFSMountManagerHelper.closeFileSystems(Arrays.asList(validUrls));
boolean onlyExceptions = true;
for (FileObject closedFo : fos) {
try {
FileObject toto = closedFo.resolveFile("toto");
toto.createFile();
onlyExceptions = false;
logger.error(toto.getURL() + " exists : " + toto.exists());
} catch (FileSystemException e) {
// this should occur
}
}
Assert.assertTrue("Only Exceptions received", onlyExceptions);
}
public AnnotationDB getAnnotationDB(FileObject fileObject)
throws FileSystemException, IOException {
AnnotationDB result = annotationMap.get(fileObject);
if (result == null) {
result = new AnnotationDB();
result.scanArchives(fileObject.getURL());
annotationMap.put(fileObject, result);
}
return result;
}
/**
* Attempt to find any Hadoop configuration as a direct descendant of the provided directory.
*
* @param baseDir Directory to look for Hadoop configurations in
* @throws ConfigurationException
*/
private void findHadoopConfigurations(
FileObject baseDir, ActiveHadoopConfigurationLocator activeLocator)
throws ConfigurationException {
configurations = new HashMap<String, HadoopConfiguration>();
try {
if (!baseDir.exists()) {
throw new ConfigurationException(
BaseMessages.getString(
PKG, "Error.HadoopConfigurationDirectoryDoesNotExist", baseDir.getURL()));
}
for (FileObject f :
baseDir.findFiles(
new FileSelector() {
@Override
public boolean includeFile(FileSelectInfo info) throws Exception {
return info.getDepth() == 1 && FileType.FOLDER.equals(info.getFile().getType());
}
@Override
public boolean traverseDescendents(FileSelectInfo info) throws Exception {
return info.getDepth() == 0;
}
})) {
// Only load the specified configuration (ID should match the basename, we allow
// case-insensitivity)
if (f.getName().getBaseName().equalsIgnoreCase(activeLocator.getActiveConfigurationId())) {
HadoopConfiguration config = loadHadoopConfiguration(f);
if (config != null) {
configurations.put(config.getIdentifier(), config);
}
}
}
} catch (FileSystemException ex) {
throw new ConfigurationException(
BaseMessages.getString(
PKG, "Error.UnableToLoadConfigurations", baseDir.getName().getFriendlyURI()),
ex);
}
}
public void setPath(String path) {
FileSystemManager fileSystemManager;
try {
fileSystemManager = VFS.getManager();
FileObject fileObject;
fileObject = fileSystemManager.resolveFile(path);
if (fileObject == null) {
throw new IOException("File cannot be resolved: " + path);
}
if (!fileObject.exists()) {
throw new IOException("File does not exist: " + path);
}
repoURL = fileObject.getURL();
if (repoURL == null) {
throw new Exception("Cannot load connection repository from path: " + path);
} else {
load();
}
} catch (Exception e) {
e.printStackTrace();
}
}
protected boolean configureJaasModule() {
String tcInstallDir = getProperty("tomcatInstallDir");
String jbInstallDir = getProperty("jbossInstallDir");
final String JOSSO_TOMCAT_MODULE_DEFINITION =
"\n\njosso {\n"
+ "org.josso.liferay5.agent.jaas.SSOGatewayLoginModule required debug=true;\n"
+ "};";
if (tcInstallDir != null) {
log.debug("[configureJaasModule]: Tomcat install dir: " + tcInstallDir);
try {
FileObject tomcatInstallDir = getFileSystemManager().resolveFile(tcInstallDir);
FileObject jaasConfigFile = tomcatInstallDir.resolveFile("conf/jaas.config");
if (jaasConfigFile != null) {
BufferedWriter writerJaas =
new BufferedWriter(
new OutputStreamWriter(
new FileOutputStream(jaasConfigFile.getURL().getFile(), true)));
writerJaas.write(JOSSO_TOMCAT_MODULE_DEFINITION);
writerJaas.flush();
writerJaas.close();
return true;
} else {
getPrinter()
.printActionErrStatus(
"Configure", "JOSSO SSO Filter", "jaas.conf doesn't exist on given path");
return false;
}
} catch (FileSystemException e) {
getPrinter()
.printActionErrStatus(
"Configure", "JOSSO SSO Filter", "Tomcat install directory is wrong.");
} catch (IOException e) {
getPrinter()
.printActionErrStatus("Configure", "JOSSO SSO Filter", "Can not write to jaas.conf.");
}
}
if (jbInstallDir != null) {
log.debug("[configureJaasModule]: JBoss install dir: " + jbInstallDir);
FileObject jbossInstallDir = null;
try {
jbossInstallDir = getFileSystemManager().resolveFile(jbInstallDir);
FileObject loginConfig =
jbossInstallDir.resolveFile("server/default/conf/login-config.xml");
Node xDom = readContentAsDom(loginConfig);
if (xDom == null) {
log.debug(
"[configureJaasModule]: XML is not loaded. "
+ loginConfig.getName().getFriendlyURI());
return false;
}
String xupdJossoModule =
"\n\t<xupdate:append select=\"/policy\" >\n"
+ "\t\t<xupdate:element name=\"application-policy\">\n"
+ "\t\t\t<xupdate:attribute name=\"name\">josso</xupdate:attribute>\n"
+ "\t\t\t<authentication>\n"
+ "\t\t\t\t<login-module code=\"org.josso.liferay5.agent.jaas.SSOGatewayLoginModule\" flag=\"required\">\n"
+ "\t\t\t\t\t<module-option name=\"debug\">true</module-option>\n"
+ "\t\t\t\t</login-module>\n"
+ "\t\t\t</authentication>\n"
+ "\t\t</xupdate:element>\n"
+ "\t</xupdate:append>";
String qry = XUpdateUtil.XUPDATE_START + xupdJossoModule + XUpdateUtil.XUPDATE_END;
log.debug("XUPDATE QUERY: \n" + qry);
XUpdateQuery xq = new XUpdateQueryImpl();
xq.setQString(qry);
xq.execute(xDom);
writeContentFromDom(xDom, loginConfig);
getPrinter()
.printActionOkStatus(
"Changed login-config.xml",
"JOSSO Liferay 5 Agent ",
"server/default/conf/login-config.xml");
return true;
} catch (FileSystemException e) {
getPrinter()
.printActionErrStatus(
"Configure", "JOSSO SSO Filter", "JBoss install directory is wrong.");
} catch (Exception e) {
e.printStackTrace();
}
}
return false;
}
public Object execute(final CommandContext context) throws Exception {
assert context != null;
IO io = context.getIo();
FileObject file = resolveFile(context, path);
io.println("URL: {}", file.getURL());
io.println("Name: {}", file.getName());
io.println("BaseName: {}", file.getName().getBaseName());
io.println("Extension: {}", file.getName().getExtension());
io.println("Path: {}", file.getName().getPath());
io.println("Scheme: {}", file.getName().getScheme());
io.println("URI: {}", file.getName().getURI());
io.println("Root URI: {}", file.getName().getRootURI());
io.println("Parent: {}", file.getName().getParent());
io.println("Type: {}", file.getType());
io.println("Exists: {}", file.exists());
io.println("Readable: {}", file.isReadable());
io.println("Writeable: {}", file.isWriteable());
io.println("Root path: {}", file.getFileSystem().getRoot().getName().getPath());
if (file.exists()) {
FileContent content = file.getContent();
FileContentInfo contentInfo = content.getContentInfo();
io.println("Content type: {}", contentInfo.getContentType());
io.println("Content encoding: {}", contentInfo.getContentEncoding());
try {
// noinspection unchecked
Map<String, Object> attrs = content.getAttributes();
if (attrs != null && !attrs.isEmpty()) {
io.println("Attributes:");
for (Map.Entry<String, Object> entry : attrs.entrySet()) {
io.println(" {}='{}'", entry.getKey(), entry.getValue());
}
}
} catch (FileSystemException e) {
io.println("File attributes are NOT supported");
}
try {
Certificate[] certs = content.getCertificates();
if (certs != null && certs.length != 0) {
io.println("Certificate:");
for (Certificate cert : certs) {
io.println(" {}", cert);
}
}
} catch (FileSystemException e) {
io.println("File certificates are NOT supported");
}
if (file.getType().equals(FileType.FILE)) {
io.println("Size: {} bytes", content.getSize());
} else if (file.getType().hasChildren() && file.isReadable()) {
FileObject[] children = file.getChildren();
io.println("Directory with {} files", children.length);
for (int iterChildren = 0; iterChildren < children.length; iterChildren++) {
io.println("#{}:{}", iterChildren, children[iterChildren].getName());
if (iterChildren > 5) {
break;
}
}
}
io.println(
"Last modified: {}",
DateFormat.getInstance().format(new Date(content.getLastModifiedTime())));
} else {
io.println("The file does not exist");
}
FileObjects.close(file);
return Result.SUCCESS;
}
