public void testSimpleFileObject() throws IOException { FileSystemManager fs = new FileSystemManager(); // String pathFile = "com/sinosoft/one/mvc/scanning/vfs/SimpleFileObject.class"; URL urlFile = loader.getResource(pathFile); assertNotNull(urlFile); FileObject fileObjectFile = fs.resolveFile(urlFile); assertEquals(SimpleFileObject.class, fileObjectFile.getClass()); // String pathDir = new File(urlFile.getPath()).getParent().replace('\\', '/'); pathDir = StringUtils.removeEnd(pathDir, "/"); URL urlDir = ResourceUtils.getURL(pathDir); assertNotNull(urlDir); File fileDir = new File(urlDir.getFile()); assertTrue(fileDir.exists()); FileObject fileObjectDir = fs.resolveFile(urlDir); assertEquals(SimpleFileObject.class, fileObjectDir.getClass()); File dirFile = ResourceUtils.getFile(urlDir); assertTrue(urlDir.toString().endsWith("/")); assertTrue(urlDir.getPath().endsWith("/")); assertFalse(dirFile.getPath().endsWith("/")); assertTrue(fileObjectDir.toString().endsWith("/")); // exists assertTrue(fileObjectFile.exists()); assertTrue(fileObjectDir.exists()); assertFalse(fileObjectDir.getChild("a_not_exists_file.txt").exists()); // getName assertEquals("vfs", fileObjectDir.getName().getBaseName()); assertEquals("SimpleFileObject.class", fileObjectFile.getName().getBaseName()); // getRelativeName assertEquals( "SimpleFileObject.class", fileObjectDir.getName().getRelativeName(fileObjectFile.getName())); assertEquals("", fileObjectDir.getName().getRelativeName(fileObjectDir.getName())); assertEquals("", fileObjectFile.getName().getRelativeName(fileObjectFile.getName())); // getType assertSame(FileType.FOLDER, fileObjectDir.getType()); assertSame(FileType.FILE, fileObjectFile.getType()); // getChild, getParent, and equals, getChildren assertEquals(fileObjectFile, fileObjectDir.getChild("SimpleFileObject.class")); assertEquals(fileObjectDir, fileObjectFile.getParent()); assertSame(fileObjectFile, fileObjectDir.getChild("SimpleFileObject.class")); assertSame(fileObjectDir, fileObjectFile.getParent()); assertTrue(ArrayUtils.contains(fileObjectDir.getChildren(), fileObjectFile)); // getURL assertEquals(urlDir, fileObjectDir.getURL()); assertEquals(urlFile, fileObjectFile.getURL()); }
@Override public FileObject[] getChildren() throws MalformedURLException, IOException { File[] files = file.listFiles(); FileObject[] children = new FileObject[files.length]; for (int i = 0; i < children.length; i++) { if (files[i].isDirectory()) { children[i] = fs.resolveFile(urlString + files[i].getName() + "/"); } else { children[i] = fs.resolveFile(urlString + files[i].getName()); } } return children; }
public void testJarFileRootObject() throws IOException { FileSystemManager fs = new FileSystemManager(); URL urlFile = loader.getResource("org/apache/commons/lang/StringUtils.class"); String urlString = urlFile.toString(); int index = urlString.indexOf("!/"); assertTrue(index > 0); String root = urlString.substring(0, index + 2); FileObject rootObject = fs.resolveFile(root); assertTrue(root, rootObject.exists()); assertTrue(rootObject.getChildren().length > 0); assertTrue(rootObject.getChild("org/").exists()); assertTrue(rootObject.getChild("org/") == fs.resolveFile(root + "org/")); }
/** * Take specified action to either move or delete the processed file, depending on the outcome * * @param entry the PollTableEntry for the file that has been processed * @param fileObject the FileObject representing the file to be moved or deleted */ private void moveOrDeleteAfterProcessing(final PollTableEntry entry, FileObject fileObject) throws AxisFault { String moveToDirectoryURI = null; try { switch (entry.getLastPollState()) { case PollTableEntry.SUCCSESSFUL: if (entry.getActionAfterProcess() == PollTableEntry.MOVE) { moveToDirectoryURI = entry.getMoveAfterProcess(); } break; case PollTableEntry.FAILED: if (entry.getActionAfterFailure() == PollTableEntry.MOVE) { moveToDirectoryURI = entry.getMoveAfterFailure(); } break; default: return; } if (moveToDirectoryURI != null) { FileObject moveToDirectory = fsManager.resolveFile(moveToDirectoryURI); String prefix; if (entry.getMoveTimestampFormat() != null) { prefix = entry.getMoveTimestampFormat().format(new Date()); } else { prefix = ""; } FileObject dest = moveToDirectory.resolveFile(prefix + fileObject.getName().getBaseName()); if (log.isDebugEnabled()) { log.debug("Moving to file :" + dest.getName().getURI()); } try { fileObject.moveTo(dest); } catch (FileSystemException e) { handleException("Error moving file : " + fileObject + " to " + moveToDirectoryURI, e); } } else { try { if (log.isDebugEnabled()) { log.debug("Deleting file :" + fileObject); } fileObject.close(); if (!fileObject.delete()) { String msg = "Cannot delete file : " + fileObject; log.error(msg); throw new AxisFault(msg); } } catch (FileSystemException e) { log.error("Error deleting file : " + fileObject, e); } } } catch (FileSystemException e) { handleException( "Error resolving directory to move after processing : " + moveToDirectoryURI, e); } }
@Override public FileObject getParent() throws MalformedURLException, IOException { File parent = file.getParentFile(); if (parent == null) { return null; } return fs.resolveFile(parent.toURI().toURL()); }
public void testJarFileObject() throws IOException { FileSystemManager fs = new FileSystemManager(); URL urlFile = loader.getResource("org/apache/commons/lang/StringUtils.class"); assertTrue(urlFile.toString().startsWith("jar:")); FileObject fileObjectFile = fs.resolveFile(urlFile); assertEquals(JarFileObject.class, fileObjectFile.getClass()); // URL urlDir = new URL(urlFile.toString().substring(0, urlFile.toString().lastIndexOf('/') + 1)); assertNotNull(urlDir); FileObject fileObjectDir = fs.resolveFile(urlDir); assertEquals(JarFileObject.class, fileObjectDir.getClass()); // exists assertTrue(fileObjectFile.exists()); assertTrue(fileObjectDir.exists()); assertFalse(fileObjectDir.getChild("a_not_exists_file.txt").exists()); // getName assertEquals("lang", fileObjectDir.getName().getBaseName()); assertEquals("StringUtils.class", fileObjectFile.getName().getBaseName()); // getRelativeName assertEquals( "StringUtils.class", fileObjectDir.getName().getRelativeName(fileObjectFile.getName())); assertEquals("", fileObjectDir.getName().getRelativeName(fileObjectDir.getName())); assertEquals("", fileObjectFile.getName().getRelativeName(fileObjectFile.getName())); // getType assertSame(FileType.FOLDER, fileObjectDir.getType()); assertSame(FileType.FILE, fileObjectFile.getType()); // getChild, getParent, and equals assertEquals(fileObjectFile, fileObjectDir.getChild("StringUtils.class")); assertEquals(fileObjectDir, fileObjectFile.getParent()); assertSame(fileObjectFile, fileObjectDir.getChild("StringUtils.class")); assertSame(fileObjectDir, fileObjectFile.getParent()); assertTrue(ArrayUtils.contains(fileObjectDir.getChildren(), fileObjectFile)); // getURL assertEquals(urlDir, fileObjectDir.getURL()); assertEquals(urlFile, fileObjectFile.getURL()); }
public void testSimpleFileObjectDirEnd() throws IOException { FileSystemManager fs = new FileSystemManager(); String pathDir = "com/sinosoft/one/mvc/testcases"; URL urlDir = loader.getResource(pathDir); FileObject parent = fs.resolveFile(urlDir); assertTrue(parent.getURL().toString().endsWith("/")); assertTrue(parent.getURL().getPath().endsWith("/")); FileObject childDir = parent.getChild("scanning"); assertTrue(childDir.getURL().toString().endsWith("/")); // getChildren assertTrue(ArrayUtils.contains(parent.getChildren(), childDir)); }
public void testJarFileObjectDirEnd() throws IOException { FileSystemManager fs = new FileSystemManager(); String pathDir = "org/apache/commons/lang/"; URL urlDir = loader.getResource(pathDir); FileObject parent = fs.resolveFile(urlDir); assertTrue(parent.exists()); assertTrue(parent.getURL().toString().endsWith("/")); FileObject childDir = parent.getChild("math"); FileObject childDir2 = parent.getChild("math/"); assertTrue(childDir.exists()); assertTrue(childDir.getURL().toString().endsWith("/")); assertSame(childDir, childDir2); assertEquals("math/", parent.getName().getRelativeName(childDir.getName())); assertEquals("math/", parent.getName().getRelativeName(childDir2.getName())); }
private FileObject getChildFolder( final FileSystemManager fileSystem, final FileObject baseFolder, final String baseName) throws FileSystemException { for (int counter = 0; counter < TEMPORARY_FOLDER_ATTEMPTS; ++counter) { final FileObject temporaryFolder = fileSystem.resolveFile(baseFolder, baseName + counter); if (!temporaryFolder.exists()) { temporaryFolder.createFolder(); return temporaryFolder; } } throw new FileSystemException( "Failed to create directory within " + TEMPORARY_FOLDER_ATTEMPTS + " attempts (tried " + baseName + "0 to " + baseName + (TEMPORARY_FOLDER_ATTEMPTS - 1) + ')'); }
@Override public FileObject getChild(final String child) throws IOException { return fs.resolveFile(urlString + child); }
private FileObject getTemporaryFolder(final FileSystemManager fileSystem) throws FileSystemException { return fileSystem.resolveFile(System.getProperty("java.io.tmpdir")); }
/** * Search for files that match the given regex pattern and create a list Then process each of * these files and update the status of the scan on the poll table * * @param entry the poll table entry for the scan * @param fileURI the file or directory to be scanned */ private void scanFileOrDirectory(final PollTableEntry entry, String fileURI) { FileObject fileObject = null; if (log.isDebugEnabled()) { log.debug("Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI)); } boolean wasError = true; int retryCount = 0; int maxRetryCount = entry.getMaxRetryCount(); long reconnectionTimeout = entry.getReconnectTimeout(); while (wasError) { try { retryCount++; fileObject = fsManager.resolveFile(fileURI); if (fileObject == null) { log.error("fileObject is null"); throw new FileSystemException("fileObject is null"); } wasError = false; } catch (FileSystemException e) { if (retryCount >= maxRetryCount) { processFailure( "Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI), e, entry); return; } else { log.warn( "Failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI) + ", in attempt " + retryCount + ", " + e.getMessage() + " Retrying in " + reconnectionTimeout + " milliseconds."); } } if (wasError) { try { Thread.sleep(reconnectionTimeout); } catch (InterruptedException e2) { log.error("Thread was interrupted while waiting to reconnect.", e2); } } } try { if (fileObject.exists() && fileObject.isReadable()) { entry.setLastPollState(PollTableEntry.NONE); FileObject[] children = null; try { children = fileObject.getChildren(); } catch (FileSystemException ignore) { } // if this is a file that would translate to a single message if (children == null || children.length == 0) { boolean isFailedRecord = false; if (entry.getMoveAfterMoveFailure() != null) { isFailedRecord = isFailedRecord(fileObject, entry); } if (fileObject.getType() == FileType.FILE && !isFailedRecord) { if (!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, fileObject))) { try { processFile(entry, fileObject); entry.setLastPollState(PollTableEntry.SUCCSESSFUL); metrics.incrementMessagesReceived(); } catch (AxisFault e) { logException("Error processing File URI : " + fileObject.getName(), e); entry.setLastPollState(PollTableEntry.FAILED); metrics.incrementFaultsReceiving(); } try { moveOrDeleteAfterProcessing(entry, fileObject); } catch (AxisFault axisFault) { logException( "File object '" + fileObject.getURL().toString() + "' " + "cloud not be moved", axisFault); entry.setLastPollState(PollTableEntry.FAILED); String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat()); addFailedRecord(entry, fileObject, timeStamp); } if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, fileObject); if (log.isDebugEnabled()) { log.debug( "Removed the lock file '" + fileObject.toString() + ".lock' of the file '" + fileObject.toString()); } } } else if (log.isDebugEnabled()) { log.debug("Couldn't get the lock for processing the file : " + fileObject.getName()); } else if (isFailedRecord) { if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, fileObject); } // schedule a cleanup task if the file is there if (fsManager.resolveFile(fileObject.getURL().toString()) != null && removeTaskState == STATE_STOPPED && entry.getMoveAfterMoveFailure() != null) { workerPool.execute(new FileRemoveTask(entry, fileObject)); } if (log.isDebugEnabled()) { log.debug( "File '" + fileObject.getURL() + "' has been marked as a failed" + " record, it will not process"); } } } } else { int failCount = 0; int successCount = 0; if (log.isDebugEnabled()) { log.debug("File name pattern : " + entry.getFileNamePattern()); } for (FileObject child : children) { boolean isFailedRecord = false; if (entry.getMoveAfterMoveFailure() != null) { isFailedRecord = isFailedRecord(child, entry); } if (log.isDebugEnabled()) { log.debug("Matching file : " + child.getName().getBaseName()); } if ((entry.getFileNamePattern() != null) && (child.getName().getBaseName().matches(entry.getFileNamePattern())) && (!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, child))) && !isFailedRecord) { try { if (log.isDebugEnabled()) { log.debug("Processing file :" + child); } processFile(entry, child); successCount++; // tell moveOrDeleteAfterProcessing() file was success entry.setLastPollState(PollTableEntry.SUCCSESSFUL); metrics.incrementMessagesReceived(); } catch (Exception e) { logException("Error processing File URI : " + child.getName(), e); failCount++; // tell moveOrDeleteAfterProcessing() file failed entry.setLastPollState(PollTableEntry.FAILED); metrics.incrementFaultsReceiving(); } try { moveOrDeleteAfterProcessing(entry, child); } catch (AxisFault axisFault) { logException( "File object '" + child.getURL().toString() + "'cloud not be moved", axisFault); failCount++; entry.setLastPollState(PollTableEntry.FAILED); String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat()); addFailedRecord(entry, child, timeStamp); } // if there is a failure or not we'll try to release the lock if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, child); } } else if (!(!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, fileObject))) && log.isDebugEnabled()) { log.debug("Couldn't get the lock for processing the file : " + child.getName()); } else if (isFailedRecord) { if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, child); VFSUtils.releaseLock(fsManager, fileObject); } if (fsManager.resolveFile(child.getURL().toString()) != null && removeTaskState == STATE_STOPPED && entry.getMoveAfterMoveFailure() != null) { workerPool.execute(new FileRemoveTask(entry, child)); } if (log.isDebugEnabled()) { log.debug( "File '" + fileObject.getURL() + "' has been marked as a failed record, it will not " + "process"); } } } if (failCount == 0 && successCount > 0) { entry.setLastPollState(PollTableEntry.SUCCSESSFUL); } else if (successCount == 0 && failCount > 0) { entry.setLastPollState(PollTableEntry.FAILED); } else { entry.setLastPollState(PollTableEntry.WITH_ERRORS); } } // processing of this poll table entry is complete long now = System.currentTimeMillis(); entry.setLastPollTime(now); entry.setNextPollTime(now + entry.getPollInterval()); } else if (log.isDebugEnabled()) { log.debug( "Unable to access or read file or directory : " + VFSUtils.maskURLPassword(fileURI) + "." + " Reason: " + (fileObject.exists() ? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!") : "The file does not exists!")); } onPollCompletion(entry); } catch (FileSystemException e) { processFailure( "Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI), e, entry); } }