public void testSimpleFileObject() throws IOException {
FileSystemManager fs = new FileSystemManager();
//
String pathFile = "com/sinosoft/one/mvc/scanning/vfs/SimpleFileObject.class";
URL urlFile = loader.getResource(pathFile);
assertNotNull(urlFile);
FileObject fileObjectFile = fs.resolveFile(urlFile);
assertEquals(SimpleFileObject.class, fileObjectFile.getClass());
//
String pathDir = new File(urlFile.getPath()).getParent().replace('\\', '/');
pathDir = StringUtils.removeEnd(pathDir, "/");
URL urlDir = ResourceUtils.getURL(pathDir);
assertNotNull(urlDir);
File fileDir = new File(urlDir.getFile());
assertTrue(fileDir.exists());
FileObject fileObjectDir = fs.resolveFile(urlDir);
assertEquals(SimpleFileObject.class, fileObjectDir.getClass());
File dirFile = ResourceUtils.getFile(urlDir);
assertTrue(urlDir.toString().endsWith("/"));
assertTrue(urlDir.getPath().endsWith("/"));
assertFalse(dirFile.getPath().endsWith("/"));
assertTrue(fileObjectDir.toString().endsWith("/"));
// exists
assertTrue(fileObjectFile.exists());
assertTrue(fileObjectDir.exists());
assertFalse(fileObjectDir.getChild("a_not_exists_file.txt").exists());
// getName
assertEquals("vfs", fileObjectDir.getName().getBaseName());
assertEquals("SimpleFileObject.class", fileObjectFile.getName().getBaseName());
// getRelativeName
assertEquals(
"SimpleFileObject.class",
fileObjectDir.getName().getRelativeName(fileObjectFile.getName()));
assertEquals("", fileObjectDir.getName().getRelativeName(fileObjectDir.getName()));
assertEquals("", fileObjectFile.getName().getRelativeName(fileObjectFile.getName()));
// getType
assertSame(FileType.FOLDER, fileObjectDir.getType());
assertSame(FileType.FILE, fileObjectFile.getType());
// getChild, getParent, and equals, getChildren
assertEquals(fileObjectFile, fileObjectDir.getChild("SimpleFileObject.class"));
assertEquals(fileObjectDir, fileObjectFile.getParent());
assertSame(fileObjectFile, fileObjectDir.getChild("SimpleFileObject.class"));
assertSame(fileObjectDir, fileObjectFile.getParent());
assertTrue(ArrayUtils.contains(fileObjectDir.getChildren(), fileObjectFile));
// getURL
assertEquals(urlDir, fileObjectDir.getURL());
assertEquals(urlFile, fileObjectFile.getURL());
}
public void testSimpleFileObjectDirEnd() throws IOException {
FileSystemManager fs = new FileSystemManager();
String pathDir = "com/sinosoft/one/mvc/testcases";
URL urlDir = loader.getResource(pathDir);
FileObject parent = fs.resolveFile(urlDir);
assertTrue(parent.getURL().toString().endsWith("/"));
assertTrue(parent.getURL().getPath().endsWith("/"));
FileObject childDir = parent.getChild("scanning");
assertTrue(childDir.getURL().toString().endsWith("/"));
// getChildren
assertTrue(ArrayUtils.contains(parent.getChildren(), childDir));
}
public void testJarFileObjectDirEnd() throws IOException {
FileSystemManager fs = new FileSystemManager();
String pathDir = "org/apache/commons/lang/";
URL urlDir = loader.getResource(pathDir);
FileObject parent = fs.resolveFile(urlDir);
assertTrue(parent.exists());
assertTrue(parent.getURL().toString().endsWith("/"));
FileObject childDir = parent.getChild("math");
FileObject childDir2 = parent.getChild("math/");
assertTrue(childDir.exists());
assertTrue(childDir.getURL().toString().endsWith("/"));
assertSame(childDir, childDir2);
assertEquals("math/", parent.getName().getRelativeName(childDir.getName()));
assertEquals("math/", parent.getName().getRelativeName(childDir2.getName()));
}
public void testJarFileObject() throws IOException {
FileSystemManager fs = new FileSystemManager();
URL urlFile = loader.getResource("org/apache/commons/lang/StringUtils.class");
assertTrue(urlFile.toString().startsWith("jar:"));
FileObject fileObjectFile = fs.resolveFile(urlFile);
assertEquals(JarFileObject.class, fileObjectFile.getClass());
//
URL urlDir = new URL(urlFile.toString().substring(0, urlFile.toString().lastIndexOf('/') + 1));
assertNotNull(urlDir);
FileObject fileObjectDir = fs.resolveFile(urlDir);
assertEquals(JarFileObject.class, fileObjectDir.getClass());
// exists
assertTrue(fileObjectFile.exists());
assertTrue(fileObjectDir.exists());
assertFalse(fileObjectDir.getChild("a_not_exists_file.txt").exists());
// getName
assertEquals("lang", fileObjectDir.getName().getBaseName());
assertEquals("StringUtils.class", fileObjectFile.getName().getBaseName());
// getRelativeName
assertEquals(
"StringUtils.class", fileObjectDir.getName().getRelativeName(fileObjectFile.getName()));
assertEquals("", fileObjectDir.getName().getRelativeName(fileObjectDir.getName()));
assertEquals("", fileObjectFile.getName().getRelativeName(fileObjectFile.getName()));
// getType
assertSame(FileType.FOLDER, fileObjectDir.getType());
assertSame(FileType.FILE, fileObjectFile.getType());
// getChild, getParent, and equals
assertEquals(fileObjectFile, fileObjectDir.getChild("StringUtils.class"));
assertEquals(fileObjectDir, fileObjectFile.getParent());
assertSame(fileObjectFile, fileObjectDir.getChild("StringUtils.class"));
assertSame(fileObjectDir, fileObjectFile.getParent());
assertTrue(ArrayUtils.contains(fileObjectDir.getChildren(), fileObjectFile));
// getURL
assertEquals(urlDir, fileObjectDir.getURL());
assertEquals(urlFile, fileObjectFile.getURL());
}
public void run() {
if (log.isDebugEnabled()) {
log.debug(
"New file remove task is starting..thread id : " + Thread.currentThread().getName());
}
// there has been a failure, basically it should be a move operation
// failure. we'll re-try to move in a loop suspending with the
// configured amount of time
// we'll check if the lock is still there and if the lock is there
// then we assume that the respective file object is also there
// try to remove the folder waiting on a busy loop, if the remove operation success
// we just exit from the busy loop and mark end of the file move task.
boolean isDeletionSucceed = false;
int nextRetryDuration = pollTableEntry.getNextRetryDuration();
int count = 0;
while (!isDeletionSucceed) {
try {
reTryFailedMove(pollTableEntry, failedFileObject);
isDeletionSucceed = true;
removeTaskState = STATE_STOPPED;
} catch (AxisFault axisFault) {
removeTaskState = STATE_RUNNING;
try {
log.error(
"Remove attempt '"
+ (count++)
+ "' failed for the file '"
+ failedFileObject.getURL().toString()
+ "', next re-try will be "
+ "after '"
+ nextRetryDuration
+ "' milliseconds");
} catch (FileSystemException e) {
log.error(
"Error while retrying the file url of the file object '" + failedFileObject + "'");
}
try {
Thread.sleep(nextRetryDuration);
} catch (InterruptedException ignore) {
// ignore
}
}
}
}
/**
* Search for files that match the given regex pattern and create a list Then process each of
* these files and update the status of the scan on the poll table
*
* @param entry the poll table entry for the scan
* @param fileURI the file or directory to be scanned
*/
private void scanFileOrDirectory(final PollTableEntry entry, String fileURI) {
FileObject fileObject = null;
if (log.isDebugEnabled()) {
log.debug("Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI));
}
boolean wasError = true;
int retryCount = 0;
int maxRetryCount = entry.getMaxRetryCount();
long reconnectionTimeout = entry.getReconnectTimeout();
while (wasError) {
try {
retryCount++;
fileObject = fsManager.resolveFile(fileURI);
if (fileObject == null) {
log.error("fileObject is null");
throw new FileSystemException("fileObject is null");
}
wasError = false;
} catch (FileSystemException e) {
if (retryCount >= maxRetryCount) {
processFailure(
"Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI),
e,
entry);
return;
} else {
log.warn(
"Failed to resolve the file URI: "
+ VFSUtils.maskURLPassword(fileURI)
+ ", in attempt "
+ retryCount
+ ", "
+ e.getMessage()
+ " Retrying in "
+ reconnectionTimeout
+ " milliseconds.");
}
}
if (wasError) {
try {
Thread.sleep(reconnectionTimeout);
} catch (InterruptedException e2) {
log.error("Thread was interrupted while waiting to reconnect.", e2);
}
}
}
try {
if (fileObject.exists() && fileObject.isReadable()) {
entry.setLastPollState(PollTableEntry.NONE);
FileObject[] children = null;
try {
children = fileObject.getChildren();
} catch (FileSystemException ignore) {
}
// if this is a file that would translate to a single message
if (children == null || children.length == 0) {
boolean isFailedRecord = false;
if (entry.getMoveAfterMoveFailure() != null) {
isFailedRecord = isFailedRecord(fileObject, entry);
}
if (fileObject.getType() == FileType.FILE && !isFailedRecord) {
if (!entry.isFileLockingEnabled()
|| (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, fileObject))) {
try {
processFile(entry, fileObject);
entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
metrics.incrementMessagesReceived();
} catch (AxisFault e) {
logException("Error processing File URI : " + fileObject.getName(), e);
entry.setLastPollState(PollTableEntry.FAILED);
metrics.incrementFaultsReceiving();
}
try {
moveOrDeleteAfterProcessing(entry, fileObject);
} catch (AxisFault axisFault) {
logException(
"File object '" + fileObject.getURL().toString() + "' " + "cloud not be moved",
axisFault);
entry.setLastPollState(PollTableEntry.FAILED);
String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat());
addFailedRecord(entry, fileObject, timeStamp);
}
if (entry.isFileLockingEnabled()) {
VFSUtils.releaseLock(fsManager, fileObject);
if (log.isDebugEnabled()) {
log.debug(
"Removed the lock file '"
+ fileObject.toString()
+ ".lock' of the file '"
+ fileObject.toString());
}
}
} else if (log.isDebugEnabled()) {
log.debug("Couldn't get the lock for processing the file : " + fileObject.getName());
} else if (isFailedRecord) {
if (entry.isFileLockingEnabled()) {
VFSUtils.releaseLock(fsManager, fileObject);
}
// schedule a cleanup task if the file is there
if (fsManager.resolveFile(fileObject.getURL().toString()) != null
&& removeTaskState == STATE_STOPPED
&& entry.getMoveAfterMoveFailure() != null) {
workerPool.execute(new FileRemoveTask(entry, fileObject));
}
if (log.isDebugEnabled()) {
log.debug(
"File '"
+ fileObject.getURL()
+ "' has been marked as a failed"
+ " record, it will not process");
}
}
}
} else {
int failCount = 0;
int successCount = 0;
if (log.isDebugEnabled()) {
log.debug("File name pattern : " + entry.getFileNamePattern());
}
for (FileObject child : children) {
boolean isFailedRecord = false;
if (entry.getMoveAfterMoveFailure() != null) {
isFailedRecord = isFailedRecord(child, entry);
}
if (log.isDebugEnabled()) {
log.debug("Matching file : " + child.getName().getBaseName());
}
if ((entry.getFileNamePattern() != null)
&& (child.getName().getBaseName().matches(entry.getFileNamePattern()))
&& (!entry.isFileLockingEnabled()
|| (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, child)))
&& !isFailedRecord) {
try {
if (log.isDebugEnabled()) {
log.debug("Processing file :" + child);
}
processFile(entry, child);
successCount++;
// tell moveOrDeleteAfterProcessing() file was success
entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
metrics.incrementMessagesReceived();
} catch (Exception e) {
logException("Error processing File URI : " + child.getName(), e);
failCount++;
// tell moveOrDeleteAfterProcessing() file failed
entry.setLastPollState(PollTableEntry.FAILED);
metrics.incrementFaultsReceiving();
}
try {
moveOrDeleteAfterProcessing(entry, child);
} catch (AxisFault axisFault) {
logException(
"File object '" + child.getURL().toString() + "'cloud not be moved", axisFault);
failCount++;
entry.setLastPollState(PollTableEntry.FAILED);
String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat());
addFailedRecord(entry, child, timeStamp);
}
// if there is a failure or not we'll try to release the lock
if (entry.isFileLockingEnabled()) {
VFSUtils.releaseLock(fsManager, child);
}
} else if (!(!entry.isFileLockingEnabled()
|| (entry.isFileLockingEnabled()
&& VFSUtils.acquireLock(fsManager, fileObject)))
&& log.isDebugEnabled()) {
log.debug("Couldn't get the lock for processing the file : " + child.getName());
} else if (isFailedRecord) {
if (entry.isFileLockingEnabled()) {
VFSUtils.releaseLock(fsManager, child);
VFSUtils.releaseLock(fsManager, fileObject);
}
if (fsManager.resolveFile(child.getURL().toString()) != null
&& removeTaskState == STATE_STOPPED
&& entry.getMoveAfterMoveFailure() != null) {
workerPool.execute(new FileRemoveTask(entry, child));
}
if (log.isDebugEnabled()) {
log.debug(
"File '"
+ fileObject.getURL()
+ "' has been marked as a failed record, it will not "
+ "process");
}
}
}
if (failCount == 0 && successCount > 0) {
entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
} else if (successCount == 0 && failCount > 0) {
entry.setLastPollState(PollTableEntry.FAILED);
} else {
entry.setLastPollState(PollTableEntry.WITH_ERRORS);
}
}
// processing of this poll table entry is complete
long now = System.currentTimeMillis();
entry.setLastPollTime(now);
entry.setNextPollTime(now + entry.getPollInterval());
} else if (log.isDebugEnabled()) {
log.debug(
"Unable to access or read file or directory : "
+ VFSUtils.maskURLPassword(fileURI)
+ "."
+ " Reason: "
+ (fileObject.exists()
? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!")
: "The file does not exists!"));
}
onPollCompletion(entry);
} catch (FileSystemException e) {
processFailure(
"Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI),
e,
entry);
}
}