/**
* If not a folder just a file handle the flow
*
* @throws FileSystemException
*/
private void fileHandler() throws FileSystemException {
if (fileObject.getType() == FileType.FILE) {
if (!fileLock || (fileLock && acquireLock(fsManager, fileObject))) {
boolean runPostProcess = true;
try {
if (processFile(fileObject) == null) {
runPostProcess = false;
}
lastCycle = 1;
} catch (SynapseException e) {
lastCycle = 2;
log.error("Error processing File URI : " + fileObject.getName(), e);
}
if (runPostProcess) {
try {
moveOrDeleteAfterProcessing(fileObject);
} catch (SynapseException synapseException) {
lastCycle = 3;
log.error(
"File object '" + fileObject.getURL().toString() + "' " + "cloud not be moved",
synapseException);
VFSUtils.markFailRecord(fsManager, fileObject);
}
}
if (fileLock) {
// TODO: passing null to avoid build break. Fix properly
VFSUtils.releaseLock(fsManager, fileObject, fso);
if (log.isDebugEnabled()) {
log.debug(
"Removed the lock file '"
+ fileObject.toString()
+ ".lock' of the file '"
+ fileObject.toString());
}
}
} else {
log.error("Couldn't get the lock for processing the file : " + fileObject.getName());
}
} else {
if (log.isDebugEnabled()) {
log.debug(
"Cannot find the file or failed file record. File : "
+ VFSUtils.maskURLPassword(fileURI));
}
}
}
public FilePollingConsumer(
Properties vfsProperties,
String name,
SynapseEnvironment synapseEnvironment,
long scanInterval) {
this.vfsProperties = vfsProperties;
this.name = name;
this.synapseEnvironment = synapseEnvironment;
this.scanInterval = scanInterval;
this.lastRanTime = null;
setupParams();
try {
StandardFileSystemManager fsm = new StandardFileSystemManager();
fsm.setConfiguration(getClass().getClassLoader().getResource("providers.xml"));
fsm.init();
fsManager = fsm;
} catch (Exception e) {
log.error(e);
throw new RuntimeException(e);
}
// Setup SFTP Options
try {
fso = VFSUtils.attachFileSystemOptions(parseSchemeFileOptions(fileURI), fsManager);
} catch (Exception e) {
log.warn("Unable to set the sftp Options", e);
fso = null;
}
}
/** Check if the file/folder exists before proceeding and retrying */
private boolean initFileCheck() {
boolean wasError = true;
int retryCount = 0;
fileObject = null;
while (wasError) {
try {
retryCount++;
fileObject = fsManager.resolveFile(fileURI, fso);
if (fileObject == null) {
log.error("fileObject is null");
throw new FileSystemException("fileObject is null");
}
wasError = false;
} catch (FileSystemException e) {
if (retryCount >= maxRetryCount) {
log.error(
"Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI), e);
return false;
} else {
log.warn(
"Failed to resolve the file URI: "
+ VFSUtils.maskURLPassword(fileURI)
+ ", in attempt "
+ retryCount
+ ", "
+ e.getMessage()
+ " Retrying in "
+ reconnectionTimeout
+ " milliseconds.");
}
}
if (wasError) {
try {
Thread.sleep(reconnectionTimeout);
} catch (InterruptedException e2) {
log.error("Thread was interrupted while waiting to reconnect.", e2);
}
}
}
return true;
}
/**
* Acquire distributed lock if required first, then do the file level locking
*
* @param fsManager
* @param fileObject
* @return
*/
private boolean acquireLock(FileSystemManager fsManager, FileObject fileObject) {
String strContext = fileObject.getName().getURI();
boolean rtnValue = false;
try {
if (distributedLock) {
if (distributedLockTimeout != null) {
if (!ClusteringServiceUtil.setLock(strContext, distributedLockTimeout)) {
return false;
}
} else if (!ClusteringServiceUtil.setLock(strContext)) {
return false;
}
}
// When processing a directory list is fetched initially. Therefore
// there is still a chance of file processed by another process.
// Need to check the source file before processing.
try {
FileObject sourceFile = fsManager.resolveFile(strContext);
if (!sourceFile.exists()) {
return false;
}
} catch (FileSystemException e) {
return false;
}
VFSParamDTO vfsParamDTO = new VFSParamDTO();
vfsParamDTO.setAutoLockRelease(autoLockRelease);
vfsParamDTO.setAutoLockReleaseSameNode(autoLockReleaseSameNode);
vfsParamDTO.setAutoLockReleaseInterval(autoLockReleaseInterval);
rtnValue = VFSUtils.acquireLock(fsManager, fileObject, vfsParamDTO, fso);
} finally {
if (distributedLock) {
ClusteringServiceUtil.releaseLock(strContext);
}
}
return rtnValue;
}
/**
* Do the post processing actions
*
* @param fileObject
* @throws synapseException
*/
private void moveOrDeleteAfterProcessing(FileObject fileObject) throws SynapseException {
String moveToDirectoryURI = null;
try {
switch (lastCycle) {
case 1:
if ("MOVE"
.equals(
vfsProperties.getProperty(VFSConstants.TRANSPORT_FILE_ACTION_AFTER_PROCESS))) {
moveToDirectoryURI =
vfsProperties.getProperty(VFSConstants.TRANSPORT_FILE_MOVE_AFTER_PROCESS);
// Postfix the date given timestamp format
String strSubfoldertimestamp =
vfsProperties.getProperty(VFSConstants.SUBFOLDER_TIMESTAMP);
if (strSubfoldertimestamp != null) {
try {
SimpleDateFormat sdf = new SimpleDateFormat(strSubfoldertimestamp);
String strDateformat = sdf.format(new Date());
int iIndex = moveToDirectoryURI.indexOf("?");
if (iIndex > -1) {
moveToDirectoryURI =
moveToDirectoryURI.substring(0, iIndex)
+ strDateformat
+ moveToDirectoryURI.substring(iIndex, moveToDirectoryURI.length());
} else {
moveToDirectoryURI += strDateformat;
}
} catch (Exception e) {
log.warn("Error generating subfolder name with date", e);
}
}
}
break;
case 2:
if ("MOVE"
.equals(
vfsProperties.getProperty(VFSConstants.TRANSPORT_FILE_ACTION_AFTER_FAILURE))) {
moveToDirectoryURI =
vfsProperties.getProperty(VFSConstants.TRANSPORT_FILE_MOVE_AFTER_FAILURE);
}
break;
default:
return;
}
if (moveToDirectoryURI != null) {
FileObject moveToDirectory = fsManager.resolveFile(moveToDirectoryURI, fso);
String prefix;
if (vfsProperties.getProperty(VFSConstants.TRANSPORT_FILE_MOVE_TIMESTAMP_FORMAT) != null) {
prefix =
new SimpleDateFormat(
vfsProperties.getProperty(VFSConstants.TRANSPORT_FILE_MOVE_TIMESTAMP_FORMAT))
.format(new Date());
} else {
prefix = "";
}
// Forcefully create the folder(s) if does not exists
String strForceCreateFolder = vfsProperties.getProperty(VFSConstants.FORCE_CREATE_FOLDER);
if (strForceCreateFolder != null
&& strForceCreateFolder.toLowerCase().equals("true")
&& !moveToDirectory.exists()) {
moveToDirectory.createFolder();
}
FileObject dest = moveToDirectory.resolveFile(prefix + fileObject.getName().getBaseName());
if (log.isDebugEnabled()) {
log.debug("Moving to file :" + dest.getName().getURI());
}
try {
fileObject.moveTo(dest);
if (VFSUtils.isFailRecord(fsManager, fileObject)) {
VFSUtils.releaseFail(fsManager, fileObject);
}
} catch (FileSystemException e) {
if (!VFSUtils.isFailRecord(fsManager, fileObject)) {
VFSUtils.markFailRecord(fsManager, fileObject);
}
log.error("Error moving file : " + fileObject + " to " + moveToDirectoryURI, e);
}
} else {
try {
if (log.isDebugEnabled()) {
log.debug("Deleting file :" + fileObject);
}
fileObject.close();
if (!fileObject.delete()) {
String msg = "Cannot delete file : " + fileObject;
log.error(msg);
throw new SynapseException(msg);
}
} catch (FileSystemException e) {
log.error("Error deleting file : " + fileObject, e);
}
}
} catch (FileSystemException e) {
if (!VFSUtils.isFailRecord(fsManager, fileObject)) {
VFSUtils.markFailRecord(fsManager, fileObject);
log.error("Error resolving directory to move after processing : " + moveToDirectoryURI, e);
}
}
}
/**
* Handle directory with chile elements
*
* @param children
* @return
* @throws FileSystemException
*/
private FileObject directoryHandler(FileObject[] children) throws FileSystemException {
// Process Directory
lastCycle = 0;
int failCount = 0;
int successCount = 0;
int processCount = 0;
if (log.isDebugEnabled()) {
log.debug(
"File name pattern : "
+ vfsProperties.getProperty(VFSConstants.TRANSPORT_FILE_FILE_NAME_PATTERN));
}
// Sort the files
String strSortParam = vfsProperties.getProperty(VFSConstants.FILE_SORT_PARAM);
if (strSortParam != null && !"NONE".equals(strSortParam)) {
log.debug("Start Sorting the files.");
String strSortOrder = vfsProperties.getProperty(VFSConstants.FILE_SORT_ORDER);
boolean bSortOrderAsscending = true;
if (strSortOrder != null && strSortOrder.toLowerCase().equals("false")) {
bSortOrderAsscending = false;
}
if (log.isDebugEnabled()) {
log.debug("Sorting the files by : " + strSortOrder + ". (" + bSortOrderAsscending + ")");
}
if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_NAME) && bSortOrderAsscending) {
Arrays.sort(children, new FileNameAscComparator());
} else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_NAME) && !bSortOrderAsscending) {
Arrays.sort(children, new FileNameDesComparator());
} else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_SIZE) && bSortOrderAsscending) {
Arrays.sort(children, new FileSizeAscComparator());
} else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_SIZE) && !bSortOrderAsscending) {
Arrays.sort(children, new FileSizeDesComparator());
} else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_LASTMODIFIEDTIMESTAMP)
&& bSortOrderAsscending) {
Arrays.sort(children, new FileLastmodifiedtimestampAscComparator());
} else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_LASTMODIFIEDTIMESTAMP)
&& !bSortOrderAsscending) {
Arrays.sort(children, new FileLastmodifiedtimestampDesComparator());
}
log.debug("End Sorting the files.");
}
for (FileObject child : children) {
// skipping *.lock / *.fail file
if (child.getName().getBaseName().endsWith(".lock")
|| child.getName().getBaseName().endsWith(".fail")) {
continue;
}
boolean isFailedRecord = VFSUtils.isFailRecord(fsManager, child);
// child's file name matches the file name pattern or process all
// files now we try to get the lock and process
if ((strFilePattern == null || child.getName().getBaseName().matches(strFilePattern))
&& !isFailedRecord) {
if (log.isDebugEnabled()) {
log.debug("Matching file : " + child.getName().getBaseName());
}
if ((!fileLock || (fileLock && acquireLock(fsManager, child)))) {
// process the file
boolean runPostProcess = true;
try {
if (log.isDebugEnabled()) {
log.debug("Processing file :" + child);
}
processCount++;
if (processFile(child) == null) {
runPostProcess = false;
} else {
successCount++;
}
// tell moveOrDeleteAfterProcessing() file was success
lastCycle = 1;
} catch (Exception e) {
if (e.getCause() instanceof FileNotFoundException) {
log.warn(
"Error processing File URI : "
+ child.getName()
+ ". This can be due to file moved from another process.");
runPostProcess = false;
} else {
log.error("Error processing File URI : " + child.getName(), e);
failCount++;
// tell moveOrDeleteAfterProcessing() file failed
lastCycle = 2;
}
}
// skipping un-locking file if failed to do delete/move
// after process
boolean skipUnlock = false;
if (runPostProcess) {
try {
moveOrDeleteAfterProcessing(child);
} catch (SynapseException synapseException) {
log.error(
"File object '"
+ child.getURL().toString()
+ "'cloud not be moved, will remain in \"locked\" state",
synapseException);
skipUnlock = true;
failCount++;
lastCycle = 3;
VFSUtils.markFailRecord(fsManager, child);
}
}
// if there is a failure or not we'll try to release the
// lock
if (fileLock && !skipUnlock) {
// TODO: passing null to avoid build break. Fix properly
VFSUtils.releaseLock(fsManager, child, fso);
}
if (injectHandler == null) {
return child;
}
}
} else if (log.isDebugEnabled()
&& strFilePattern != null
&& !child.getName().getBaseName().matches(strFilePattern)
&& !isFailedRecord) {
// child's file name does not match the file name pattern
log.debug("Non-Matching file : " + child.getName().getBaseName());
} else if (isFailedRecord) {
// it is a failed record
try {
lastCycle = 1;
moveOrDeleteAfterProcessing(child);
} catch (SynapseException synapseException) {
log.error(
"File object '"
+ child.getURL().toString()
+ "'cloud not be moved, will remain in \"fail\" state",
synapseException);
}
if (fileLock) {
// TODO: passing null to avoid build break. Fix properly
VFSUtils.releaseLock(fsManager, child, fso);
VFSUtils.releaseLock(fsManager, fileObject, fso);
}
if (log.isDebugEnabled()) {
log.debug(
"File '"
+ fileObject.getURL()
+ "' has been marked as a failed record, it will not "
+ "process");
}
}
// close the file system after processing
try {
child.close();
} catch (Exception e) {
}
// Manage throttling of file processing
if (iFileProcessingInterval != null && iFileProcessingInterval > 0) {
try {
if (log.isDebugEnabled()) {
log.debug("Put the VFS processor to sleep for : " + iFileProcessingInterval);
}
Thread.sleep(iFileProcessingInterval);
} catch (InterruptedException ie) {
log.error("Unable to set the interval between file processors." + ie);
}
} else if (iFileProcessingCount != null && iFileProcessingCount <= processCount) {
break;
}
}
if (failCount == 0 && successCount > 0) {
lastCycle = 1;
} else if (successCount == 0 && failCount > 0) {
lastCycle = 4;
} else {
lastCycle = 5;
}
return null;
}
/**
* Do the file processing operation for the given set of properties. Do the checks and pass the
* control to processFile method
*/
public FileObject poll() {
if (fileURI == null || fileURI.trim().equals("")) {
log.error(
"Invalid file url. Check the inbound endpoint configuration. Endpoint Name : "
+ name
+ ", File URL : "
+ fileURI);
return null;
}
if (log.isDebugEnabled()) {
log.debug("Start : Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI));
}
if (!initFileCheck()) {
// Unable to read from the source location provided.
return null;
}
// If file/folder found proceed to the processing stage
try {
lastCycle = 0;
if (fileObject.exists() && fileObject.isReadable()) {
FileObject[] children = null;
try {
children = fileObject.getChildren();
} catch (FileNotFolderException ignored) {
if (log.isDebugEnabled()) {
log.debug("No Folder found. Only file found on : " + VFSUtils.maskURLPassword(fileURI));
}
} catch (FileSystemException ex) {
log.error(ex.getMessage(), ex);
}
// if this is a file that would translate to a single message
if (children == null || children.length == 0) {
// Fail record is a one that is processed but was not moved
// or deleted due to an error.
boolean isFailedRecord = VFSUtils.isFailRecord(fsManager, fileObject);
if (!isFailedRecord) {
fileHandler();
if (injectHandler == null) {
return fileObject;
}
} else {
try {
lastCycle = 2;
moveOrDeleteAfterProcessing(fileObject);
} catch (SynapseException synapseException) {
log.error(
"File object '"
+ fileObject.getURL().toString()
+ "' "
+ "cloud not be moved after first attempt",
synapseException);
}
if (fileLock) {
// TODO: passing null to avoid build break. Fix properly
VFSUtils.releaseLock(fsManager, fileObject, fso);
}
if (log.isDebugEnabled()) {
log.debug(
"File '"
+ fileObject.getURL()
+ "' has been marked as a failed"
+ " record, it will not process");
}
}
} else {
FileObject fileObject = directoryHandler(children);
if (fileObject != null) {
return fileObject;
}
}
} else {
log.warn(
"Unable to access or read file or directory : "
+ VFSUtils.maskURLPassword(fileURI)
+ "."
+ " Reason: "
+ (fileObject.exists()
? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!")
: "The file does not exists!"));
return null;
}
} catch (FileSystemException e) {
log.error(
"Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI), e);
return null;
} catch (Exception e) {
log.error(
"Error while processing the file/folder in URL : " + VFSUtils.maskURLPassword(fileURI),
e);
return null;
} finally {
try {
fsManager.closeFileSystem(fileObject.getParent().getFileSystem());
fileObject.close();
} catch (Exception e) {
log.error("Unable to close the file system. " + e.getMessage());
log.error(e);
}
}
if (log.isDebugEnabled()) {
log.debug("End : Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI));
}
return null;
}
