/** * If not a folder just a file handle the flow * * @throws FileSystemException */ private void fileHandler() throws FileSystemException { if (fileObject.getType() == FileType.FILE) { if (!fileLock || (fileLock && acquireLock(fsManager, fileObject))) { boolean runPostProcess = true; try { if (processFile(fileObject) == null) { runPostProcess = false; } lastCycle = 1; } catch (SynapseException e) { lastCycle = 2; log.error("Error processing File URI : " + fileObject.getName(), e); } if (runPostProcess) { try { moveOrDeleteAfterProcessing(fileObject); } catch (SynapseException synapseException) { lastCycle = 3; log.error( "File object '" + fileObject.getURL().toString() + "' " + "cloud not be moved", synapseException); VFSUtils.markFailRecord(fsManager, fileObject); } } if (fileLock) { // TODO: passing null to avoid build break. Fix properly VFSUtils.releaseLock(fsManager, fileObject, fso); if (log.isDebugEnabled()) { log.debug( "Removed the lock file '" + fileObject.toString() + ".lock' of the file '" + fileObject.toString()); } } } else { log.error("Couldn't get the lock for processing the file : " + fileObject.getName()); } } else { if (log.isDebugEnabled()) { log.debug( "Cannot find the file or failed file record. File : " + VFSUtils.maskURLPassword(fileURI)); } } }
/** * Do the post processing actions * * @param fileObject * @throws synapseException */ private void moveOrDeleteAfterProcessing(FileObject fileObject) throws SynapseException { String moveToDirectoryURI = null; try { switch (lastCycle) { case 1: if ("MOVE" .equals( vfsProperties.getProperty(VFSConstants.TRANSPORT_FILE_ACTION_AFTER_PROCESS))) { moveToDirectoryURI = vfsProperties.getProperty(VFSConstants.TRANSPORT_FILE_MOVE_AFTER_PROCESS); // Postfix the date given timestamp format String strSubfoldertimestamp = vfsProperties.getProperty(VFSConstants.SUBFOLDER_TIMESTAMP); if (strSubfoldertimestamp != null) { try { SimpleDateFormat sdf = new SimpleDateFormat(strSubfoldertimestamp); String strDateformat = sdf.format(new Date()); int iIndex = moveToDirectoryURI.indexOf("?"); if (iIndex > -1) { moveToDirectoryURI = moveToDirectoryURI.substring(0, iIndex) + strDateformat + moveToDirectoryURI.substring(iIndex, moveToDirectoryURI.length()); } else { moveToDirectoryURI += strDateformat; } } catch (Exception e) { log.warn("Error generating subfolder name with date", e); } } } break; case 2: if ("MOVE" .equals( vfsProperties.getProperty(VFSConstants.TRANSPORT_FILE_ACTION_AFTER_FAILURE))) { moveToDirectoryURI = vfsProperties.getProperty(VFSConstants.TRANSPORT_FILE_MOVE_AFTER_FAILURE); } break; default: return; } if (moveToDirectoryURI != null) { FileObject moveToDirectory = fsManager.resolveFile(moveToDirectoryURI, fso); String prefix; if (vfsProperties.getProperty(VFSConstants.TRANSPORT_FILE_MOVE_TIMESTAMP_FORMAT) != null) { prefix = new SimpleDateFormat( vfsProperties.getProperty(VFSConstants.TRANSPORT_FILE_MOVE_TIMESTAMP_FORMAT)) .format(new Date()); } else { prefix = ""; } // Forcefully create the folder(s) if does not exists String strForceCreateFolder = vfsProperties.getProperty(VFSConstants.FORCE_CREATE_FOLDER); if (strForceCreateFolder != null && strForceCreateFolder.toLowerCase().equals("true") && !moveToDirectory.exists()) { moveToDirectory.createFolder(); } FileObject dest = moveToDirectory.resolveFile(prefix + fileObject.getName().getBaseName()); if (log.isDebugEnabled()) { log.debug("Moving to file :" + dest.getName().getURI()); } try { fileObject.moveTo(dest); if (VFSUtils.isFailRecord(fsManager, fileObject)) { VFSUtils.releaseFail(fsManager, fileObject); } } catch (FileSystemException e) { if (!VFSUtils.isFailRecord(fsManager, fileObject)) { VFSUtils.markFailRecord(fsManager, fileObject); } log.error("Error moving file : " + fileObject + " to " + moveToDirectoryURI, e); } } else { try { if (log.isDebugEnabled()) { log.debug("Deleting file :" + fileObject); } fileObject.close(); if (!fileObject.delete()) { String msg = "Cannot delete file : " + fileObject; log.error(msg); throw new SynapseException(msg); } } catch (FileSystemException e) { log.error("Error deleting file : " + fileObject, e); } } } catch (FileSystemException e) { if (!VFSUtils.isFailRecord(fsManager, fileObject)) { VFSUtils.markFailRecord(fsManager, fileObject); log.error("Error resolving directory to move after processing : " + moveToDirectoryURI, e); } } }
/** * Handle directory with chile elements * * @param children * @return * @throws FileSystemException */ private FileObject directoryHandler(FileObject[] children) throws FileSystemException { // Process Directory lastCycle = 0; int failCount = 0; int successCount = 0; int processCount = 0; if (log.isDebugEnabled()) { log.debug( "File name pattern : " + vfsProperties.getProperty(VFSConstants.TRANSPORT_FILE_FILE_NAME_PATTERN)); } // Sort the files String strSortParam = vfsProperties.getProperty(VFSConstants.FILE_SORT_PARAM); if (strSortParam != null && !"NONE".equals(strSortParam)) { log.debug("Start Sorting the files."); String strSortOrder = vfsProperties.getProperty(VFSConstants.FILE_SORT_ORDER); boolean bSortOrderAsscending = true; if (strSortOrder != null && strSortOrder.toLowerCase().equals("false")) { bSortOrderAsscending = false; } if (log.isDebugEnabled()) { log.debug("Sorting the files by : " + strSortOrder + ". (" + bSortOrderAsscending + ")"); } if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_NAME) && bSortOrderAsscending) { Arrays.sort(children, new FileNameAscComparator()); } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_NAME) && !bSortOrderAsscending) { Arrays.sort(children, new FileNameDesComparator()); } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_SIZE) && bSortOrderAsscending) { Arrays.sort(children, new FileSizeAscComparator()); } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_SIZE) && !bSortOrderAsscending) { Arrays.sort(children, new FileSizeDesComparator()); } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_LASTMODIFIEDTIMESTAMP) && bSortOrderAsscending) { Arrays.sort(children, new FileLastmodifiedtimestampAscComparator()); } else if (strSortParam.equals(VFSConstants.FILE_SORT_VALUE_LASTMODIFIEDTIMESTAMP) && !bSortOrderAsscending) { Arrays.sort(children, new FileLastmodifiedtimestampDesComparator()); } log.debug("End Sorting the files."); } for (FileObject child : children) { // skipping *.lock / *.fail file if (child.getName().getBaseName().endsWith(".lock") || child.getName().getBaseName().endsWith(".fail")) { continue; } boolean isFailedRecord = VFSUtils.isFailRecord(fsManager, child); // child's file name matches the file name pattern or process all // files now we try to get the lock and process if ((strFilePattern == null || child.getName().getBaseName().matches(strFilePattern)) && !isFailedRecord) { if (log.isDebugEnabled()) { log.debug("Matching file : " + child.getName().getBaseName()); } if ((!fileLock || (fileLock && acquireLock(fsManager, child)))) { // process the file boolean runPostProcess = true; try { if (log.isDebugEnabled()) { log.debug("Processing file :" + child); } processCount++; if (processFile(child) == null) { runPostProcess = false; } else { successCount++; } // tell moveOrDeleteAfterProcessing() file was success lastCycle = 1; } catch (Exception e) { if (e.getCause() instanceof FileNotFoundException) { log.warn( "Error processing File URI : " + child.getName() + ". This can be due to file moved from another process."); runPostProcess = false; } else { log.error("Error processing File URI : " + child.getName(), e); failCount++; // tell moveOrDeleteAfterProcessing() file failed lastCycle = 2; } } // skipping un-locking file if failed to do delete/move // after process boolean skipUnlock = false; if (runPostProcess) { try { moveOrDeleteAfterProcessing(child); } catch (SynapseException synapseException) { log.error( "File object '" + child.getURL().toString() + "'cloud not be moved, will remain in \"locked\" state", synapseException); skipUnlock = true; failCount++; lastCycle = 3; VFSUtils.markFailRecord(fsManager, child); } } // if there is a failure or not we'll try to release the // lock if (fileLock && !skipUnlock) { // TODO: passing null to avoid build break. Fix properly VFSUtils.releaseLock(fsManager, child, fso); } if (injectHandler == null) { return child; } } } else if (log.isDebugEnabled() && strFilePattern != null && !child.getName().getBaseName().matches(strFilePattern) && !isFailedRecord) { // child's file name does not match the file name pattern log.debug("Non-Matching file : " + child.getName().getBaseName()); } else if (isFailedRecord) { // it is a failed record try { lastCycle = 1; moveOrDeleteAfterProcessing(child); } catch (SynapseException synapseException) { log.error( "File object '" + child.getURL().toString() + "'cloud not be moved, will remain in \"fail\" state", synapseException); } if (fileLock) { // TODO: passing null to avoid build break. Fix properly VFSUtils.releaseLock(fsManager, child, fso); VFSUtils.releaseLock(fsManager, fileObject, fso); } if (log.isDebugEnabled()) { log.debug( "File '" + fileObject.getURL() + "' has been marked as a failed record, it will not " + "process"); } } // close the file system after processing try { child.close(); } catch (Exception e) { } // Manage throttling of file processing if (iFileProcessingInterval != null && iFileProcessingInterval > 0) { try { if (log.isDebugEnabled()) { log.debug("Put the VFS processor to sleep for : " + iFileProcessingInterval); } Thread.sleep(iFileProcessingInterval); } catch (InterruptedException ie) { log.error("Unable to set the interval between file processors." + ie); } } else if (iFileProcessingCount != null && iFileProcessingCount <= processCount) { break; } } if (failCount == 0 && successCount > 0) { lastCycle = 1; } else if (successCount == 0 && failCount > 0) { lastCycle = 4; } else { lastCycle = 5; } return null; }