private synchronized void reTryFailedMove(PollTableEntry entry, FileObject fileObject) throws AxisFault { try { String moveToDirectoryURI = entry.getMoveAfterMoveFailure(); FileObject moveToDirectory = fsManager.resolveFile(moveToDirectoryURI); if (!moveToDirectory.exists()) { moveToDirectory.createFolder(); } String prefix; if (entry.getMoveTimestampFormat() != null) { prefix = entry.getMoveTimestampFormat().format(new Date()); } else { prefix = ""; } FileObject dest = moveToDirectory.resolveFile(prefix + fileObject.getName().getBaseName()); if (log.isDebugEnabled()) { log.debug("The failed file is moving to :" + dest.getName().getURI()); } try { fileObject.moveTo( dest); // FIXME - when an exception occurs here it causes the in folder to vanish } catch (FileSystemException e) { handleException( "Error moving the failed file : " + fileObject + " to " + moveToDirectoryURI, e); } } catch (FileSystemException e) { handleException("Cloud not move the failed file object '" + fileObject + "'", e); } catch (IOException e) { handleException("Cloud not create the folder", e); } }
/** * Take specified action to either move or delete the processed file, depending on the outcome * * @param entry the PollTableEntry for the file that has been processed * @param fileObject the FileObject representing the file to be moved or deleted */ private void moveOrDeleteAfterProcessing(final PollTableEntry entry, FileObject fileObject) throws AxisFault { String moveToDirectoryURI = null; try { switch (entry.getLastPollState()) { case PollTableEntry.SUCCSESSFUL: if (entry.getActionAfterProcess() == PollTableEntry.MOVE) { moveToDirectoryURI = entry.getMoveAfterProcess(); } break; case PollTableEntry.FAILED: if (entry.getActionAfterFailure() == PollTableEntry.MOVE) { moveToDirectoryURI = entry.getMoveAfterFailure(); } break; default: return; } if (moveToDirectoryURI != null) { FileObject moveToDirectory = fsManager.resolveFile(moveToDirectoryURI); String prefix; if (entry.getMoveTimestampFormat() != null) { prefix = entry.getMoveTimestampFormat().format(new Date()); } else { prefix = ""; } FileObject dest = moveToDirectory.resolveFile(prefix + fileObject.getName().getBaseName()); if (log.isDebugEnabled()) { log.debug("Moving to file :" + dest.getName().getURI()); } try { fileObject.moveTo(dest); } catch (FileSystemException e) { handleException("Error moving file : " + fileObject + " to " + moveToDirectoryURI, e); } } else { try { if (log.isDebugEnabled()) { log.debug("Deleting file :" + fileObject); } fileObject.close(); if (!fileObject.delete()) { String msg = "Cannot delete file : " + fileObject; log.error(msg); throw new AxisFault(msg); } } catch (FileSystemException e) { log.error("Error deleting file : " + fileObject, e); } } } catch (FileSystemException e) { handleException( "Error resolving directory to move after processing : " + moveToDirectoryURI, e); } }
private synchronized void addFailedRecord( PollTableEntry pollTableEntry, FileObject failedObject, String timeString) { try { String record = failedObject.getName().getBaseName() + VFSConstants.FAILED_RECORD_DELIMITER + timeString; String recordFile = pollTableEntry.getFailedRecordFileDestination() + pollTableEntry.getFailedRecordFileName(); File failedRecordFile = new File(recordFile); if (!failedRecordFile.exists()) { FileUtils.writeStringToFile(failedRecordFile, record); if (log.isDebugEnabled()) { log.debug("Added fail record '" + record + "' into the record file '" + recordFile + "'"); } } else { List<String> content = FileUtils.readLines(failedRecordFile); if (!content.contains(record)) { content.add(record); } FileUtils.writeLines(failedRecordFile, content); } } catch (IOException e) { log.fatal("Failure while writing the failed records!", e); } }
private boolean isFailedRecord(FileObject fileObject, PollTableEntry entry) { String failedFile = entry.getFailedRecordFileDestination() + entry.getFailedRecordFileName(); File file = new File(failedFile); if (file.exists()) { try { List list = FileUtils.readLines(file); for (Object aList : list) { String str = (String) aList; StringTokenizer st = new StringTokenizer(str, VFSConstants.FAILED_RECORD_DELIMITER); String fileName = st.nextToken(); if (fileName != null && fileName.equals(fileObject.getName().getBaseName())) { return true; } } } catch (IOException e) { log.fatal("Error while reading the file '" + failedFile + "'", e); } } return false; }
/** * Process a single file through Axis2 * * @param entry the PollTableEntry for the file (or its parent directory or archive) * @param file the file that contains the actual message pumped into Axis2 * @throws AxisFault on error */ private void processFile(PollTableEntry entry, FileObject file) throws AxisFault { try { FileContent content = file.getContent(); String fileName = file.getName().getBaseName(); String filePath = file.getName().getPath(); metrics.incrementBytesReceived(content.getSize()); Map<String, Object> transportHeaders = new HashMap<String, Object>(); transportHeaders.put(VFSConstants.FILE_PATH, filePath); transportHeaders.put(VFSConstants.FILE_NAME, fileName); try { transportHeaders.put(VFSConstants.FILE_LENGTH, content.getSize()); transportHeaders.put(VFSConstants.LAST_MODIFIED, content.getLastModifiedTime()); } catch (FileSystemException ignore) { } MessageContext msgContext = entry.createMessageContext(); String contentType = entry.getContentType(); if (BaseUtils.isBlank(contentType)) { if (file.getName().getExtension().toLowerCase().endsWith(".xml")) { contentType = "text/xml"; } else if (file.getName().getExtension().toLowerCase().endsWith(".txt")) { contentType = "text/plain"; } } else { // Extract the charset encoding from the configured content type and // set the CHARACTER_SET_ENCODING property as e.g. SOAPBuilder relies on this. String charSetEnc = null; try { if (contentType != null) { charSetEnc = new ContentType(contentType).getParameter("charset"); } } catch (ParseException ex) { // ignore } msgContext.setProperty(Constants.Configuration.CHARACTER_SET_ENCODING, charSetEnc); } // if the content type was not found, but the service defined it.. use it if (contentType == null) { if (entry.getContentType() != null) { contentType = entry.getContentType(); } else if (VFSUtils.getProperty(content, BaseConstants.CONTENT_TYPE) != null) { contentType = VFSUtils.getProperty(content, BaseConstants.CONTENT_TYPE); } } // does the service specify a default reply file URI ? String replyFileURI = entry.getReplyFileURI(); if (replyFileURI != null) { msgContext.setProperty( Constants.OUT_TRANSPORT_INFO, new VFSOutTransportInfo(replyFileURI, entry.isFileLockingEnabled())); } // Determine the message builder to use Builder builder; if (contentType == null) { log.debug("No content type specified. Using SOAP builder."); builder = new SOAPBuilder(); } else { int index = contentType.indexOf(';'); String type = index > 0 ? contentType.substring(0, index) : contentType; builder = BuilderUtil.getBuilderFromSelector(type, msgContext); if (builder == null) { if (log.isDebugEnabled()) { log.debug("No message builder found for type '" + type + "'. Falling back to SOAP."); } builder = new SOAPBuilder(); } } // set the message payload to the message context InputStream in; ManagedDataSource dataSource; if (builder instanceof DataSourceMessageBuilder && entry.isStreaming()) { in = null; dataSource = ManagedDataSourceFactory.create(new FileObjectDataSource(file, contentType)); } else { in = new AutoCloseInputStream(content.getInputStream()); dataSource = null; } try { OMElement documentElement; if (in != null) { documentElement = builder.processDocument(in, contentType, msgContext); } else { documentElement = ((DataSourceMessageBuilder) builder) .processDocument(dataSource, contentType, msgContext); } msgContext.setEnvelope(TransportUtils.createSOAPEnvelope(documentElement)); handleIncomingMessage( msgContext, transportHeaders, null, // * SOAP Action - not applicable *// contentType); } finally { if (dataSource != null) { dataSource.destroy(); } } if (log.isDebugEnabled()) { log.debug("Processed file : " + file + " of Content-type : " + contentType); } } catch (FileSystemException e) { handleException("Error reading file content or attributes : " + file, e); } finally { try { file.close(); } catch (FileSystemException warn) { // log.warn("Cannot close file after processing : " + file.getName().getPath(), warn); // ignore the warning, since we handed over the stream close job to AutocloseInputstream.. } } }
/** * Search for files that match the given regex pattern and create a list Then process each of * these files and update the status of the scan on the poll table * * @param entry the poll table entry for the scan * @param fileURI the file or directory to be scanned */ private void scanFileOrDirectory(final PollTableEntry entry, String fileURI) { FileObject fileObject = null; if (log.isDebugEnabled()) { log.debug("Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI)); } boolean wasError = true; int retryCount = 0; int maxRetryCount = entry.getMaxRetryCount(); long reconnectionTimeout = entry.getReconnectTimeout(); while (wasError) { try { retryCount++; fileObject = fsManager.resolveFile(fileURI); if (fileObject == null) { log.error("fileObject is null"); throw new FileSystemException("fileObject is null"); } wasError = false; } catch (FileSystemException e) { if (retryCount >= maxRetryCount) { processFailure( "Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI), e, entry); return; } else { log.warn( "Failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI) + ", in attempt " + retryCount + ", " + e.getMessage() + " Retrying in " + reconnectionTimeout + " milliseconds."); } } if (wasError) { try { Thread.sleep(reconnectionTimeout); } catch (InterruptedException e2) { log.error("Thread was interrupted while waiting to reconnect.", e2); } } } try { if (fileObject.exists() && fileObject.isReadable()) { entry.setLastPollState(PollTableEntry.NONE); FileObject[] children = null; try { children = fileObject.getChildren(); } catch (FileSystemException ignore) { } // if this is a file that would translate to a single message if (children == null || children.length == 0) { boolean isFailedRecord = false; if (entry.getMoveAfterMoveFailure() != null) { isFailedRecord = isFailedRecord(fileObject, entry); } if (fileObject.getType() == FileType.FILE && !isFailedRecord) { if (!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, fileObject))) { try { processFile(entry, fileObject); entry.setLastPollState(PollTableEntry.SUCCSESSFUL); metrics.incrementMessagesReceived(); } catch (AxisFault e) { logException("Error processing File URI : " + fileObject.getName(), e); entry.setLastPollState(PollTableEntry.FAILED); metrics.incrementFaultsReceiving(); } try { moveOrDeleteAfterProcessing(entry, fileObject); } catch (AxisFault axisFault) { logException( "File object '" + fileObject.getURL().toString() + "' " + "cloud not be moved", axisFault); entry.setLastPollState(PollTableEntry.FAILED); String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat()); addFailedRecord(entry, fileObject, timeStamp); } if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, fileObject); if (log.isDebugEnabled()) { log.debug( "Removed the lock file '" + fileObject.toString() + ".lock' of the file '" + fileObject.toString()); } } } else if (log.isDebugEnabled()) { log.debug("Couldn't get the lock for processing the file : " + fileObject.getName()); } else if (isFailedRecord) { if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, fileObject); } // schedule a cleanup task if the file is there if (fsManager.resolveFile(fileObject.getURL().toString()) != null && removeTaskState == STATE_STOPPED && entry.getMoveAfterMoveFailure() != null) { workerPool.execute(new FileRemoveTask(entry, fileObject)); } if (log.isDebugEnabled()) { log.debug( "File '" + fileObject.getURL() + "' has been marked as a failed" + " record, it will not process"); } } } } else { int failCount = 0; int successCount = 0; if (log.isDebugEnabled()) { log.debug("File name pattern : " + entry.getFileNamePattern()); } for (FileObject child : children) { boolean isFailedRecord = false; if (entry.getMoveAfterMoveFailure() != null) { isFailedRecord = isFailedRecord(child, entry); } if (log.isDebugEnabled()) { log.debug("Matching file : " + child.getName().getBaseName()); } if ((entry.getFileNamePattern() != null) && (child.getName().getBaseName().matches(entry.getFileNamePattern())) && (!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, child))) && !isFailedRecord) { try { if (log.isDebugEnabled()) { log.debug("Processing file :" + child); } processFile(entry, child); successCount++; // tell moveOrDeleteAfterProcessing() file was success entry.setLastPollState(PollTableEntry.SUCCSESSFUL); metrics.incrementMessagesReceived(); } catch (Exception e) { logException("Error processing File URI : " + child.getName(), e); failCount++; // tell moveOrDeleteAfterProcessing() file failed entry.setLastPollState(PollTableEntry.FAILED); metrics.incrementFaultsReceiving(); } try { moveOrDeleteAfterProcessing(entry, child); } catch (AxisFault axisFault) { logException( "File object '" + child.getURL().toString() + "'cloud not be moved", axisFault); failCount++; entry.setLastPollState(PollTableEntry.FAILED); String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat()); addFailedRecord(entry, child, timeStamp); } // if there is a failure or not we'll try to release the lock if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, child); } } else if (!(!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, fileObject))) && log.isDebugEnabled()) { log.debug("Couldn't get the lock for processing the file : " + child.getName()); } else if (isFailedRecord) { if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, child); VFSUtils.releaseLock(fsManager, fileObject); } if (fsManager.resolveFile(child.getURL().toString()) != null && removeTaskState == STATE_STOPPED && entry.getMoveAfterMoveFailure() != null) { workerPool.execute(new FileRemoveTask(entry, child)); } if (log.isDebugEnabled()) { log.debug( "File '" + fileObject.getURL() + "' has been marked as a failed record, it will not " + "process"); } } } if (failCount == 0 && successCount > 0) { entry.setLastPollState(PollTableEntry.SUCCSESSFUL); } else if (successCount == 0 && failCount > 0) { entry.setLastPollState(PollTableEntry.FAILED); } else { entry.setLastPollState(PollTableEntry.WITH_ERRORS); } } // processing of this poll table entry is complete long now = System.currentTimeMillis(); entry.setLastPollTime(now); entry.setNextPollTime(now + entry.getPollInterval()); } else if (log.isDebugEnabled()) { log.debug( "Unable to access or read file or directory : " + VFSUtils.maskURLPassword(fileURI) + "." + " Reason: " + (fileObject.exists() ? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!") : "The file does not exists!")); } onPollCompletion(entry); } catch (FileSystemException e) { processFailure( "Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI), e, entry); } }