/** Check if the file/folder exists before proceeding and retrying */ private boolean initFileCheck() { boolean wasError = true; int retryCount = 0; fileObject = null; while (wasError) { try { retryCount++; fileObject = fsManager.resolveFile(fileURI, fso); if (fileObject == null) { log.error("fileObject is null"); throw new FileSystemException("fileObject is null"); } wasError = false; } catch (FileSystemException e) { if (retryCount >= maxRetryCount) { log.error( "Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI), e); return false; } else { log.warn( "Failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI) + ", in attempt " + retryCount + ", " + e.getMessage() + " Retrying in " + reconnectionTimeout + " milliseconds."); } } if (wasError) { try { Thread.sleep(reconnectionTimeout); } catch (InterruptedException e2) { log.error("Thread was interrupted while waiting to reconnect.", e2); } } } return true; }
@Override protected void doInit() throws AxisFault { super.doInit(); try { StandardFileSystemManager fsm = new StandardFileSystemManager(); fsm.setConfiguration(getClass().getClassLoader().getResource("providers.xml")); fsm.init(); this.workerPool = super.workerPool; fsManager = fsm; Parameter lockFlagParam = getTransportInDescription().getParameter(VFSConstants.TRANSPORT_FILE_LOCKING); if (lockFlagParam != null) { String strLockingFlag = lockFlagParam.getValue().toString(); // by-default enabled, if explicitly specified as "disable" make it disable if (VFSConstants.TRANSPORT_FILE_LOCKING_DISABLED.equals(strLockingFlag)) { globalFileLockingFlag = false; } } } catch (FileSystemException e) { handleException("Error initializing the file transport : " + e.getMessage(), e); } }
public void connect(MessageContext messageContext) throws ConnectException { System.out.println("File deletion started..."); String fileLocation = getParameter(messageContext, "filelocation") == null ? "" : getParameter(messageContext, "filelocation").toString(); String filename = getParameter(messageContext, "file") == null ? "" : getParameter(messageContext, "file").toString(); String filebeforepprocess = getParameter(messageContext, "filebeforeprocess") == null ? "" : getParameter(messageContext, "filebeforeprocess").toString(); String fileafterprocess = getParameter(messageContext, "fileafterprocess") == null ? "" : getParameter(messageContext, "fileafterprocess").toString(); String newFileLocation = getParameter(messageContext, "newfilelocation") == null ? "" : getParameter(messageContext, "newfilelocation").toString(); if (log.isDebugEnabled()) { log.info("File deletion started..." + filename.toString()); log.info("File Location..." + fileLocation); } boolean resultStatus = false; try { resultStatus = moveFile(fileLocation, filename, filebeforepprocess, fileafterprocess, newFileLocation); } catch (FileSystemException e) { handleException(e.getMessage(), messageContext); } generateResults(messageContext, resultStatus); }
/** * Do the file processing operation for the given set of properties. Do the checks and pass the * control to processFile method */ public FileObject poll() { if (fileURI == null || fileURI.trim().equals("")) { log.error( "Invalid file url. Check the inbound endpoint configuration. Endpoint Name : " + name + ", File URL : " + fileURI); return null; } if (log.isDebugEnabled()) { log.debug("Start : Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI)); } if (!initFileCheck()) { // Unable to read from the source location provided. return null; } // If file/folder found proceed to the processing stage try { lastCycle = 0; if (fileObject.exists() && fileObject.isReadable()) { FileObject[] children = null; try { children = fileObject.getChildren(); } catch (FileNotFolderException ignored) { if (log.isDebugEnabled()) { log.debug("No Folder found. Only file found on : " + VFSUtils.maskURLPassword(fileURI)); } } catch (FileSystemException ex) { log.error(ex.getMessage(), ex); } // if this is a file that would translate to a single message if (children == null || children.length == 0) { // Fail record is a one that is processed but was not moved // or deleted due to an error. boolean isFailedRecord = VFSUtils.isFailRecord(fsManager, fileObject); if (!isFailedRecord) { fileHandler(); if (injectHandler == null) { return fileObject; } } else { try { lastCycle = 2; moveOrDeleteAfterProcessing(fileObject); } catch (SynapseException synapseException) { log.error( "File object '" + fileObject.getURL().toString() + "' " + "cloud not be moved after first attempt", synapseException); } if (fileLock) { // TODO: passing null to avoid build break. Fix properly VFSUtils.releaseLock(fsManager, fileObject, fso); } if (log.isDebugEnabled()) { log.debug( "File '" + fileObject.getURL() + "' has been marked as a failed" + " record, it will not process"); } } } else { FileObject fileObject = directoryHandler(children); if (fileObject != null) { return fileObject; } } } else { log.warn( "Unable to access or read file or directory : " + VFSUtils.maskURLPassword(fileURI) + "." + " Reason: " + (fileObject.exists() ? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!") : "The file does not exists!")); return null; } } catch (FileSystemException e) { log.error( "Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI), e); return null; } catch (Exception e) { log.error( "Error while processing the file/folder in URL : " + VFSUtils.maskURLPassword(fileURI), e); return null; } finally { try { fsManager.closeFileSystem(fileObject.getParent().getFileSystem()); fileObject.close(); } catch (Exception e) { log.error("Unable to close the file system. " + e.getMessage()); log.error(e); } } if (log.isDebugEnabled()) { log.debug("End : Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI)); } return null; }
private void killLogstashAgent(int logAgentNumber, String logstashLogPath) { FileObject listendir; CustomFileListener listener = new CustomFileListener(); long TIMEOUT_BETWEEN_FILE_QUERYING = 1000; long LOOP_TIMEOUT_IN_MILLIS = 10 * 1000; try { FileSystemManager fileSystemManager = VFS.getManager(); listendir = fileSystemManager.resolveFile(logstashLogPath); } catch (FileSystemException e) { e.printStackTrace(); return; } DefaultFileMonitor fm = new DefaultFileMonitor(listener); fm.setRecursive(true); fm.addFile(listendir); fm.setDelay(TIMEOUT_BETWEEN_FILE_QUERYING); fm.start(); LogUtils.log("waiting to destroy logger"); long startTimeMillis = System.currentTimeMillis(); while (true) { if (!listener.isProcessUp()) { break; } listener.setProcessUp(false); try { TimeUnit.MILLISECONDS.sleep(LOOP_TIMEOUT_IN_MILLIS); } catch (InterruptedException e) { e.printStackTrace(); } } long endTimeMillis = System.currentTimeMillis(); LogUtils.log("destroying logstash agent " + logAgentNumber); LogUtils.log("waited " + (endTimeMillis - startTimeMillis) / 1000 + " seconds"); fm.stop(); // File logstashOutputFile = new File(logstashLogPath); if (logAgentNumber == 1) { process.destroy(); process = null; } else { process2.destroy(); process2 = null; } // try { // TimeUnit.SECONDS.sleep(5); // // LogUtils.log("returning logstash config file to initial state"); // if(logAgentNumber == 1){ // IOUtils.replaceFileWithMove(new File(confFilePath), new File(backupFilePath)); // FileUtils.deleteQuietly(new File(backupFilePath)); // } // else{ // IOUtils.replaceFileWithMove(new File(confFilePath2), new // File(backupFilePath2)); // FileUtils.deleteQuietly(new File(backupFilePath2)); // // } // } catch (IOException e) { // e.printStackTrace(); // } catch (InterruptedException e) { // e.printStackTrace(); // } // if(logstashOutputFile.exists()){ // FileUtils.deleteQuietly(logstashOutputFile); // } }
/** * Search for files that match the given regex pattern and create a list Then process each of * these files and update the status of the scan on the poll table * * @param entry the poll table entry for the scan * @param fileURI the file or directory to be scanned */ private void scanFileOrDirectory(final PollTableEntry entry, String fileURI) { FileObject fileObject = null; if (log.isDebugEnabled()) { log.debug("Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI)); } boolean wasError = true; int retryCount = 0; int maxRetryCount = entry.getMaxRetryCount(); long reconnectionTimeout = entry.getReconnectTimeout(); while (wasError) { try { retryCount++; fileObject = fsManager.resolveFile(fileURI); if (fileObject == null) { log.error("fileObject is null"); throw new FileSystemException("fileObject is null"); } wasError = false; } catch (FileSystemException e) { if (retryCount >= maxRetryCount) { processFailure( "Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI), e, entry); return; } else { log.warn( "Failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI) + ", in attempt " + retryCount + ", " + e.getMessage() + " Retrying in " + reconnectionTimeout + " milliseconds."); } } if (wasError) { try { Thread.sleep(reconnectionTimeout); } catch (InterruptedException e2) { log.error("Thread was interrupted while waiting to reconnect.", e2); } } } try { if (fileObject.exists() && fileObject.isReadable()) { entry.setLastPollState(PollTableEntry.NONE); FileObject[] children = null; try { children = fileObject.getChildren(); } catch (FileSystemException ignore) { } // if this is a file that would translate to a single message if (children == null || children.length == 0) { boolean isFailedRecord = false; if (entry.getMoveAfterMoveFailure() != null) { isFailedRecord = isFailedRecord(fileObject, entry); } if (fileObject.getType() == FileType.FILE && !isFailedRecord) { if (!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, fileObject))) { try { processFile(entry, fileObject); entry.setLastPollState(PollTableEntry.SUCCSESSFUL); metrics.incrementMessagesReceived(); } catch (AxisFault e) { logException("Error processing File URI : " + fileObject.getName(), e); entry.setLastPollState(PollTableEntry.FAILED); metrics.incrementFaultsReceiving(); } try { moveOrDeleteAfterProcessing(entry, fileObject); } catch (AxisFault axisFault) { logException( "File object '" + fileObject.getURL().toString() + "' " + "cloud not be moved", axisFault); entry.setLastPollState(PollTableEntry.FAILED); String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat()); addFailedRecord(entry, fileObject, timeStamp); } if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, fileObject); if (log.isDebugEnabled()) { log.debug( "Removed the lock file '" + fileObject.toString() + ".lock' of the file '" + fileObject.toString()); } } } else if (log.isDebugEnabled()) { log.debug("Couldn't get the lock for processing the file : " + fileObject.getName()); } else if (isFailedRecord) { if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, fileObject); } // schedule a cleanup task if the file is there if (fsManager.resolveFile(fileObject.getURL().toString()) != null && removeTaskState == STATE_STOPPED && entry.getMoveAfterMoveFailure() != null) { workerPool.execute(new FileRemoveTask(entry, fileObject)); } if (log.isDebugEnabled()) { log.debug( "File '" + fileObject.getURL() + "' has been marked as a failed" + " record, it will not process"); } } } } else { int failCount = 0; int successCount = 0; if (log.isDebugEnabled()) { log.debug("File name pattern : " + entry.getFileNamePattern()); } for (FileObject child : children) { boolean isFailedRecord = false; if (entry.getMoveAfterMoveFailure() != null) { isFailedRecord = isFailedRecord(child, entry); } if (log.isDebugEnabled()) { log.debug("Matching file : " + child.getName().getBaseName()); } if ((entry.getFileNamePattern() != null) && (child.getName().getBaseName().matches(entry.getFileNamePattern())) && (!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, child))) && !isFailedRecord) { try { if (log.isDebugEnabled()) { log.debug("Processing file :" + child); } processFile(entry, child); successCount++; // tell moveOrDeleteAfterProcessing() file was success entry.setLastPollState(PollTableEntry.SUCCSESSFUL); metrics.incrementMessagesReceived(); } catch (Exception e) { logException("Error processing File URI : " + child.getName(), e); failCount++; // tell moveOrDeleteAfterProcessing() file failed entry.setLastPollState(PollTableEntry.FAILED); metrics.incrementFaultsReceiving(); } try { moveOrDeleteAfterProcessing(entry, child); } catch (AxisFault axisFault) { logException( "File object '" + child.getURL().toString() + "'cloud not be moved", axisFault); failCount++; entry.setLastPollState(PollTableEntry.FAILED); String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat()); addFailedRecord(entry, child, timeStamp); } // if there is a failure or not we'll try to release the lock if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, child); } } else if (!(!entry.isFileLockingEnabled() || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, fileObject))) && log.isDebugEnabled()) { log.debug("Couldn't get the lock for processing the file : " + child.getName()); } else if (isFailedRecord) { if (entry.isFileLockingEnabled()) { VFSUtils.releaseLock(fsManager, child); VFSUtils.releaseLock(fsManager, fileObject); } if (fsManager.resolveFile(child.getURL().toString()) != null && removeTaskState == STATE_STOPPED && entry.getMoveAfterMoveFailure() != null) { workerPool.execute(new FileRemoveTask(entry, child)); } if (log.isDebugEnabled()) { log.debug( "File '" + fileObject.getURL() + "' has been marked as a failed record, it will not " + "process"); } } } if (failCount == 0 && successCount > 0) { entry.setLastPollState(PollTableEntry.SUCCSESSFUL); } else if (successCount == 0 && failCount > 0) { entry.setLastPollState(PollTableEntry.FAILED); } else { entry.setLastPollState(PollTableEntry.WITH_ERRORS); } } // processing of this poll table entry is complete long now = System.currentTimeMillis(); entry.setLastPollTime(now); entry.setNextPollTime(now + entry.getPollInterval()); } else if (log.isDebugEnabled()) { log.debug( "Unable to access or read file or directory : " + VFSUtils.maskURLPassword(fileURI) + "." + " Reason: " + (fileObject.exists() ? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!") : "The file does not exists!")); } onPollCompletion(entry); } catch (FileSystemException e) { processFailure( "Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI), e, entry); } }