/** Check if the file/folder exists before proceeding and retrying */
private boolean initFileCheck() {
boolean wasError = true;
int retryCount = 0;
fileObject = null;
while (wasError) {
try {
retryCount++;
fileObject = fsManager.resolveFile(fileURI, fso);
if (fileObject == null) {
log.error("fileObject is null");
throw new FileSystemException("fileObject is null");
}
wasError = false;
} catch (FileSystemException e) {
if (retryCount >= maxRetryCount) {
log.error(
"Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI), e);
return false;
} else {
log.warn(
"Failed to resolve the file URI: "
+ VFSUtils.maskURLPassword(fileURI)
+ ", in attempt "
+ retryCount
+ ", "
+ e.getMessage()
+ " Retrying in "
+ reconnectionTimeout
+ " milliseconds.");
}
}
if (wasError) {
try {
Thread.sleep(reconnectionTimeout);
} catch (InterruptedException e2) {
log.error("Thread was interrupted while waiting to reconnect.", e2);
}
}
}
return true;
}
@Override
protected void doInit() throws AxisFault {
super.doInit();
try {
StandardFileSystemManager fsm = new StandardFileSystemManager();
fsm.setConfiguration(getClass().getClassLoader().getResource("providers.xml"));
fsm.init();
this.workerPool = super.workerPool;
fsManager = fsm;
Parameter lockFlagParam =
getTransportInDescription().getParameter(VFSConstants.TRANSPORT_FILE_LOCKING);
if (lockFlagParam != null) {
String strLockingFlag = lockFlagParam.getValue().toString();
// by-default enabled, if explicitly specified as "disable" make it disable
if (VFSConstants.TRANSPORT_FILE_LOCKING_DISABLED.equals(strLockingFlag)) {
globalFileLockingFlag = false;
}
}
} catch (FileSystemException e) {
handleException("Error initializing the file transport : " + e.getMessage(), e);
}
}
public void connect(MessageContext messageContext) throws ConnectException {
System.out.println("File deletion started...");
String fileLocation =
getParameter(messageContext, "filelocation") == null
? ""
: getParameter(messageContext, "filelocation").toString();
String filename =
getParameter(messageContext, "file") == null
? ""
: getParameter(messageContext, "file").toString();
String filebeforepprocess =
getParameter(messageContext, "filebeforeprocess") == null
? ""
: getParameter(messageContext, "filebeforeprocess").toString();
String fileafterprocess =
getParameter(messageContext, "fileafterprocess") == null
? ""
: getParameter(messageContext, "fileafterprocess").toString();
String newFileLocation =
getParameter(messageContext, "newfilelocation") == null
? ""
: getParameter(messageContext, "newfilelocation").toString();
if (log.isDebugEnabled()) {
log.info("File deletion started..." + filename.toString());
log.info("File Location..." + fileLocation);
}
boolean resultStatus = false;
try {
resultStatus =
moveFile(fileLocation, filename, filebeforepprocess, fileafterprocess, newFileLocation);
} catch (FileSystemException e) {
handleException(e.getMessage(), messageContext);
}
generateResults(messageContext, resultStatus);
}
/**
* Do the file processing operation for the given set of properties. Do the checks and pass the
* control to processFile method
*/
public FileObject poll() {
if (fileURI == null || fileURI.trim().equals("")) {
log.error(
"Invalid file url. Check the inbound endpoint configuration. Endpoint Name : "
+ name
+ ", File URL : "
+ fileURI);
return null;
}
if (log.isDebugEnabled()) {
log.debug("Start : Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI));
}
if (!initFileCheck()) {
// Unable to read from the source location provided.
return null;
}
// If file/folder found proceed to the processing stage
try {
lastCycle = 0;
if (fileObject.exists() && fileObject.isReadable()) {
FileObject[] children = null;
try {
children = fileObject.getChildren();
} catch (FileNotFolderException ignored) {
if (log.isDebugEnabled()) {
log.debug("No Folder found. Only file found on : " + VFSUtils.maskURLPassword(fileURI));
}
} catch (FileSystemException ex) {
log.error(ex.getMessage(), ex);
}
// if this is a file that would translate to a single message
if (children == null || children.length == 0) {
// Fail record is a one that is processed but was not moved
// or deleted due to an error.
boolean isFailedRecord = VFSUtils.isFailRecord(fsManager, fileObject);
if (!isFailedRecord) {
fileHandler();
if (injectHandler == null) {
return fileObject;
}
} else {
try {
lastCycle = 2;
moveOrDeleteAfterProcessing(fileObject);
} catch (SynapseException synapseException) {
log.error(
"File object '"
+ fileObject.getURL().toString()
+ "' "
+ "cloud not be moved after first attempt",
synapseException);
}
if (fileLock) {
// TODO: passing null to avoid build break. Fix properly
VFSUtils.releaseLock(fsManager, fileObject, fso);
}
if (log.isDebugEnabled()) {
log.debug(
"File '"
+ fileObject.getURL()
+ "' has been marked as a failed"
+ " record, it will not process");
}
}
} else {
FileObject fileObject = directoryHandler(children);
if (fileObject != null) {
return fileObject;
}
}
} else {
log.warn(
"Unable to access or read file or directory : "
+ VFSUtils.maskURLPassword(fileURI)
+ "."
+ " Reason: "
+ (fileObject.exists()
? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!")
: "The file does not exists!"));
return null;
}
} catch (FileSystemException e) {
log.error(
"Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI), e);
return null;
} catch (Exception e) {
log.error(
"Error while processing the file/folder in URL : " + VFSUtils.maskURLPassword(fileURI),
e);
return null;
} finally {
try {
fsManager.closeFileSystem(fileObject.getParent().getFileSystem());
fileObject.close();
} catch (Exception e) {
log.error("Unable to close the file system. " + e.getMessage());
log.error(e);
}
}
if (log.isDebugEnabled()) {
log.debug("End : Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI));
}
return null;
}
private void killLogstashAgent(int logAgentNumber, String logstashLogPath) {
FileObject listendir;
CustomFileListener listener = new CustomFileListener();
long TIMEOUT_BETWEEN_FILE_QUERYING = 1000;
long LOOP_TIMEOUT_IN_MILLIS = 10 * 1000;
try {
FileSystemManager fileSystemManager = VFS.getManager();
listendir = fileSystemManager.resolveFile(logstashLogPath);
} catch (FileSystemException e) {
e.printStackTrace();
return;
}
DefaultFileMonitor fm = new DefaultFileMonitor(listener);
fm.setRecursive(true);
fm.addFile(listendir);
fm.setDelay(TIMEOUT_BETWEEN_FILE_QUERYING);
fm.start();
LogUtils.log("waiting to destroy logger");
long startTimeMillis = System.currentTimeMillis();
while (true) {
if (!listener.isProcessUp()) {
break;
}
listener.setProcessUp(false);
try {
TimeUnit.MILLISECONDS.sleep(LOOP_TIMEOUT_IN_MILLIS);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
long endTimeMillis = System.currentTimeMillis();
LogUtils.log("destroying logstash agent " + logAgentNumber);
LogUtils.log("waited " + (endTimeMillis - startTimeMillis) / 1000 + " seconds");
fm.stop();
// File logstashOutputFile = new File(logstashLogPath);
if (logAgentNumber == 1) {
process.destroy();
process = null;
} else {
process2.destroy();
process2 = null;
}
// try {
// TimeUnit.SECONDS.sleep(5);
//
// LogUtils.log("returning logstash config file to initial state");
// if(logAgentNumber == 1){
// IOUtils.replaceFileWithMove(new File(confFilePath), new File(backupFilePath));
// FileUtils.deleteQuietly(new File(backupFilePath));
// }
// else{
// IOUtils.replaceFileWithMove(new File(confFilePath2), new
// File(backupFilePath2));
// FileUtils.deleteQuietly(new File(backupFilePath2));
//
// }
// } catch (IOException e) {
// e.printStackTrace();
// } catch (InterruptedException e) {
// e.printStackTrace();
// }
// if(logstashOutputFile.exists()){
// FileUtils.deleteQuietly(logstashOutputFile);
// }
}
/**
* Search for files that match the given regex pattern and create a list Then process each of
* these files and update the status of the scan on the poll table
*
* @param entry the poll table entry for the scan
* @param fileURI the file or directory to be scanned
*/
private void scanFileOrDirectory(final PollTableEntry entry, String fileURI) {
FileObject fileObject = null;
if (log.isDebugEnabled()) {
log.debug("Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI));
}
boolean wasError = true;
int retryCount = 0;
int maxRetryCount = entry.getMaxRetryCount();
long reconnectionTimeout = entry.getReconnectTimeout();
while (wasError) {
try {
retryCount++;
fileObject = fsManager.resolveFile(fileURI);
if (fileObject == null) {
log.error("fileObject is null");
throw new FileSystemException("fileObject is null");
}
wasError = false;
} catch (FileSystemException e) {
if (retryCount >= maxRetryCount) {
processFailure(
"Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI),
e,
entry);
return;
} else {
log.warn(
"Failed to resolve the file URI: "
+ VFSUtils.maskURLPassword(fileURI)
+ ", in attempt "
+ retryCount
+ ", "
+ e.getMessage()
+ " Retrying in "
+ reconnectionTimeout
+ " milliseconds.");
}
}
if (wasError) {
try {
Thread.sleep(reconnectionTimeout);
} catch (InterruptedException e2) {
log.error("Thread was interrupted while waiting to reconnect.", e2);
}
}
}
try {
if (fileObject.exists() && fileObject.isReadable()) {
entry.setLastPollState(PollTableEntry.NONE);
FileObject[] children = null;
try {
children = fileObject.getChildren();
} catch (FileSystemException ignore) {
}
// if this is a file that would translate to a single message
if (children == null || children.length == 0) {
boolean isFailedRecord = false;
if (entry.getMoveAfterMoveFailure() != null) {
isFailedRecord = isFailedRecord(fileObject, entry);
}
if (fileObject.getType() == FileType.FILE && !isFailedRecord) {
if (!entry.isFileLockingEnabled()
|| (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, fileObject))) {
try {
processFile(entry, fileObject);
entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
metrics.incrementMessagesReceived();
} catch (AxisFault e) {
logException("Error processing File URI : " + fileObject.getName(), e);
entry.setLastPollState(PollTableEntry.FAILED);
metrics.incrementFaultsReceiving();
}
try {
moveOrDeleteAfterProcessing(entry, fileObject);
} catch (AxisFault axisFault) {
logException(
"File object '" + fileObject.getURL().toString() + "' " + "cloud not be moved",
axisFault);
entry.setLastPollState(PollTableEntry.FAILED);
String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat());
addFailedRecord(entry, fileObject, timeStamp);
}
if (entry.isFileLockingEnabled()) {
VFSUtils.releaseLock(fsManager, fileObject);
if (log.isDebugEnabled()) {
log.debug(
"Removed the lock file '"
+ fileObject.toString()
+ ".lock' of the file '"
+ fileObject.toString());
}
}
} else if (log.isDebugEnabled()) {
log.debug("Couldn't get the lock for processing the file : " + fileObject.getName());
} else if (isFailedRecord) {
if (entry.isFileLockingEnabled()) {
VFSUtils.releaseLock(fsManager, fileObject);
}
// schedule a cleanup task if the file is there
if (fsManager.resolveFile(fileObject.getURL().toString()) != null
&& removeTaskState == STATE_STOPPED
&& entry.getMoveAfterMoveFailure() != null) {
workerPool.execute(new FileRemoveTask(entry, fileObject));
}
if (log.isDebugEnabled()) {
log.debug(
"File '"
+ fileObject.getURL()
+ "' has been marked as a failed"
+ " record, it will not process");
}
}
}
} else {
int failCount = 0;
int successCount = 0;
if (log.isDebugEnabled()) {
log.debug("File name pattern : " + entry.getFileNamePattern());
}
for (FileObject child : children) {
boolean isFailedRecord = false;
if (entry.getMoveAfterMoveFailure() != null) {
isFailedRecord = isFailedRecord(child, entry);
}
if (log.isDebugEnabled()) {
log.debug("Matching file : " + child.getName().getBaseName());
}
if ((entry.getFileNamePattern() != null)
&& (child.getName().getBaseName().matches(entry.getFileNamePattern()))
&& (!entry.isFileLockingEnabled()
|| (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, child)))
&& !isFailedRecord) {
try {
if (log.isDebugEnabled()) {
log.debug("Processing file :" + child);
}
processFile(entry, child);
successCount++;
// tell moveOrDeleteAfterProcessing() file was success
entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
metrics.incrementMessagesReceived();
} catch (Exception e) {
logException("Error processing File URI : " + child.getName(), e);
failCount++;
// tell moveOrDeleteAfterProcessing() file failed
entry.setLastPollState(PollTableEntry.FAILED);
metrics.incrementFaultsReceiving();
}
try {
moveOrDeleteAfterProcessing(entry, child);
} catch (AxisFault axisFault) {
logException(
"File object '" + child.getURL().toString() + "'cloud not be moved", axisFault);
failCount++;
entry.setLastPollState(PollTableEntry.FAILED);
String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat());
addFailedRecord(entry, child, timeStamp);
}
// if there is a failure or not we'll try to release the lock
if (entry.isFileLockingEnabled()) {
VFSUtils.releaseLock(fsManager, child);
}
} else if (!(!entry.isFileLockingEnabled()
|| (entry.isFileLockingEnabled()
&& VFSUtils.acquireLock(fsManager, fileObject)))
&& log.isDebugEnabled()) {
log.debug("Couldn't get the lock for processing the file : " + child.getName());
} else if (isFailedRecord) {
if (entry.isFileLockingEnabled()) {
VFSUtils.releaseLock(fsManager, child);
VFSUtils.releaseLock(fsManager, fileObject);
}
if (fsManager.resolveFile(child.getURL().toString()) != null
&& removeTaskState == STATE_STOPPED
&& entry.getMoveAfterMoveFailure() != null) {
workerPool.execute(new FileRemoveTask(entry, child));
}
if (log.isDebugEnabled()) {
log.debug(
"File '"
+ fileObject.getURL()
+ "' has been marked as a failed record, it will not "
+ "process");
}
}
}
if (failCount == 0 && successCount > 0) {
entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
} else if (successCount == 0 && failCount > 0) {
entry.setLastPollState(PollTableEntry.FAILED);
} else {
entry.setLastPollState(PollTableEntry.WITH_ERRORS);
}
}
// processing of this poll table entry is complete
long now = System.currentTimeMillis();
entry.setLastPollTime(now);
entry.setNextPollTime(now + entry.getPollInterval());
} else if (log.isDebugEnabled()) {
log.debug(
"Unable to access or read file or directory : "
+ VFSUtils.maskURLPassword(fileURI)
+ "."
+ " Reason: "
+ (fileObject.exists()
? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!")
: "The file does not exists!"));
}
onPollCompletion(entry);
} catch (FileSystemException e) {
processFailure(
"Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI),
e,
entry);
}
}