@SuppressWarnings("unchecked")
private boolean retrieveFileToStreamInBody(String name, Exchange exchange)
throws GenericFileOperationFailedException {
OutputStream os = null;
String currentDir = null;
try {
GenericFile<ChannelSftp.LsEntry> target =
(GenericFile<ChannelSftp.LsEntry>) exchange.getProperty(FileComponent.FILE_EXCHANGE_FILE);
ObjectHelper.notNull(
target, "Exchange should have the " + FileComponent.FILE_EXCHANGE_FILE + " set");
String remoteName = name;
if (endpoint.getConfiguration().isStepwise()) {
// remember current directory
currentDir = getCurrentDirectory();
// change directory to path where the file is to be retrieved
// (must do this as some FTP servers cannot retrieve using absolute path)
String path = FileUtil.onlyPath(name);
if (path != null) {
changeCurrentDirectory(path);
}
// remote name is now only the file name as we just changed directory
remoteName = FileUtil.stripPath(name);
}
// use input stream which works with Apache SSHD used for testing
InputStream is = channel.get(remoteName);
if (endpoint.getConfiguration().isStreamDownload()) {
target.setBody(is);
exchange.getIn().setHeader(RemoteFileComponent.REMOTE_FILE_INPUT_STREAM, is);
} else {
os = new ByteArrayOutputStream();
target.setBody(os);
IOHelper.copyAndCloseInput(is, os);
}
return true;
} catch (IOException e) {
throw new GenericFileOperationFailedException("Cannot retrieve file: " + name, e);
} catch (SftpException e) {
throw new GenericFileOperationFailedException("Cannot retrieve file: " + name, e);
} finally {
IOHelper.close(os, "retrieve: " + name, LOG);
// change back to current directory if we changed directory
if (currentDir != null) {
changeCurrentDirectory(currentDir);
}
}
}
@Override
public Exchange createExchange(GenericFile<T> file) {
Exchange answer = new DefaultExchange(this);
if (file != null) {
file.bindToExchange(answer);
}
return answer;
}
@Test
public void testFtpSimpleConsumeAbsolute() throws Exception {
if (!canTest()) {
return;
}
String expected = "Hello World";
// create file using regular file
// FTP Server does not support absolute path, so lets simulate it
String path = FTP_ROOT_DIR + "/tmp/mytemp";
template.sendBodyAndHeader("file:" + path, expected, Exchange.FILE_NAME, "hello.txt");
MockEndpoint mock = getMockEndpoint("mock:result");
mock.expectedMessageCount(1);
mock.expectedHeaderReceived(Exchange.FILE_NAME, "hello.txt");
context.startRoute("foo");
assertMockEndpointsSatisfied();
GenericFile<?> remoteFile = (GenericFile<?>) mock.getExchanges().get(0).getIn().getBody();
assertTrue(remoteFile.getBody() instanceof InputStream);
}
@Override
public void releaseExclusiveReadLock(
GenericFileOperations<File> operations, GenericFile<File> file, Exchange exchange)
throws Exception {
// must call super
super.releaseExclusiveReadLock(operations, file, exchange);
String target = file.getFileName();
FileLock lock = exchange.getProperty(Exchange.FILE_LOCK_EXCLUSIVE_LOCK, FileLock.class);
RandomAccessFile rac =
exchange.getProperty(Exchange.FILE_LOCK_RANDOM_ACCESS_FILE, RandomAccessFile.class);
if (lock != null) {
Channel channel = lock.acquiredBy();
try {
lock.release();
} finally {
// close channel as well
IOHelper.close(channel, "while releasing exclusive read lock for file: " + target, LOG);
IOHelper.close(rac, "while releasing exclusive read lock for file: " + target, LOG);
}
}
}
@SuppressWarnings("unchecked")
private boolean retrieveFileToFileInLocalWorkDirectory(String name, Exchange exchange)
throws GenericFileOperationFailedException {
File temp;
File local = new File(endpoint.getLocalWorkDirectory());
OutputStream os;
GenericFile<ChannelSftp.LsEntry> file =
(GenericFile<ChannelSftp.LsEntry>) exchange.getProperty(FileComponent.FILE_EXCHANGE_FILE);
ObjectHelper.notNull(
file, "Exchange should have the " + FileComponent.FILE_EXCHANGE_FILE + " set");
try {
// use relative filename in local work directory
String relativeName = file.getRelativeFilePath();
temp = new File(local, relativeName + ".inprogress");
local = new File(local, relativeName);
// create directory to local work file
local.mkdirs();
// delete any existing files
if (temp.exists()) {
if (!FileUtil.deleteFile(temp)) {
throw new GenericFileOperationFailedException(
"Cannot delete existing local work file: " + temp);
}
}
if (local.exists()) {
if (!FileUtil.deleteFile(local)) {
throw new GenericFileOperationFailedException(
"Cannot delete existing local work file: " + local);
}
}
// create new temp local work file
if (!temp.createNewFile()) {
throw new GenericFileOperationFailedException("Cannot create new local work file: " + temp);
}
// store content as a file in the local work directory in the temp handle
os = new FileOutputStream(temp);
// set header with the path to the local work file
exchange.getIn().setHeader(Exchange.FILE_LOCAL_WORK_PATH, local.getPath());
} catch (Exception e) {
throw new GenericFileOperationFailedException("Cannot create new local work file: " + local);
}
String currentDir = null;
try {
// store the java.io.File handle as the body
file.setBody(local);
String remoteName = name;
if (endpoint.getConfiguration().isStepwise()) {
// remember current directory
currentDir = getCurrentDirectory();
// change directory to path where the file is to be retrieved
// (must do this as some FTP servers cannot retrieve using absolute path)
String path = FileUtil.onlyPath(name);
if (path != null) {
changeCurrentDirectory(path);
}
// remote name is now only the file name as we just changed directory
remoteName = FileUtil.stripPath(name);
}
channel.get(remoteName, os);
} catch (SftpException e) {
LOG.trace(
"Error occurred during retrieving file: {} to local directory. Deleting local work file: {}",
name,
temp);
// failed to retrieve the file so we need to close streams and delete in progress file
// must close stream before deleting file
IOHelper.close(os, "retrieve: " + name, LOG);
boolean deleted = FileUtil.deleteFile(temp);
if (!deleted) {
LOG.warn(
"Error occurred during retrieving file: "
+ name
+ " to local directory. Cannot delete local work file: "
+ temp);
}
throw new GenericFileOperationFailedException("Cannot retrieve file: " + name, e);
} finally {
IOHelper.close(os, "retrieve: " + name, LOG);
// change back to current directory if we changed directory
if (currentDir != null) {
changeCurrentDirectory(currentDir);
}
}
LOG.debug("Retrieve file to local work file result: true");
// operation went okay so rename temp to local after we have retrieved the data
LOG.trace("Renaming local in progress file from: {} to: {}", temp, local);
try {
if (!FileUtil.renameFile(temp, local, false)) {
throw new GenericFileOperationFailedException(
"Cannot rename local work file from: " + temp + " to: " + local);
}
} catch (IOException e) {
throw new GenericFileOperationFailedException(
"Cannot rename local work file from: " + temp + " to: " + local, e);
}
return true;
}
@Override
public boolean acquireExclusiveReadLock(
GenericFileOperations<File> operations, GenericFile<File> file, Exchange exchange)
throws Exception {
// must call super
if (!super.acquireExclusiveReadLock(operations, file, exchange)) {
return false;
}
File target = new File(file.getAbsoluteFilePath());
LOG.trace("Waiting for exclusive read lock to file: {}", target);
FileChannel channel = null;
RandomAccessFile randomAccessFile = null;
boolean exclusive = false;
FileLock lock = null;
try {
randomAccessFile = new RandomAccessFile(target, "rw");
// try to acquire rw lock on the file before we can consume it
channel = randomAccessFile.getChannel();
StopWatch watch = new StopWatch();
while (!exclusive) {
// timeout check
if (timeout > 0) {
long delta = watch.taken();
if (delta > timeout) {
CamelLogger.log(
LOG,
readLockLoggingLevel,
"Cannot acquire read lock within "
+ timeout
+ " millis. Will skip the file: "
+ target);
// we could not get the lock within the timeout period, so return false
return false;
}
}
// get the lock using either try lock or not depending on if we are using timeout or not
try {
lock = timeout > 0 ? channel.tryLock() : channel.lock();
} catch (IllegalStateException ex) {
// Also catch the OverlappingFileLockException here. Do nothing here
}
if (lock != null) {
LOG.trace("Acquired exclusive read lock: {} to file: {}", lock, target);
exclusive = true;
} else {
boolean interrupted = sleep();
if (interrupted) {
// we were interrupted while sleeping, we are likely being shutdown so return false
return false;
}
}
}
} catch (IOException e) {
// must handle IOException as some apps on Windows etc. will still somehow hold a lock to a
// file
// such as AntiVirus or MS Office that has special locks for it's supported files
if (timeout == 0) {
// if not using timeout, then we cant retry, so return false
return false;
}
LOG.debug("Cannot acquire read lock. Will try again.", e);
boolean interrupted = sleep();
if (interrupted) {
// we were interrupted while sleeping, we are likely being shutdown so return false
return false;
}
} finally {
// close channels if we did not grab the lock
if (!exclusive) {
IOHelper.close(channel, "while acquiring exclusive read lock for file: " + target, LOG);
IOHelper.close(
randomAccessFile, "while acquiring exclusive read lock for file: " + target, LOG);
// and also must release super lock
super.releaseExclusiveReadLock(operations, file, exchange);
}
}
// we grabbed the lock
exchange.setProperty(Exchange.FILE_LOCK_EXCLUSIVE_LOCK, lock);
exchange.setProperty(Exchange.FILE_LOCK_RANDOM_ACCESS_FILE, randomAccessFile);
return true;
}
public boolean acquireExclusiveReadLock(
GenericFileOperations<FTPFile> operations, GenericFile<FTPFile> file, Exchange exchange)
throws Exception {
boolean exclusive = false;
LOG.trace("Waiting for exclusive read lock to file: " + file);
long lastModified = Long.MIN_VALUE;
long length = Long.MIN_VALUE;
StopWatch watch = new StopWatch();
long startTime = (new Date()).getTime();
while (!exclusive) {
// timeout check
if (timeout > 0) {
long delta = watch.taken();
if (delta > timeout) {
CamelLogger.log(
LOG,
readLockLoggingLevel,
"Cannot acquire read lock within "
+ timeout
+ " millis. Will skip the file: "
+ file);
// we could not get the lock within the timeout period, so return false
return false;
}
}
long newLastModified = 0;
long newLength = 0;
List<FTPFile> files;
if (fastExistsCheck) {
// use the absolute file path to only pickup the file we want to check, this avoids
// expensive
// list operations if we have a lot of files in the directory
LOG.trace("Using fast exists to update file information for {}", file);
files = operations.listFiles(file.getAbsoluteFilePath());
} else {
LOG.trace(
"Using full directory listing to update file information for {}. Consider enabling fastExistsCheck option.",
file);
// fast option not enabled, so list the directory and filter the file name
files = operations.listFiles(file.getParent());
}
LOG.trace("List files {} found {} files", file.getAbsoluteFilePath(), files.size());
for (FTPFile f : files) {
if (f.getName().equals(file.getFileNameOnly())) {
newLength = f.getSize();
if (f.getTimestamp() != null) {
newLastModified = f.getTimestamp().getTimeInMillis();
}
}
}
LOG.trace(
"Previous last modified: " + lastModified + ", new last modified: " + newLastModified);
LOG.trace("Previous length: " + length + ", new length: " + newLength);
long newOlderThan = startTime + watch.taken() - minAge;
LOG.trace("New older than threshold: {}", newOlderThan);
if (newLength >= minLength
&& ((minAge == 0 && newLastModified == lastModified && newLength == length)
|| (minAge != 0 && newLastModified < newOlderThan))) {
LOG.trace("Read lock acquired.");
exclusive = true;
} else {
// set new base file change information
lastModified = newLastModified;
length = newLength;
boolean interrupted = sleep();
if (interrupted) {
// we were interrupted while sleeping, we are likely being shutdown so return false
return false;
}
}
}
return exclusive;
}