Ejemplo n.º 1
0
  @SuppressWarnings("unchecked")
  private boolean retrieveFileToStreamInBody(String name, Exchange exchange)
      throws GenericFileOperationFailedException {
    OutputStream os = null;
    String currentDir = null;
    try {
      GenericFile<ChannelSftp.LsEntry> target =
          (GenericFile<ChannelSftp.LsEntry>) exchange.getProperty(FileComponent.FILE_EXCHANGE_FILE);
      ObjectHelper.notNull(
          target, "Exchange should have the " + FileComponent.FILE_EXCHANGE_FILE + " set");

      String remoteName = name;
      if (endpoint.getConfiguration().isStepwise()) {
        // remember current directory
        currentDir = getCurrentDirectory();

        // change directory to path where the file is to be retrieved
        // (must do this as some FTP servers cannot retrieve using absolute path)
        String path = FileUtil.onlyPath(name);
        if (path != null) {
          changeCurrentDirectory(path);
        }
        // remote name is now only the file name as we just changed directory
        remoteName = FileUtil.stripPath(name);
      }

      // use input stream which works with Apache SSHD used for testing
      InputStream is = channel.get(remoteName);

      if (endpoint.getConfiguration().isStreamDownload()) {
        target.setBody(is);
        exchange.getIn().setHeader(RemoteFileComponent.REMOTE_FILE_INPUT_STREAM, is);
      } else {
        os = new ByteArrayOutputStream();
        target.setBody(os);
        IOHelper.copyAndCloseInput(is, os);
      }

      return true;
    } catch (IOException e) {
      throw new GenericFileOperationFailedException("Cannot retrieve file: " + name, e);
    } catch (SftpException e) {
      throw new GenericFileOperationFailedException("Cannot retrieve file: " + name, e);
    } finally {
      IOHelper.close(os, "retrieve: " + name, LOG);
      // change back to current directory if we changed directory
      if (currentDir != null) {
        changeCurrentDirectory(currentDir);
      }
    }
  }
Ejemplo n.º 2
0
 @Override
 public Exchange createExchange(GenericFile<T> file) {
   Exchange answer = new DefaultExchange(this);
   if (file != null) {
     file.bindToExchange(answer);
   }
   return answer;
 }
  @Test
  public void testFtpSimpleConsumeAbsolute() throws Exception {
    if (!canTest()) {
      return;
    }

    String expected = "Hello World";

    // create file using regular file

    // FTP Server does not support absolute path, so lets simulate it
    String path = FTP_ROOT_DIR + "/tmp/mytemp";
    template.sendBodyAndHeader("file:" + path, expected, Exchange.FILE_NAME, "hello.txt");

    MockEndpoint mock = getMockEndpoint("mock:result");
    mock.expectedMessageCount(1);
    mock.expectedHeaderReceived(Exchange.FILE_NAME, "hello.txt");

    context.startRoute("foo");

    assertMockEndpointsSatisfied();
    GenericFile<?> remoteFile = (GenericFile<?>) mock.getExchanges().get(0).getIn().getBody();
    assertTrue(remoteFile.getBody() instanceof InputStream);
  }
  @Override
  public void releaseExclusiveReadLock(
      GenericFileOperations<File> operations, GenericFile<File> file, Exchange exchange)
      throws Exception {

    // must call super
    super.releaseExclusiveReadLock(operations, file, exchange);

    String target = file.getFileName();
    FileLock lock = exchange.getProperty(Exchange.FILE_LOCK_EXCLUSIVE_LOCK, FileLock.class);
    RandomAccessFile rac =
        exchange.getProperty(Exchange.FILE_LOCK_RANDOM_ACCESS_FILE, RandomAccessFile.class);

    if (lock != null) {
      Channel channel = lock.acquiredBy();
      try {
        lock.release();
      } finally {
        // close channel as well
        IOHelper.close(channel, "while releasing exclusive read lock for file: " + target, LOG);
        IOHelper.close(rac, "while releasing exclusive read lock for file: " + target, LOG);
      }
    }
  }
Ejemplo n.º 5
0
  @SuppressWarnings("unchecked")
  private boolean retrieveFileToFileInLocalWorkDirectory(String name, Exchange exchange)
      throws GenericFileOperationFailedException {
    File temp;
    File local = new File(endpoint.getLocalWorkDirectory());
    OutputStream os;
    GenericFile<ChannelSftp.LsEntry> file =
        (GenericFile<ChannelSftp.LsEntry>) exchange.getProperty(FileComponent.FILE_EXCHANGE_FILE);
    ObjectHelper.notNull(
        file, "Exchange should have the " + FileComponent.FILE_EXCHANGE_FILE + " set");
    try {
      // use relative filename in local work directory
      String relativeName = file.getRelativeFilePath();

      temp = new File(local, relativeName + ".inprogress");
      local = new File(local, relativeName);

      // create directory to local work file
      local.mkdirs();

      // delete any existing files
      if (temp.exists()) {
        if (!FileUtil.deleteFile(temp)) {
          throw new GenericFileOperationFailedException(
              "Cannot delete existing local work file: " + temp);
        }
      }
      if (local.exists()) {
        if (!FileUtil.deleteFile(local)) {
          throw new GenericFileOperationFailedException(
              "Cannot delete existing local work file: " + local);
        }
      }

      // create new temp local work file
      if (!temp.createNewFile()) {
        throw new GenericFileOperationFailedException("Cannot create new local work file: " + temp);
      }

      // store content as a file in the local work directory in the temp handle
      os = new FileOutputStream(temp);

      // set header with the path to the local work file
      exchange.getIn().setHeader(Exchange.FILE_LOCAL_WORK_PATH, local.getPath());
    } catch (Exception e) {
      throw new GenericFileOperationFailedException("Cannot create new local work file: " + local);
    }
    String currentDir = null;
    try {
      // store the java.io.File handle as the body
      file.setBody(local);

      String remoteName = name;
      if (endpoint.getConfiguration().isStepwise()) {
        // remember current directory
        currentDir = getCurrentDirectory();

        // change directory to path where the file is to be retrieved
        // (must do this as some FTP servers cannot retrieve using absolute path)
        String path = FileUtil.onlyPath(name);
        if (path != null) {
          changeCurrentDirectory(path);
        }
        // remote name is now only the file name as we just changed directory
        remoteName = FileUtil.stripPath(name);
      }

      channel.get(remoteName, os);

    } catch (SftpException e) {
      LOG.trace(
          "Error occurred during retrieving file: {} to local directory. Deleting local work file: {}",
          name,
          temp);
      // failed to retrieve the file so we need to close streams and delete in progress file
      // must close stream before deleting file
      IOHelper.close(os, "retrieve: " + name, LOG);
      boolean deleted = FileUtil.deleteFile(temp);
      if (!deleted) {
        LOG.warn(
            "Error occurred during retrieving file: "
                + name
                + " to local directory. Cannot delete local work file: "
                + temp);
      }
      throw new GenericFileOperationFailedException("Cannot retrieve file: " + name, e);
    } finally {
      IOHelper.close(os, "retrieve: " + name, LOG);

      // change back to current directory if we changed directory
      if (currentDir != null) {
        changeCurrentDirectory(currentDir);
      }
    }

    LOG.debug("Retrieve file to local work file result: true");

    // operation went okay so rename temp to local after we have retrieved the data
    LOG.trace("Renaming local in progress file from: {} to: {}", temp, local);
    try {
      if (!FileUtil.renameFile(temp, local, false)) {
        throw new GenericFileOperationFailedException(
            "Cannot rename local work file from: " + temp + " to: " + local);
      }
    } catch (IOException e) {
      throw new GenericFileOperationFailedException(
          "Cannot rename local work file from: " + temp + " to: " + local, e);
    }

    return true;
  }
  @Override
  public boolean acquireExclusiveReadLock(
      GenericFileOperations<File> operations, GenericFile<File> file, Exchange exchange)
      throws Exception {
    // must call super
    if (!super.acquireExclusiveReadLock(operations, file, exchange)) {
      return false;
    }

    File target = new File(file.getAbsoluteFilePath());

    LOG.trace("Waiting for exclusive read lock to file: {}", target);

    FileChannel channel = null;
    RandomAccessFile randomAccessFile = null;

    boolean exclusive = false;
    FileLock lock = null;

    try {
      randomAccessFile = new RandomAccessFile(target, "rw");
      // try to acquire rw lock on the file before we can consume it
      channel = randomAccessFile.getChannel();

      StopWatch watch = new StopWatch();

      while (!exclusive) {
        // timeout check
        if (timeout > 0) {
          long delta = watch.taken();
          if (delta > timeout) {
            CamelLogger.log(
                LOG,
                readLockLoggingLevel,
                "Cannot acquire read lock within "
                    + timeout
                    + " millis. Will skip the file: "
                    + target);
            // we could not get the lock within the timeout period, so return false
            return false;
          }
        }

        // get the lock using either try lock or not depending on if we are using timeout or not
        try {
          lock = timeout > 0 ? channel.tryLock() : channel.lock();
        } catch (IllegalStateException ex) {
          // Also catch the OverlappingFileLockException here. Do nothing here
        }
        if (lock != null) {
          LOG.trace("Acquired exclusive read lock: {} to file: {}", lock, target);
          exclusive = true;
        } else {
          boolean interrupted = sleep();
          if (interrupted) {
            // we were interrupted while sleeping, we are likely being shutdown so return false
            return false;
          }
        }
      }
    } catch (IOException e) {
      // must handle IOException as some apps on Windows etc. will still somehow hold a lock to a
      // file
      // such as AntiVirus or MS Office that has special locks for it's supported files
      if (timeout == 0) {
        // if not using timeout, then we cant retry, so return false
        return false;
      }
      LOG.debug("Cannot acquire read lock. Will try again.", e);
      boolean interrupted = sleep();
      if (interrupted) {
        // we were interrupted while sleeping, we are likely being shutdown so return false
        return false;
      }
    } finally {
      // close channels if we did not grab the lock
      if (!exclusive) {
        IOHelper.close(channel, "while acquiring exclusive read lock for file: " + target, LOG);
        IOHelper.close(
            randomAccessFile, "while acquiring exclusive read lock for file: " + target, LOG);

        // and also must release super lock
        super.releaseExclusiveReadLock(operations, file, exchange);
      }
    }

    // we grabbed the lock
    exchange.setProperty(Exchange.FILE_LOCK_EXCLUSIVE_LOCK, lock);
    exchange.setProperty(Exchange.FILE_LOCK_RANDOM_ACCESS_FILE, randomAccessFile);

    return true;
  }
  public boolean acquireExclusiveReadLock(
      GenericFileOperations<FTPFile> operations, GenericFile<FTPFile> file, Exchange exchange)
      throws Exception {
    boolean exclusive = false;

    LOG.trace("Waiting for exclusive read lock to file: " + file);

    long lastModified = Long.MIN_VALUE;
    long length = Long.MIN_VALUE;
    StopWatch watch = new StopWatch();
    long startTime = (new Date()).getTime();

    while (!exclusive) {
      // timeout check
      if (timeout > 0) {
        long delta = watch.taken();
        if (delta > timeout) {
          CamelLogger.log(
              LOG,
              readLockLoggingLevel,
              "Cannot acquire read lock within "
                  + timeout
                  + " millis. Will skip the file: "
                  + file);
          // we could not get the lock within the timeout period, so return false
          return false;
        }
      }

      long newLastModified = 0;
      long newLength = 0;

      List<FTPFile> files;
      if (fastExistsCheck) {
        // use the absolute file path to only pickup the file we want to check, this avoids
        // expensive
        // list operations if we have a lot of files in the directory
        LOG.trace("Using fast exists to update file information for {}", file);
        files = operations.listFiles(file.getAbsoluteFilePath());
      } else {
        LOG.trace(
            "Using full directory listing to update file information for {}. Consider enabling fastExistsCheck option.",
            file);
        // fast option not enabled, so list the directory and filter the file name
        files = operations.listFiles(file.getParent());
      }
      LOG.trace("List files {} found {} files", file.getAbsoluteFilePath(), files.size());
      for (FTPFile f : files) {
        if (f.getName().equals(file.getFileNameOnly())) {
          newLength = f.getSize();
          if (f.getTimestamp() != null) {
            newLastModified = f.getTimestamp().getTimeInMillis();
          }
        }
      }

      LOG.trace(
          "Previous last modified: " + lastModified + ", new last modified: " + newLastModified);
      LOG.trace("Previous length: " + length + ", new length: " + newLength);
      long newOlderThan = startTime + watch.taken() - minAge;
      LOG.trace("New older than threshold: {}", newOlderThan);

      if (newLength >= minLength
          && ((minAge == 0 && newLastModified == lastModified && newLength == length)
              || (minAge != 0 && newLastModified < newOlderThan))) {
        LOG.trace("Read lock acquired.");
        exclusive = true;
      } else {
        // set new base file change information
        lastModified = newLastModified;
        length = newLength;

        boolean interrupted = sleep();
        if (interrupted) {
          // we were interrupted while sleeping, we are likely being shutdown so return false
          return false;
        }
      }
    }

    return exclusive;
  }