Ejemplo n.º 1
0
  /**
   * If <code>f</code> is a file, this method will make a single call to S3. If <code>f</code> is a
   * directory, this method will make a maximum of (<i>n</i> / 1000) + 2 calls to S3, where <i>n</i>
   * is the total number of files and directories contained directly in <code>f</code>.
   */
  @Override
  public FileStatus[] listStatus(Path f) throws IOException {

    Path absolutePath = makeAbsolute(f);
    String key = pathToKey(absolutePath);

    if (key.length() > 0) {
      FileMetadata meta = store.retrieveMetadata(key);
      if (meta != null) {
        return new FileStatus[] {newFile(meta, absolutePath)};
      }
    }

    URI pathUri = absolutePath.toUri();
    Set<FileStatus> status = new TreeSet<FileStatus>();
    String priorLastKey = null;
    do {
      PartialListing listing = store.list(key, S3_MAX_LISTING_LENGTH, priorLastKey, false);
      for (FileMetadata fileMetadata : listing.getFiles()) {
        Path subpath = keyToPath(fileMetadata.getKey());
        String relativePath = pathUri.relativize(subpath.toUri()).getPath();

        if (fileMetadata.getKey().equals(key + "/")) {
          // this is just the directory we have been asked to list
        } else if (relativePath.endsWith(FOLDER_SUFFIX)) {
          status.add(
              newDirectory(
                  new Path(
                      absolutePath,
                      relativePath.substring(0, relativePath.indexOf(FOLDER_SUFFIX)))));
        } else {
          status.add(newFile(fileMetadata, subpath));
        }
      }
      for (String commonPrefix : listing.getCommonPrefixes()) {
        Path subpath = keyToPath(commonPrefix);
        String relativePath = pathUri.relativize(subpath.toUri()).getPath();
        status.add(newDirectory(new Path(absolutePath, relativePath)));
      }
      priorLastKey = listing.getPriorLastKey();
    } while (priorLastKey != null);

    if (status.isEmpty()
        && key.length() > 0
        && store.retrieveMetadata(key + FOLDER_SUFFIX) == null) {
      throw new FileNotFoundException("File " + f + " does not exist.");
    }

    return status.toArray(new FileStatus[status.size()]);
  }
Ejemplo n.º 2
0
  @Override
  public FileStatus getFileStatus(Path f) throws IOException {
    Path absolutePath = makeAbsolute(f);
    String key = pathToKey(absolutePath);

    if (key.length() == 0) { // root always exists
      return newDirectory(absolutePath);
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("getFileStatus retrieving metadata for key '" + key + "'");
    }
    FileMetadata meta = store.retrieveMetadata(key);
    if (meta != null) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("getFileStatus returning 'file' for key '" + key + "'");
      }
      return newFile(meta, absolutePath);
    }
    if (store.retrieveMetadata(key + FOLDER_SUFFIX) != null) {
      if (LOG.isDebugEnabled()) {
        LOG.debug(
            "getFileStatus returning 'directory' for key '"
                + key
                + "' as '"
                + key
                + FOLDER_SUFFIX
                + "' exists");
      }
      return newDirectory(absolutePath);
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("getFileStatus listing key '" + key + "'");
    }
    PartialListing listing = store.list(key, 1);
    if (listing.getFiles().length > 0 || listing.getCommonPrefixes().length > 0) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("getFileStatus returning 'directory' for key '" + key + "' as it has contents");
      }
      return newDirectory(absolutePath);
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("getFileStatus could not find key '" + key + "'");
    }
    throw new FileNotFoundException("No such file or directory '" + absolutePath + "'");
  }
Ejemplo n.º 3
0
  @Override
  public boolean delete(Path f, boolean recurse) throws IOException {
    FileStatus status;
    try {
      status = getFileStatus(f);
    } catch (FileNotFoundException e) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Delete called for '" + f + "' but file does not exist, so returning false");
      }
      return false;
    }
    Path absolutePath = makeAbsolute(f);
    String key = pathToKey(absolutePath);
    if (status.isDirectory()) {
      if (!recurse && listStatus(f).length > 0) {
        throw new IOException(
            "Can not delete " + f + " at is a not empty directory and recurse option is false");
      }

      createParent(f);

      if (LOG.isDebugEnabled()) {
        LOG.debug("Deleting directory '" + f + "'");
      }
      String priorLastKey = null;
      do {
        PartialListing listing = store.list(key, S3_MAX_LISTING_LENGTH, priorLastKey, true);
        for (FileMetadata file : listing.getFiles()) {
          store.delete(file.getKey());
        }
        priorLastKey = listing.getPriorLastKey();
      } while (priorLastKey != null);

      try {
        store.delete(key + FOLDER_SUFFIX);
      } catch (FileNotFoundException e) {
        // this is fine, we don't require a marker
      }
    } else {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Deleting file '" + f + "'");
      }
      createParent(f);
      store.delete(key);
    }
    return true;
  }
Ejemplo n.º 4
0
  @Override
  public boolean rename(Path src, Path dst) throws IOException {

    String srcKey = pathToKey(makeAbsolute(src));

    if (srcKey.length() == 0) {
      // Cannot rename root of file system
      return false;
    }

    final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";

    // Figure out the final destination
    String dstKey;
    try {
      boolean dstIsFile = getFileStatus(dst).isFile();
      if (dstIsFile) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(debugPreamble + "returning false as dst is an already existing file");
        }
        return false;
      } else {
        if (LOG.isDebugEnabled()) {
          LOG.debug(debugPreamble + "using dst as output directory");
        }
        dstKey = pathToKey(makeAbsolute(new Path(dst, src.getName())));
      }
    } catch (FileNotFoundException e) {
      if (LOG.isDebugEnabled()) {
        LOG.debug(debugPreamble + "using dst as output destination");
      }
      dstKey = pathToKey(makeAbsolute(dst));
      try {
        if (getFileStatus(dst.getParent()).isFile()) {
          if (LOG.isDebugEnabled()) {
            LOG.debug(debugPreamble + "returning false as dst parent exists and is a file");
          }
          return false;
        }
      } catch (FileNotFoundException ex) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(debugPreamble + "returning false as dst parent does not exist");
        }
        return false;
      }
    }

    boolean srcIsFile;
    try {
      srcIsFile = getFileStatus(src).isFile();
    } catch (FileNotFoundException e) {
      if (LOG.isDebugEnabled()) {
        LOG.debug(debugPreamble + "returning false as src does not exist");
      }
      return false;
    }
    if (srcIsFile) {
      if (LOG.isDebugEnabled()) {
        LOG.debug(debugPreamble + "src is file, so doing copy then delete in S3");
      }
      store.copy(srcKey, dstKey);
      store.delete(srcKey);
    } else {
      if (LOG.isDebugEnabled()) {
        LOG.debug(debugPreamble + "src is directory, so copying contents");
      }
      store.storeEmptyFile(dstKey + FOLDER_SUFFIX);

      List<String> keysToDelete = new ArrayList<String>();
      String priorLastKey = null;
      do {
        PartialListing listing = store.list(srcKey, S3_MAX_LISTING_LENGTH, priorLastKey, true);
        for (FileMetadata file : listing.getFiles()) {
          keysToDelete.add(file.getKey());
          store.copy(file.getKey(), dstKey + file.getKey().substring(srcKey.length()));
        }
        priorLastKey = listing.getPriorLastKey();
      } while (priorLastKey != null);

      if (LOG.isDebugEnabled()) {
        LOG.debug(debugPreamble + "all files in src copied, now removing src files");
      }
      for (String key : keysToDelete) {
        store.delete(key);
      }

      try {
        store.delete(srcKey + FOLDER_SUFFIX);
      } catch (FileNotFoundException e) {
        // this is fine, we don't require a marker
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug(debugPreamble + "done");
      }
    }

    return true;
  }