예제 #1
0
 /**
  * Determines if the key represents a folder. If false is returned, it is not guaranteed that the
  * path exists.
  *
  * @param key the key to check
  * @return whether the given key identifies a folder
  */
 private boolean isFolder(String key) {
   // Root is always a folder
   if (isRoot(key)) {
     return true;
   }
   try {
     String keyAsFolder = convertToFolderName(stripPrefixIfPresent(key));
     mClient.getObjectDetails(mBucketName, keyAsFolder);
     // If no exception is thrown, the key exists as a folder
     return true;
   } catch (ServiceException s) {
     // It is possible that the folder has not been encoded as a _$folder$ file
     try {
       String dir = stripPrefixIfPresent(key);
       String dirPrefix = dir.endsWith(PATH_SEPARATOR) ? dir : dir + PATH_SEPARATOR;
       // Check if anything begins with <folder_path>/
       S3Object[] objs = mClient.listObjects(mBucketName, dirPrefix, "");
       // If there are, this is a folder and we can create the necessary metadata
       if (objs.length > 0) {
         mkdirsInternal(dir);
         return true;
       } else {
         return false;
       }
     } catch (ServiceException s2) {
       return false;
     }
   }
 }
예제 #2
0
 /**
  * Lists the files in the given path, the paths will be their logical names and not contain the
  * folder suffix.
  *
  * @param path the key to list
  * @param recursive if true will list children directories as well
  * @return an array of the file and folder names in this directory
  * @throws IOException if an I/O error occurs
  */
 private String[] listInternal(String path, boolean recursive) throws IOException {
   try {
     path = stripPrefixIfPresent(path);
     path = PathUtils.normalizePath(path, PATH_SEPARATOR);
     path = path.equals(PATH_SEPARATOR) ? "" : path;
     // Gets all the objects under the path, because we have no idea if there are non Alluxio
     // managed "directories"
     S3Object[] objs = mClient.listObjects(mBucketName, path, "");
     if (recursive) {
       List<String> ret = new ArrayList<>();
       for (S3Object obj : objs) {
         // Remove parent portion of the key
         String child = getChildName(obj.getKey(), path);
         // Prune the special folder suffix
         child = stripFolderSuffixIfPresent(child);
         // Only add if the path is not empty (removes results equal to the path)
         if (!child.isEmpty()) {
           ret.add(child);
         }
       }
       return ret.toArray(new String[ret.size()]);
     }
     // Non recursive list
     Set<String> children = new HashSet<String>();
     for (S3Object obj : objs) {
       // Remove parent portion of the key
       String child = getChildName(obj.getKey(), path);
       // Remove any portion after the path delimiter
       int childNameIndex = child.indexOf(PATH_SEPARATOR);
       child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child;
       // Prune the special folder suffix
       child = stripFolderSuffixIfPresent(child);
       // Add to the set of children, the set will deduplicate.
       if (!child.isEmpty()) {
         children.add(child);
       }
     }
     return children.toArray(new String[children.size()]);
   } catch (ServiceException e) {
     LOG.error("Failed to list path {}", path, e);
     return null;
   }
 }
예제 #3
0
  public static void main(String[] args) {

    Storage st = new Storage(args[0], args[1], args[2]);

    try {
      String awsAccessKey = "AKIAIXDL4AVDSSMH4JJQ";
      String awsSecretKey = "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";
      AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey);

      S3Service s3Service = new RestS3Service(awsCredentials);
      S3Object list[] = s3Service.listObjects("pagerankmam");
      for (int i = 0; i < list.length; i++) {
        if (list[i].getName().startsWith("input/output/")) {
          // Open the file that is the first
          // command line parameter

          // System.out.println(list[i].getName());
          S3Object objectComplete =
              s3Service.getObject("pagerankmam/input/output", list[i].getName().split("/")[2]);
          // System.out.println("S3Object, complete: " + objectComplete);

          // Read the data from the object's DataInputStream using a loop, and print it out.
          // System.out.println("Greeting:");
          BufferedReader reader =
              new BufferedReader(new InputStreamReader(objectComplete.getDataInputStream()));
          String data = null;
          st.connect();
          while ((data = reader.readLine()) != null) {
            String[] results = data.split("\t");
            st.insertPageRankRow(
                Integer.parseInt(results[0]), Double.parseDouble(results[1].split(" ")[0]));
            // System.out.println(Integer.parseInt(results[0])+" -
            // "+Double.parseDouble(results[1].split(" ")[0]));
            // System.out.println(data);
          }
          st.close();
        }
      }
    } catch (Exception e) { // Catch exception if any
      System.err.println("Error: " + e.getMessage());
    }
  }
예제 #4
0
 /**
  * Lists the files in the given path, the paths will be their logical names and not contain the
  * folder suffix
  *
  * @param path the key to list
  * @param recursive if true will list children directories as well
  * @return an array of the file and folder names in this directory
  * @throws IOException
  */
 private String[] listInternal(String path, boolean recursive) throws IOException {
   try {
     path = stripPrefixIfPresent(path);
     path = path.endsWith(PATH_SEPARATOR) ? path : path + PATH_SEPARATOR;
     path = path.equals(PATH_SEPARATOR) ? "" : path;
     // Gets all the objects under the path, because we have no idea if there are non Tachyon
     // managed "directories"
     S3Object[] objs = mClient.listObjects(mBucketName, path, "");
     if (recursive) {
       String[] ret = new String[objs.length];
       for (int i = 0; i < objs.length; i++) {
         // Remove parent portion of the key
         String child = getChildName(objs[i].getKey(), path);
         // Prune the special folder suffix
         child = stripFolderSuffixIfPresent(child);
         ret[i] = child;
       }
       return ret;
     }
     // Non recursive list
     Set<String> children = new HashSet<String>();
     for (S3Object obj : objs) {
       // Remove parent portion of the key
       String child = getChildName(obj.getKey(), path);
       // Remove any portion after the path delimiter
       int childNameIndex = child.indexOf(PATH_SEPARATOR);
       child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child;
       // Prune the special folder suffix
       child = stripFolderSuffixIfPresent(child);
       // Add to the set of children, the set will deduplicate.
       children.add(child);
     }
     return children.toArray(new String[children.size()]);
   } catch (ServiceException se) {
     LOG.error("Failed to list path " + path);
     return null;
   }
 }
예제 #5
0
  /*
   * Fetch a file that is in a S3 file system. Return a local File. It accepts "s3://" and "s3n://" prefixes.
   */
  private File s3Fetch(URI uri, Reporter reporter) throws IOException {
    String bucketName = uri.getHost();
    String path = uri.getPath();

    File destFolder = new File(tempDir, bucketName + "/" + path);
    if (destFolder.exists()) {
      FileUtils.deleteDirectory(destFolder);
    }
    destFolder.mkdirs();

    Throttler throttler = new Throttler((double) bytesPerSecThrottle);

    boolean done = false;
    try {
      s3Service = new RestS3Service(getCredentials());
      if (s3Service.checkBucketStatus(bucketName) != RestS3Service.BUCKET_STATUS__MY_BUCKET) {
        throw new IOException("Bucket doesn't exist or is already claimed: " + bucketName);
      }

      if (path.startsWith("/")) {
        path = path.substring(1, path.length());
      }

      for (S3Object object : s3Service.listObjects(new S3Bucket(bucketName), path, "")) {
        long bytesSoFar = 0;

        String fileName = path;
        if (path.contains("/")) {
          fileName = path.substring(path.lastIndexOf("/") + 1, path.length());
        }
        File fileDest = new File(destFolder, fileName);
        log.info("Downloading " + object.getKey() + " to " + fileDest + " ...");

        if (fileDest.exists()) {
          fileDest.delete();
        }

        object = s3Service.getObject(new S3Bucket(bucketName), object.getKey());
        InputStream iS = object.getDataInputStream();
        FileOutputStream writer = new FileOutputStream(fileDest);
        byte[] buffer = new byte[downloadBufferSize];

        int nRead;
        while ((nRead = iS.read(buffer, 0, buffer.length)) != -1) {
          bytesSoFar += nRead;
          writer.write(buffer, 0, nRead);
          throttler.incrementAndThrottle(nRead);
          if (bytesSoFar >= bytesToReportProgress) {
            reporter.progress(bytesSoFar);
            bytesSoFar = 0l;
          }
        }

        if (reporter != null) {
          reporter.progress(bytesSoFar);
        }

        writer.close();
        iS.close();
        done = true;
      }

      if (!done) {
        throw new IOException("Bucket is empty! " + bucketName + " path: " + path);
      }
    } catch (S3ServiceException e) {
      throw new IOException(e);
    }

    return destFolder;
  }