@ResponseBody
  @RequestMapping(value = "/admin/upload.ajax", method = RequestMethod.POST)
  public ResponseEntity<byte[]> upload(
      @RequestParam("restaurantId") String restaurantId,
      @RequestParam("file") CommonsMultipartFile file)
      throws Exception {

    Map<String, Object> model = new HashMap<String, Object>();

    try {
      S3Object object = new S3Object(basePath + "/" + restaurantId);
      object.setDataInputStream(file.getInputStream());
      object.setContentLength(file.getSize());
      object.setContentType(file.getContentType());
      S3Bucket bucket = s3Service.getBucket(bucketName);
      s3Service.putObject(bucket, object);

      Restaurant restaurant = restaurantRepository.findByRestaurantId(restaurantId);
      restaurant.setHasUploadedImage(true);
      restaurantRepository.saveRestaurant(restaurant);

      model.put("success", true);
    } catch (Exception ex) {
      LOGGER.error("", ex);
      model.put("success", false);
      model.put("message", ex.getMessage());
    }

    String json = jsonUtils.serializeAndEscape(model);
    final HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.TEXT_HTML);
    headers.setCacheControl("no-cache");
    return new ResponseEntity<byte[]>(json.getBytes("utf-8"), headers, HttpStatus.OK);
  }
예제 #2
0
 /**
  * Determines if the key represents a folder. If false is returned, it is not guaranteed that the
  * path exists.
  *
  * @param key the key to check
  * @return whether the given key identifies a folder
  */
 private boolean isFolder(String key) {
   // Root is always a folder
   if (isRoot(key)) {
     return true;
   }
   try {
     String keyAsFolder = convertToFolderName(stripPrefixIfPresent(key));
     mClient.getObjectDetails(mBucketName, keyAsFolder);
     // If no exception is thrown, the key exists as a folder
     return true;
   } catch (ServiceException s) {
     // It is possible that the folder has not been encoded as a _$folder$ file
     try {
       String dir = stripPrefixIfPresent(key);
       String dirPrefix = dir.endsWith(PATH_SEPARATOR) ? dir : dir + PATH_SEPARATOR;
       // Check if anything begins with <folder_path>/
       S3Object[] objs = mClient.listObjects(mBucketName, dirPrefix, "");
       // If there are, this is a folder and we can create the necessary metadata
       if (objs.length > 0) {
         mkdirsInternal(dir);
         return true;
       } else {
         return false;
       }
     } catch (ServiceException s2) {
       return false;
     }
   }
 }
예제 #3
0
 /**
  * @param key the key to get the object details of
  * @return {@link StorageObject} of the key, or null if the key does not exist
  */
 private StorageObject getObjectDetails(String key) {
   try {
     if (isFolder(key)) {
       String keyAsFolder = convertToFolderName(stripPrefixIfPresent(key));
       return mClient.getObjectDetails(mBucketName, keyAsFolder);
     } else {
       return mClient.getObjectDetails(mBucketName, stripPrefixIfPresent(key));
     }
   } catch (ServiceException e) {
     return null;
   }
 }
예제 #4
0
 /**
  * Internal function to delete a key in S3.
  *
  * @param key the key to delete
  * @return true if successful, false if an exception is thrown
  */
 private boolean deleteInternal(String key) {
   try {
     if (isFolder(key)) {
       String keyAsFolder = convertToFolderName(stripPrefixIfPresent(key));
       mClient.deleteObject(mBucketName, keyAsFolder);
     } else {
       mClient.deleteObject(mBucketName, stripPrefixIfPresent(key));
     }
   } catch (ServiceException e) {
     LOG.error("Failed to delete {}", key, e);
     return false;
   }
   return true;
 }
 @Override
 protected boolean putInputStream(String bucket, String key, InputStream data, String contentType)
     throws Exception {
   org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(key);
   object.setContentType(contentType);
   object.setDataInputStream(data);
   object.setContentLength(data.available());
   return jetClient.putObject(bucket, object) != null;
 }
예제 #6
0
 private S3Object get(String key) {
   try {
     return s3Service.getObject(bucket, key);
   } catch (S3ServiceException e) {
     if ("NoSuchKey".equals(e.getS3ErrorCode())) {
       return null;
     }
   }
   return null;
 }
예제 #7
0
  public static void main(String[] args) {

    Storage st = new Storage(args[0], args[1], args[2]);

    try {
      String awsAccessKey = "AKIAIXDL4AVDSSMH4JJQ";
      String awsSecretKey = "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";
      AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey);

      S3Service s3Service = new RestS3Service(awsCredentials);
      S3Object list[] = s3Service.listObjects("pagerankmam");
      for (int i = 0; i < list.length; i++) {
        if (list[i].getName().startsWith("input/output/")) {
          // Open the file that is the first
          // command line parameter

          // System.out.println(list[i].getName());
          S3Object objectComplete =
              s3Service.getObject("pagerankmam/input/output", list[i].getName().split("/")[2]);
          // System.out.println("S3Object, complete: " + objectComplete);

          // Read the data from the object's DataInputStream using a loop, and print it out.
          // System.out.println("Greeting:");
          BufferedReader reader =
              new BufferedReader(new InputStreamReader(objectComplete.getDataInputStream()));
          String data = null;
          st.connect();
          while ((data = reader.readLine()) != null) {
            String[] results = data.split("\t");
            st.insertPageRankRow(
                Integer.parseInt(results[0]), Double.parseDouble(results[1].split(" ")[0]));
            // System.out.println(Integer.parseInt(results[0])+" -
            // "+Double.parseDouble(results[1].split(" ")[0]));
            // System.out.println(data);
          }
          st.close();
        }
      }
    } catch (Exception e) { // Catch exception if any
      System.err.println("Error: " + e.getMessage());
    }
  }
예제 #8
0
 /**
  * Copies an object to another key.
  *
  * @param src the source key to copy
  * @param dst the destination key to copy to
  * @return true if the operation was successful, false otherwise
  */
 private boolean copy(String src, String dst) {
   try {
     src = stripPrefixIfPresent(src);
     dst = stripPrefixIfPresent(dst);
     LOG.info("Copying {} to {}", src, dst);
     S3Object obj = new S3Object(dst);
     mClient.copyObject(mBucketName, src, mBucketName, obj, false);
     return true;
   } catch (ServiceException e) {
     LOG.error("Failed to rename file {} to {}", src, dst, e);
     return false;
   }
 }
예제 #9
0
 /**
  * Creates a directory flagged file with the key and folder suffix.
  *
  * @param key the key to create a folder
  * @return true if the operation was successful, false otherwise
  */
 private boolean mkdirsInternal(String key) {
   try {
     String keyAsFolder = convertToFolderName(stripPrefixIfPresent(key));
     S3Object obj = new S3Object(keyAsFolder);
     obj.setDataInputStream(new ByteArrayInputStream(new byte[0]));
     obj.setContentLength(0);
     obj.setContentType(Mimetypes.MIMETYPE_BINARY_OCTET_STREAM);
     mClient.putObject(mBucketName, obj);
     return true;
   } catch (ServiceException se) {
     LOG.error("Failed to create directory: " + key, se);
     return false;
   }
 }
예제 #10
0
 /**
  * Determines if the key represents a folder. If false is returned, it is not guaranteed that the
  * path exists.
  *
  * @param key the key to check
  * @return S3Object containing metadata
  */
 private boolean isFolder(String key) {
   key = key.endsWith(PATH_SEPARATOR) ? key.substring(0, key.length() - 1) : key;
   // Root is always a folder
   if (isRoot(key)) {
     return true;
   }
   try {
     String keyAsFolder = convertToFolderName(stripPrefixIfPresent(key));
     mClient.getObjectDetails(mBucketName, keyAsFolder);
     // If no exception is thrown, the key exists as a folder
     return true;
   } catch (ServiceException se) {
     return false;
   }
 }
예제 #11
0
 /**
  * Lists the files in the given path, the paths will be their logical names and not contain the
  * folder suffix.
  *
  * @param path the key to list
  * @param recursive if true will list children directories as well
  * @return an array of the file and folder names in this directory
  * @throws IOException if an I/O error occurs
  */
 private String[] listInternal(String path, boolean recursive) throws IOException {
   try {
     path = stripPrefixIfPresent(path);
     path = PathUtils.normalizePath(path, PATH_SEPARATOR);
     path = path.equals(PATH_SEPARATOR) ? "" : path;
     // Gets all the objects under the path, because we have no idea if there are non Alluxio
     // managed "directories"
     S3Object[] objs = mClient.listObjects(mBucketName, path, "");
     if (recursive) {
       List<String> ret = new ArrayList<>();
       for (S3Object obj : objs) {
         // Remove parent portion of the key
         String child = getChildName(obj.getKey(), path);
         // Prune the special folder suffix
         child = stripFolderSuffixIfPresent(child);
         // Only add if the path is not empty (removes results equal to the path)
         if (!child.isEmpty()) {
           ret.add(child);
         }
       }
       return ret.toArray(new String[ret.size()]);
     }
     // Non recursive list
     Set<String> children = new HashSet<String>();
     for (S3Object obj : objs) {
       // Remove parent portion of the key
       String child = getChildName(obj.getKey(), path);
       // Remove any portion after the path delimiter
       int childNameIndex = child.indexOf(PATH_SEPARATOR);
       child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child;
       // Prune the special folder suffix
       child = stripFolderSuffixIfPresent(child);
       // Add to the set of children, the set will deduplicate.
       if (!child.isEmpty()) {
         children.add(child);
       }
     }
     return children.toArray(new String[children.size()]);
   } catch (ServiceException e) {
     LOG.error("Failed to list path {}", path, e);
     return null;
   }
 }
예제 #12
0
 /**
  * Lists the files in the given path, the paths will be their logical names and not contain the
  * folder suffix
  *
  * @param path the key to list
  * @param recursive if true will list children directories as well
  * @return an array of the file and folder names in this directory
  * @throws IOException
  */
 private String[] listInternal(String path, boolean recursive) throws IOException {
   try {
     path = stripPrefixIfPresent(path);
     path = path.endsWith(PATH_SEPARATOR) ? path : path + PATH_SEPARATOR;
     path = path.equals(PATH_SEPARATOR) ? "" : path;
     // Gets all the objects under the path, because we have no idea if there are non Tachyon
     // managed "directories"
     S3Object[] objs = mClient.listObjects(mBucketName, path, "");
     if (recursive) {
       String[] ret = new String[objs.length];
       for (int i = 0; i < objs.length; i++) {
         // Remove parent portion of the key
         String child = getChildName(objs[i].getKey(), path);
         // Prune the special folder suffix
         child = stripFolderSuffixIfPresent(child);
         ret[i] = child;
       }
       return ret;
     }
     // Non recursive list
     Set<String> children = new HashSet<String>();
     for (S3Object obj : objs) {
       // Remove parent portion of the key
       String child = getChildName(obj.getKey(), path);
       // Remove any portion after the path delimiter
       int childNameIndex = child.indexOf(PATH_SEPARATOR);
       child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child;
       // Prune the special folder suffix
       child = stripFolderSuffixIfPresent(child);
       // Add to the set of children, the set will deduplicate.
       children.add(child);
     }
     return children.toArray(new String[children.size()]);
   } catch (ServiceException se) {
     LOG.error("Failed to list path " + path);
     return null;
   }
 }
예제 #13
0
  /*
   * Fetch a file that is in a S3 file system. Return a local File. It accepts "s3://" and "s3n://" prefixes.
   */
  private File s3Fetch(URI uri, Reporter reporter) throws IOException {
    String bucketName = uri.getHost();
    String path = uri.getPath();

    File destFolder = new File(tempDir, bucketName + "/" + path);
    if (destFolder.exists()) {
      FileUtils.deleteDirectory(destFolder);
    }
    destFolder.mkdirs();

    Throttler throttler = new Throttler((double) bytesPerSecThrottle);

    boolean done = false;
    try {
      s3Service = new RestS3Service(getCredentials());
      if (s3Service.checkBucketStatus(bucketName) != RestS3Service.BUCKET_STATUS__MY_BUCKET) {
        throw new IOException("Bucket doesn't exist or is already claimed: " + bucketName);
      }

      if (path.startsWith("/")) {
        path = path.substring(1, path.length());
      }

      for (S3Object object : s3Service.listObjects(new S3Bucket(bucketName), path, "")) {
        long bytesSoFar = 0;

        String fileName = path;
        if (path.contains("/")) {
          fileName = path.substring(path.lastIndexOf("/") + 1, path.length());
        }
        File fileDest = new File(destFolder, fileName);
        log.info("Downloading " + object.getKey() + " to " + fileDest + " ...");

        if (fileDest.exists()) {
          fileDest.delete();
        }

        object = s3Service.getObject(new S3Bucket(bucketName), object.getKey());
        InputStream iS = object.getDataInputStream();
        FileOutputStream writer = new FileOutputStream(fileDest);
        byte[] buffer = new byte[downloadBufferSize];

        int nRead;
        while ((nRead = iS.read(buffer, 0, buffer.length)) != -1) {
          bytesSoFar += nRead;
          writer.write(buffer, 0, nRead);
          throttler.incrementAndThrottle(nRead);
          if (bytesSoFar >= bytesToReportProgress) {
            reporter.progress(bytesSoFar);
            bytesSoFar = 0l;
          }
        }

        if (reporter != null) {
          reporter.progress(bytesSoFar);
        }

        writer.close();
        iS.close();
        done = true;
      }

      if (!done) {
        throw new IOException("Bucket is empty! " + bucketName + " path: " + path);
      }
    } catch (S3ServiceException e) {
      throw new IOException(e);
    }

    return destFolder;
  }
  private void downloadThrows(final S3Artifact s3Artifact, final Path downloadTo) throws Exception {
    log.info("Downloading {}", s3Artifact);

    Jets3tProperties jets3tProperties =
        Jets3tProperties.getInstance(Constants.JETS3T_PROPERTIES_FILENAME);
    jets3tProperties.setProperty(
        "httpclient.socket-timeout-ms",
        Long.toString(configuration.getS3ChunkDownloadTimeoutMillis()));

    final S3Service s3 =
        new RestS3Service(
            getCredentialsForBucket(s3Artifact.getS3Bucket()), null, null, jets3tProperties);

    long length = 0;

    if (s3Artifact.getFilesize().isPresent()) {
      length = s3Artifact.getFilesize().get();
    } else {
      StorageObject details =
          s3.getObjectDetails(s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());

      Preconditions.checkNotNull(
          details,
          "Couldn't find object at %s/%s",
          s3Artifact.getS3Bucket(),
          s3Artifact.getS3ObjectKey());

      length = details.getContentLength();
    }

    int numChunks = (int) (length / configuration.getS3ChunkSize());

    if (length % configuration.getS3ChunkSize() > 0) {
      numChunks++;
    }

    final long chunkSize = length / numChunks + (length % numChunks);

    log.info(
        "Downloading {}/{} in {} chunks of {} bytes to {}",
        s3Artifact.getS3Bucket(),
        s3Artifact.getS3ObjectKey(),
        numChunks,
        chunkSize,
        downloadTo);

    final ExecutorService chunkExecutorService =
        Executors.newFixedThreadPool(
            numChunks,
            new ThreadFactoryBuilder()
                .setDaemon(true)
                .setNameFormat("S3ArtifactDownloaderChunkThread-%d")
                .build());
    final List<Future<Path>> futures = Lists.newArrayListWithCapacity(numChunks);

    for (int chunk = 0; chunk < numChunks; chunk++) {
      futures.add(
          chunkExecutorService.submit(
              new S3ArtifactChunkDownloader(
                  configuration,
                  log,
                  s3,
                  s3Artifact,
                  downloadTo,
                  chunk,
                  chunkSize,
                  length,
                  exceptionNotifier)));
    }

    long remainingMillis = configuration.getS3DownloadTimeoutMillis();
    boolean failed = false;

    for (int chunk = 0; chunk < numChunks; chunk++) {
      final Future<Path> future = futures.get(chunk);

      if (failed) {
        future.cancel(true);
        continue;
      }

      final long start = System.currentTimeMillis();

      if (!handleChunk(s3Artifact, future, downloadTo, chunk, start, remainingMillis)) {
        failed = true;
      }

      remainingMillis -= (System.currentTimeMillis() - start);
    }

    chunkExecutorService.shutdownNow();

    Preconditions.checkState(
        !failed, "Downloading %s/%s failed", s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());
  }