private File retrieveS3File(String sqsdMessageBody) throws UnsupportedEncodingException {
    File localFile = null;

    if (!sqsdMessageBody.isEmpty()) {

      AmazonS3 s3 = new AmazonS3Client();

      List<S3EventNotificationRecord> records =
          S3EventNotification.parseJson(sqsdMessageBody).getRecords();

      S3EventNotificationRecord firstRecord = records.get(0);

      String bucketName = firstRecord.getS3().getBucket().getName();

      String objectRegion = firstRecord.getAwsRegion();
      Region s3Region = Region.getRegion(Regions.fromName(objectRegion));
      s3.setRegion(s3Region);

      // Object key may have spaces or unicode non-ASCII characters.
      String keyName = firstRecord.getS3().getObject().getKey().replace('+', ' ');
      keyName = URLDecoder.decode(keyName, "UTF-8");

      localFile = new File(keyName);

      System.out.println("Downloading file: " + objectRegion + "/" + bucketName + "/" + keyName);
      s3.getObject(new GetObjectRequest(bucketName, keyName), localFile);

      if (!localFile.canRead()) {
        localFile = null;
      }
    }
    return localFile;
  }
 public InputStream downloadRange(long start, long end, String file) {
   GetObjectRequest rangeObjectRequest = new GetObjectRequest(bucketName, file);
   rangeObjectRequest.setRange(start, end); // retrieve 1st 10 bytes.
   S3Object objectPortion = s3client.getObject(rangeObjectRequest);
   InputStream objectData = objectPortion.getObjectContent();
   return objectData;
 }
 public UrlList getUrlList(String url) {
   UrlList urlList = null;
   try {
     String key = Hash.hashKey(url);
     GetObjectRequest req = new GetObjectRequest(bucketName, key);
     S3Object s3Object = s3client.getObject(req);
     InputStream objectData = s3Object.getObjectContent();
     BufferedReader reader = new BufferedReader(new InputStreamReader(objectData));
     StringBuilder s3Content = new StringBuilder();
     String line;
     while ((line = reader.readLine()) != null) {
       s3Content.append(line + "\r\n");
     }
     reader.close();
     objectData.close();
     ObjectMapper mapper = new ObjectMapper();
     mapper.setVisibility(PropertyAccessor.ALL, Visibility.NONE);
     mapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY);
     urlList = mapper.readValue(s3Content.toString(), UrlList.class);
   } catch (AmazonS3Exception ase) {
     System.out.println("S3UrlListDA : document does not exist");
     ase.printStackTrace();
   } catch (IOException e) {
     System.out.println("S3UrlListDA : IOException while fetching document from S3");
     e.printStackTrace();
   } catch (NoSuchAlgorithmException e) {
     // TODO Auto-generated catch block
     e.printStackTrace();
   }
   return urlList;
 }
Esempio n. 4
0
 private S3Object getS3Object(final Path path, final long start) throws IOException {
   try {
     return retry()
         .maxAttempts(maxClientRetry)
         .exponentialBackoff(
             new Duration(1, TimeUnit.SECONDS), maxBackoffTime, maxRetryTime, 2.0)
         .stopOn(InterruptedException.class, UnrecoverableS3OperationException.class)
         .run(
             "getS3Object",
             () -> {
               try {
                 return s3.getObject(
                     new GetObjectRequest(host, keyFromPath(path))
                         .withRange(start, Long.MAX_VALUE));
               } catch (AmazonServiceException e) {
                 if (e.getStatusCode() == SC_FORBIDDEN) {
                   throw new UnrecoverableS3OperationException(e);
                 }
                 throw Throwables.propagate(e);
               }
             });
   } catch (InterruptedException e) {
     Thread.currentThread().interrupt();
     throw Throwables.propagate(e);
   } catch (Exception e) {
     Throwables.propagateIfInstanceOf(e, IOException.class);
     throw Throwables.propagate(e);
   }
 }
Esempio n. 5
0
  private Note getNote(String key) throws IOException {
    GsonBuilder gsonBuilder = new GsonBuilder();
    gsonBuilder.setPrettyPrinting();
    Gson gson =
        gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create();

    S3Object s3object;
    try {
      s3object = s3client.getObject(new GetObjectRequest(bucketName, key));
    } catch (AmazonClientException ace) {
      throw new IOException("Unable to retrieve object from S3: " + ace, ace);
    }

    Note note;
    try (InputStream ins = s3object.getObjectContent()) {
      String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
      note = gson.fromJson(json, Note.class);
    }

    for (Paragraph p : note.getParagraphs()) {
      if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) {
        p.setStatus(Status.ABORT);
      }
    }

    return note;
  }
  /** Data cleansing method */
  public void cleanseData(AmazonS3 client) throws Exception {

    AwsDataLoader loader = new AwsDataLoader();
    CSVReader reader = null;
    String prefix = loader.getS3Prefix(source);
    client.setEndpoint(S3_ENDPOINT);
    S3Object object = client.getObject(new GetObjectRequest(BUCKET_NM, prefix));
    reader =
        new CSVReader(
            new BufferedReader(new InputStreamReader(object.getObjectContent())),
            CSVParser.DEFAULT_SEPARATOR,
            CSVParser.DEFAULT_QUOTE_CHARACTER,
            CSVParser.DEFAULT_ESCAPE_CHARACTER,
            HEADERS_LINE);
    ColumnPositionMappingStrategy<ProductLanguage> strat =
        new ColumnPositionMappingStrategy<ProductLanguage>();
    strat.setType(ProductLanguage.class);
    String[] columns =
        new String[] {"refId", "code", "name", "locale", "state", "format", "displayNameLanguage"};
    strat.setColumnMapping(columns);
    CsvToBean<ProductLanguage> csv = new CsvToBean<ProductLanguage>();
    list = csv.parse(strat, reader);

    System.out.println("ProductLanguageCleanser input size: " + list.size());

    this.updateDataset(list);

    BeanToCsv<ProductLanguage> csvWriter = new BeanToCsv<ProductLanguage>();
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    CSVWriter writer = new CSVWriter(new OutputStreamWriter(os), ',', '"');
    // strat.setColumnMapping(columns);
    log.info("List size: " + list.size());
    csvWriter.write(strat, writer, list);
    writer.flush();
    String dataset = os.toString();
    String outPrefix = PREFIX + OUTPUT_KEY + source + ".csv";

    client.setEndpoint(S3_ENDPOINT);
    ObjectMetadata omd = new ObjectMetadata();

    try {
      byte[] content = dataset.getBytes(StandardCharsets.UTF_8);
      ByteArrayInputStream input = new ByteArrayInputStream(content);
      BufferedReader buffReader = new BufferedReader(new InputStreamReader(input));
      buffReader.readLine();
      InputStream inputObj = new ReaderInputStream(buffReader);
      // omd.setContentLength(IOUtils.toByteArray(input).length);
      client.putObject(BUCKET_NM, outPrefix, inputObj, omd);
      input.close();
    } catch (IOException e) {
      log.error("Axon data write to s3 failed: " + e.getMessage());
    }
  }
Esempio n. 7
0
  @Override
  public InputStream getObject(String container, String object, Config config) {
    super.getObject(container, object, config);
    InputStream stream;
    try {

      S3Object s3Obj = client.getObject(container, object);
      stream = s3Obj.getObjectContent();

    } catch (Exception e) {
      throw new StorageException(e);
    }
    return stream;
  }
  public static String scaleImage(String bucket, String key) throws IOException {
    if (key.startsWith(PREFIX)) {
      System.out.println("Target image is already scaled");
      return "Nothing";
    }

    Optional<String> optionalImageType = getImageType(key);
    if (!optionalImageType.isPresent()) {
      return "";
    }
    String imageType = optionalImageType.get();

    // Download the image from S3 into a stream
    AmazonS3 s3Client = new AmazonS3Client();
    S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, key));
    InputStream objectData = s3Object.getObjectContent();

    // Read the source image
    BufferedImage srcImage = ImageIO.read(objectData);
    int srcHeight = srcImage.getHeight();
    int srcWidth = srcImage.getWidth();

    for (int scaleDimension : SCALE_DIMENSIONS) {
      // Infer the scaling factor to avoid stretching the image
      // unnaturally
      float scalingFactor =
          Math.min((float) scaleDimension / srcWidth, (float) scaleDimension / srcHeight);
      int width = (int) (scalingFactor * srcWidth);
      int height = (int) (scalingFactor * srcHeight);

      BufferedImage resizedImage =
          getHighQualityScaledInstance(
              srcImage, width, height, RenderingHints.VALUE_INTERPOLATION_BICUBIC);

      BufferedImage squaredImage =
          getSquaredImage(resizedImage, RenderingHints.VALUE_INTERPOLATION_BICUBIC);

      String dstKeyResized = PREFIX + "-" + scaleDimension + "-" + key;
      String dstKeyResizedSquared = PREFIX + "-" + scaleDimension + "-squared-" + key;

      saveToS3(bucket, key, imageType, s3Client, dstKeyResized, resizedImage);
      saveToS3(bucket, key, imageType, s3Client, dstKeyResizedSquared, squaredImage);
    }
    return "Ok";
  }
  public static void main(String[] args) {

    if (args.length < 2) {
      System.out.println("USAGE: localPath bucketname <bucketPrefix>");
      System.exit(1);
    }

    String localPath = args[0];
    String bucketName = args[1];
    String bucketPrefix = "";

    // add on extra slash
    if (!localPath.endsWith("/")) localPath += "/";

    if (args.length == 3) bucketPrefix = args[2];

    // check local dir, make if it does not exist
    File localDir = new File(localPath);
    if (!localDir.exists()) localDir.mkdirs();

    if (!localDir.isDirectory()) {
      System.out.println("Local Dir is not a dir: " + localPath);
      System.exit(1);
    }

    long totalBytes = 0;
    long start = System.currentTimeMillis();

    AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());

    ObjectListing listObjects = s3.listObjects(bucketName, bucketPrefix);
    do {
      for (S3ObjectSummary objectSummary : listObjects.getObjectSummaries()) {
        S3Object object = s3.getObject(bucketName, objectSummary.getKey());
        S3ObjectInputStream inputStream = object.getObjectContent();
        if ("gzip".equals(object.getObjectMetadata().getContentEncoding())) {
          InputStream in = null;
          try {

            totalBytes += object.getObjectMetadata().getContentLength();

            in = new GZIPInputStream(inputStream);
            // write this sucker out
            String path = localPath + object.getKey();
            // have to take the gz off since this is not downloading compressed!
            if (path.endsWith(".gz")) path = path.substring(0, path.length() - 3);
            System.out.print("Writing file: " + path);
            File check = new File(path);
            File parentFile = check.getParentFile();
            if (!parentFile.exists()) parentFile.mkdirs();

            FileOutputStream out = new FileOutputStream(path);
            IOUtils.copy(in, out);
            System.out.println(" written.");

          } catch (IOException e) {
            System.out.println("crap");
            e.printStackTrace();
            throw new IllegalStateException("files are too hard", e);
          } finally {
            IOUtils.closeQuietly(in);
          }
        } else {
          System.out.println(
              "unhandled content encoding: " + object.getObjectMetadata().getContentEncoding());
        }
      }

      listObjects = s3.listNextBatchOfObjects(listObjects);
    } while (listObjects.isTruncated());

    long now = System.currentTimeMillis();
    System.out.println(
        (totalBytes / 1000.0 / 1000.0)
            + " mb downloaded in "
            + ((now - start) / 1000)
            + " seconds.");
  }
 public long getInputFileSize() {
   S3Object object = s3client.getObject(new GetObjectRequest(bucketName, inputFileName));
   return object.getObjectMetadata().getContentLength();
 }
 public InputStream downloadFile(String fileName) {
   S3Object object = s3client.getObject(new GetObjectRequest(bucketName, fileName));
   return object.getObjectContent();
 }
Esempio n. 12
0
  public static void main(String[] args) throws IOException {
    /*
     * This credentials provider implementation loads your AWS credentials
     * from a properties file at the root of your classpath.
     *
     * Important: Be sure to fill in your AWS access credentials in the
     *            AwsCredentials.properties file before you try to run this
     *            sample.
     * http://aws.amazon.com/security-credentials
     */
    AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());

    String bucketName = "my-first-s3-bucket-" + UUID.randomUUID();
    String key = "MyObjectKey";

    System.out.println("===========================================");
    System.out.println("Getting Started with Amazon S3");
    System.out.println("===========================================\n");

    try {
      /*
       * Create a new S3 bucket - Amazon S3 bucket names are globally unique,
       * so once a bucket name has been taken by any user, you can't create
       * another bucket with that same name.
       *
       * You can optionally specify a location for your bucket if you want to
       * keep your data closer to your applications or users.
       */
      System.out.println("Creating bucket " + bucketName + "\n");
      s3.createBucket(bucketName);

      /*
       * List the buckets in your account
       */
      System.out.println("Listing buckets");
      for (Bucket bucket : s3.listBuckets()) {
        System.out.println(" - " + bucket.getName());
      }
      System.out.println();

      /*
       * Upload an object to your bucket - You can easily upload a file to
       * S3, or upload directly an InputStream if you know the length of
       * the data in the stream. You can also specify your own metadata
       * when uploading to S3, which allows you set a variety of options
       * like content-type and content-encoding, plus additional metadata
       * specific to your applications.
       */
      System.out.println("Uploading a new object to S3 from a file\n");
      s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile()));

      /*
       * Download an object - When you download an object, you get all of
       * the object's metadata and a stream from which to read the contents.
       * It's important to read the contents of the stream as quickly as
       * possibly since the data is streamed directly from Amazon S3 and your
       * network connection will remain open until you read all the data or
       * close the input stream.
       *
       * GetObjectRequest also supports several other options, including
       * conditional downloading of objects based on modification times,
       * ETags, and selectively downloading a range of an object.
       */
      System.out.println("Downloading an object");
      S3Object object = s3.getObject(new GetObjectRequest(bucketName, key));
      System.out.println("Content-Type: " + object.getObjectMetadata().getContentType());
      displayTextInputStream(object.getObjectContent());

      /*
       * List objects in your bucket by prefix - There are many options for
       * listing the objects in your bucket.  Keep in mind that buckets with
       * many objects might truncate their results when listing their objects,
       * so be sure to check if the returned object listing is truncated, and
       * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve
       * additional results.
       */
      System.out.println("Listing objects");
      ObjectListing objectListing =
          s3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My"));
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        System.out.println(
            " - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")");
      }
      System.out.println();

      /*
       * Delete an object - Unless versioning has been turned on for your bucket,
       * there is no way to undelete an object, so use caution when deleting objects.
       */
      System.out.println("Deleting an object\n");
      s3.deleteObject(bucketName, key);

      /*
       * Delete a bucket - A bucket must be completely empty before it can be
       * deleted, so remember to delete any objects from your buckets before
       * you try to delete them.
       */
      System.out.println("Deleting bucket " + bucketName + "\n");
      s3.deleteBucket(bucketName);
    } catch (AmazonServiceException ase) {
      System.out.println(
          "Caught an AmazonServiceException, which means your request made it "
              + "to Amazon S3, but was rejected with an error response for some reason.");
      System.out.println("Error Message:    " + ase.getMessage());
      System.out.println("HTTP Status Code: " + ase.getStatusCode());
      System.out.println("AWS Error Code:   " + ase.getErrorCode());
      System.out.println("Error Type:       " + ase.getErrorType());
      System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println(
          "Caught an AmazonClientException, which means the client encountered "
              + "a serious internal problem while trying to communicate with S3, "
              + "such as not being able to access the network.");
      System.out.println("Error Message: " + ace.getMessage());
    }
  }
Esempio n. 13
0
 // 从bucket中下载object name,并放在"/home/zforCeph/download"文件夹下
 public static void getObjectFromBucket(Bucket bucket, String path, String name) {
   conn.getObject(new GetObjectRequest(bucket.getName(), name), new File(path + name));
 }
  ////////////////////////////////////////////////////////////////////////////
  ///// private methods
  ////////////////////////////////////////////////////////////////////////////
  private InputStream loadEventFromS3(AmazonS3 s3Client, String bucketName, final String key)
      throws IOException {

    S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName, key));
    return s3Object.getObjectContent();
  }
Esempio n. 15
0
 /**
  * get a single map
  *
  * @param mapName
  */
 public void getMap(String mapName) {
   s3.getObject(new GetObjectRequest(bucketName, mapName), new File(mapName));
 }