예제 #1
0
파일: UploadToS3.java 프로젝트: jq/zhong
  public static boolean upload(MusicInfo music) {
    String key = "";
    try {
      // upload  mp3 ring
      key = music.getUUID() + music.getRingName();
      File file = new File(Consts.NEW_DOWNLOAD_DIR + music.getRingName());
      s3.putObject(new PutObjectRequest(Consts.AMAZON_RING_BUCKET, key, file)); // upload ring
      s3.setObjectAcl(
          Consts.AMAZON_RING_BUCKET, key, CannedAccessControlList.PublicRead); // set access

      // upload m4r ring
      //			String m4r = music.getRingName().replace(".mp3", ".m4r");
      //			key = music.getUUID() + m4r;
      //			file = new File(Consts.NEW_DOWNLOAD_DIR+m4r);
      //			s3.putObject(new PutObjectRequest(Consts.AMAZON_M4R_BUCKET, key, file));      			//
      // upload ring
      //			s3.setObjectAcl(Consts.AMAZON_M4R_BUCKET, key, CannedAccessControlList.PublicRead);		//
      // set access

      // upload image
      key = music.getUUID() + music.getImageName();
      file = new File(Consts.NEW_DOWNLOAD_DIR + music.getImageName());
      s3.putObject(new PutObjectRequest(Consts.AMAZON_IMAGE_BUCKET, key, file)); // upload image
      s3.setObjectAcl(
          Consts.AMAZON_IMAGE_BUCKET, key, CannedAccessControlList.PublicRead); // set access

      return true;
    } catch (Exception e) {
      System.out.println("upload to S3 err");
      e.printStackTrace();
      return false;
    }
  }
  private static void saveToS3(
      String bucket,
      String key,
      String imageType,
      AmazonS3 s3Client,
      String dstKey,
      BufferedImage resizedImage)
      throws IOException {
    // Re-encode image to target format
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    ImageIO.write(resizedImage, imageType, os);
    InputStream is = new ByteArrayInputStream(os.toByteArray());
    // Set Content-Length and Content-Type
    ObjectMetadata meta = new ObjectMetadata();
    meta.setContentLength(os.size());
    String contentType = imageTypeMime.get(imageType);
    if (contentType == null) {
      throw new RuntimeException(
          String.format("Unknown content type for image type %s", imageType));
    }
    meta.setContentType(contentType);

    // Uploading to S3 destination bucket
    System.out.println("Writing to: " + bucket + "/" + dstKey);
    s3Client.putObject(bucket, dstKey, is, meta);
    System.out.println(
        "Successfully resized " + bucket + "/" + key + " and uploaded to " + bucket + "/" + dstKey);
  }
예제 #3
0
  @Override
  public void saveFile(File file, String fileName) throws IOException {

    AmazonS3 s3Client = getAmzS3Client();

    s3Client.putObject(new PutObjectRequest(bucketName, fileName, file));

    file.delete();
  }
예제 #4
0
파일: S3Util.java 프로젝트: JialiDu/zork
  /**
   * Upload the input file to S3 bucket
   *
   * @param file : the file you want to upload
   * @param name : File name
   * @return if the upload is success, return true
   */
  public boolean S3MapUpload(File file, String name, String userName, boolean ifPublic) {

    System.out.println("Uploading a new object to S3 from a file\n");

    if (ifPublic) {
      PutObjectRequest por = new PutObjectRequest(bucketName, "maps/public/" + name, file);

      por.setCannedAcl(CannedAccessControlList.PublicRead);
      s3.putObject(por);
    }

    PutObjectRequest por = new PutObjectRequest(bucketName, "maps/" + userName + "/" + name, file);

    por.setCannedAcl(CannedAccessControlList.PublicRead);
    s3.putObject(por);

    return true;
  }
예제 #5
0
  /** Data cleansing method */
  public void cleanseData(AmazonS3 client) throws Exception {

    AwsDataLoader loader = new AwsDataLoader();
    CSVReader reader = null;
    String prefix = loader.getS3Prefix(source);
    client.setEndpoint(S3_ENDPOINT);
    S3Object object = client.getObject(new GetObjectRequest(BUCKET_NM, prefix));
    reader =
        new CSVReader(
            new BufferedReader(new InputStreamReader(object.getObjectContent())),
            CSVParser.DEFAULT_SEPARATOR,
            CSVParser.DEFAULT_QUOTE_CHARACTER,
            CSVParser.DEFAULT_ESCAPE_CHARACTER,
            HEADERS_LINE);
    ColumnPositionMappingStrategy<ProductLanguage> strat =
        new ColumnPositionMappingStrategy<ProductLanguage>();
    strat.setType(ProductLanguage.class);
    String[] columns =
        new String[] {"refId", "code", "name", "locale", "state", "format", "displayNameLanguage"};
    strat.setColumnMapping(columns);
    CsvToBean<ProductLanguage> csv = new CsvToBean<ProductLanguage>();
    list = csv.parse(strat, reader);

    System.out.println("ProductLanguageCleanser input size: " + list.size());

    this.updateDataset(list);

    BeanToCsv<ProductLanguage> csvWriter = new BeanToCsv<ProductLanguage>();
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    CSVWriter writer = new CSVWriter(new OutputStreamWriter(os), ',', '"');
    // strat.setColumnMapping(columns);
    log.info("List size: " + list.size());
    csvWriter.write(strat, writer, list);
    writer.flush();
    String dataset = os.toString();
    String outPrefix = PREFIX + OUTPUT_KEY + source + ".csv";

    client.setEndpoint(S3_ENDPOINT);
    ObjectMetadata omd = new ObjectMetadata();

    try {
      byte[] content = dataset.getBytes(StandardCharsets.UTF_8);
      ByteArrayInputStream input = new ByteArrayInputStream(content);
      BufferedReader buffReader = new BufferedReader(new InputStreamReader(input));
      buffReader.readLine();
      InputStream inputObj = new ReaderInputStream(buffReader);
      // omd.setContentLength(IOUtils.toByteArray(input).length);
      client.putObject(BUCKET_NM, outPrefix, inputObj, omd);
      input.close();
    } catch (IOException e) {
      log.error("Axon data write to s3 failed: " + e.getMessage());
    }
  }
  private void uploadToS3(File file, String bucketName, String key) {
    log.info("Uploading local WAR file {} to remote WAR file: {}", file.getName(), key);

    try {
      String s3key = URLEncoder.encode(key, "UTF-8");
      AmazonS3 s3 = new AmazonS3Client(getAwsCredentials());
      s3.putObject(bucketName, s3key, file);
    } catch (UnsupportedEncodingException ex) {
      log.error("Upload file to s3 failed.");
      log.error(ex.getMessage());
    }
  }
예제 #7
0
  @Override
  public void createObject(
      String container, String object, InputStream data, long length, Config config) {
    super.createObject(container, object, data, length, config);
    try {
      ObjectMetadata metadata = new ObjectMetadata();
      metadata.setContentLength(length);
      metadata.setContentType("application/octet-stream");

      client.putObject(container, object, data, metadata);
    } catch (Exception e) {
      throw new StorageException(e);
    }
  }
예제 #8
0
파일: Utils.java 프로젝트: oryband/homework
  // Makes a putObjectRequest to make file public, and sends request.
  private static boolean putObject(AmazonS3 s3, PutObjectRequest req) {
    // Set file as public.
    req.withCannedAcl(CannedAccessControlList.PublicRead);

    // Send upload request.
    try {
      s3.putObject(req);
    } catch (AmazonServiceException e) {
      logger.severe(e.getMessage());
      return false;
    } catch (AmazonClientException e) {
      logger.severe(e.getMessage());
      return false;
    }

    return true;
  }
 @Override
 public StoredObject putObject(URI location, File source) {
   try {
     log.info("starting upload: %s", location);
     PutObjectResult result =
         s3Service.putObject(getS3Bucket(location), getS3ObjectKey(location), source);
     ObjectMetadata metadata =
         s3Service.getObjectMetadata(getS3Bucket(location), getS3ObjectKey(location));
     if (!result.getETag().equals(metadata.getETag())) {
       // this might happen in rare cases due to S3's eventual consistency
       throw new IllegalStateException("uploaded etag is different from retrieved object etag");
     }
     log.info("completed upload: %s", location);
     return updateStoredObject(location, metadata);
   } catch (Exception e) {
     throw Throwables.propagate(e);
   }
 }
예제 #10
0
    @Override
    public void run() {
      String fileName = null;
      try {
        // Init Amazon S3
        AmazonS3 s3 =
            new AmazonS3Client(
                new BasicAWSCredentials(
                    "AKIAI42VQL6SUU5S27SA", "/P3g0/U/hnKgWKRMsRXtG1w+gM+H0dwtLPnuvOBG"));
        String bucketName = "diaosi-mapreduce";

        // Download and Extract
        fileName = s3Path.substring(s3Path.indexOf("enwiki-20121001-"));
        fileName = fileName.substring(0, fileName.indexOf("bz2")) + ".xml";
        FileOutputStream fout = new FileOutputStream(fileName);

        URL url = new URL(s3Path);
        URLConnection conn = url.openConnection();
        conn.setConnectTimeout(20000);
        conn.setReadTimeout(20000);
        InputStream is = conn.getInputStream();
        is.read();
        is.read();
        CBZip2InputStream fin = new CBZip2InputStream(is);
        while (true) {
          int len = fin.read(buf, 0, buf.length);
          System.out.println(fileName + ": " + len);
          if (len == -1) break;
          fout.write(buf, 0, len);
        }
        fout.flush();
        fout.close();
        fin.close();

        // Upload to S3
        File file = new File(fileName);
        String key = fileName;
        s3.putObject(new PutObjectRequest(bucketName, key, file));

      } catch (Exception e) {
        e.printStackTrace();
      }
    }
    public void storeMessageInS3(String message) {
      AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());

      String bucketName = "mailAppTestBucket_1";
      String key = "message_" + UUID.randomUUID();
      // System.out.println("Uploading a new object to S3 from a file\n");
      try {
        s3.putObject(new PutObjectRequest(bucketName, key, createMessageFile(message)));
      } catch (AmazonServiceException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } catch (AmazonClientException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
    }
예제 #12
0
  public void putUrlList(UrlList urlList) {
    try {
      // hash url to get key
      String s3Key = Hash.hashKey(urlList.getParentUrl());

      // put document into s3
      ObjectMapper mapper = new ObjectMapper();
      mapper.setVisibility(PropertyAccessor.ALL, Visibility.NONE);
      mapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY);
      ObjectWriter ow = mapper.writer().withDefaultPrettyPrinter();
      String json = ow.writeValueAsString(urlList);
      ByteArrayInputStream inputStream = new ByteArrayInputStream(json.getBytes());
      ObjectMetadata omd = new ObjectMetadata();
      omd.setContentLength(json.getBytes().length);
      PutObjectRequest request = new PutObjectRequest(bucketName, s3Key, inputStream, omd);
      s3client.putObject(request);

      // No need to put into dynamo as it should already be there

      /*
      ObjectMapper mapper = new ObjectMapper();
      mapper.setVisibility(PropertyAccessor.ALL, Visibility.NONE);
      mapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY);
      ObjectWriter ow = mapper.writer().withDefaultPrettyPrinter();
      String json = ow.writeValueAsString(doc);
      ObjectMetadata omd = new ObjectMetadata();
      omd.setContentType("text/html");
      omd.setContentLength(json.length());
      ByteArrayInputStream inputStream = new ByteArrayInputStream(
      		json.getBytes());
      PutObjectRequest request = new PutObjectRequest(bucketName, s3Key,
      		inputStream, omd);
      s3client.putObject(request);
      dynamo.putItem("documentUrl", doc.getDocumentId(), "s3Key", s3Key);*/
    } catch (JsonProcessingException e) {
      System.out.println("S3UrlListDA : json processing excecption exception");
      e.printStackTrace();
    } catch (NoSuchAlgorithmException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
  }
예제 #13
0
  @Override
  public void save(Note note, AuthenticationInfo subject) throws IOException {
    GsonBuilder gsonBuilder = new GsonBuilder();
    gsonBuilder.setPrettyPrinting();
    Gson gson = gsonBuilder.create();
    String json = gson.toJson(note);
    String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json";

    File file = File.createTempFile("note", "json");
    try {
      Writer writer = new OutputStreamWriter(new FileOutputStream(file));
      writer.write(json);
      writer.close();
      s3client.putObject(new PutObjectRequest(bucketName, key, file));
    } catch (AmazonClientException ace) {
      throw new IOException("Unable to store note in S3: " + ace, ace);
    } finally {
      FileUtils.deleteQuietly(file);
    }
  }
예제 #14
0
 @Override
 public void upload(
     final String key,
     final String md5,
     final String contentType,
     final InputStream body,
     final long contentLength)
     throws Exception {
   final ObjectMetadata metadata = new ObjectMetadata();
   metadata.setContentLength(contentLength);
   metadata.setContentType(contentType);
   if (md5.equalsIgnoreCase(etags.get(key))) {
     System.out.println("skipping:" + key);
     return;
   }
   System.out.println("uploading:" + key);
   s3.putObject(
       new PutObjectRequest(bucket, key, body, metadata)
           .withCannedAcl(CannedAccessControlList.PublicRead));
 }
예제 #15
0
  // 将文件f放到bucket中,名为name; 权限为公共读写。
  public static void putObjectInBucket(Bucket bucket, String name, File f) {

    conn.putObject(bucket.getName(), name, f);
    conn.setObjectAcl(bucket.getName(), name, CannedAccessControlList.PublicReadWrite);
  }
예제 #16
0
  private void jButton1ActionPerformed(
      java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jButton1ActionPerformed
    // TODO add your handling code here: upload file
    JFileChooser jfc = new JFileChooser();
    jfc.setAcceptAllFileFilterUsed(false);
    jfc.setFileFilter(new FileNameExtensionFilter("Text Files", "txt"));
    int returnVal = jfc.showOpenDialog(this);
    if (returnVal == JFileChooser.APPROVE_OPTION) {
      File file = jfc.getSelectedFile();
      try {
        filepath = file.getCanonicalPath();
        FileReader fr = new FileReader(filepath);

        BufferedReader br = new BufferedReader(fr);
        String line;
        int total_no_of_words = 0;

        while ((line = br.readLine()) != null) {

          String[] words_arr = line.split(" ");
          total_no_of_words += words_arr.length;
        }
        jLabel4.setText("word count :" + total_no_of_words);
        System.out.println("Total no of words = " + total_no_of_words);
        if (total_no_of_words != 0) {
          File userFile = new File(filepath);
          keyName = userFile.getName();
          String myAccessKeyID = "AKIAI7PATIX6LHJA3A3Q";
          String mySecretKey = "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";
          AWSCredentials myCredentials = new BasicAWSCredentials(myAccessKeyID, mySecretKey);
          ClientConfiguration clientConfig = new ClientConfiguration();
          clientConfig.setProtocol(Protocol.HTTP);
          AmazonS3 conn = new AmazonS3Client(myCredentials, clientConfig);
          try {

            File file1 = new File(filepath);
            file_size = file1.length();
            if (file_size < 100000) {
              conn.putObject(new PutObjectRequest(bucketName, keyName, file1));
              jLabel2.setText("Uploading complete !");
            } else {
              jLabel2.setText("Uploading failed size greater than 100kb...");
            }
          } catch (AmazonServiceException ase) {
            System.out.println(
                "Caught an AmazonServiceException, which "
                    + "means your request made it "
                    + "to Amazon S3, but was rejected with an error response"
                    + " for some reason.");
            System.out.println("Error Message:    " + ase.getMessage());
            System.out.println("HTTP Status Code: " + ase.getStatusCode());
            System.out.println("AWS Error Code:   " + ase.getErrorCode());
            System.out.println("Error Type:       " + ase.getErrorType());
            System.out.println("Request ID:       " + ase.getRequestId());
          } catch (AmazonClientException ace) {
            System.out.println(
                "Caught an AmazonClientException, which "
                    + "means the client encountered "
                    + "an internal error while trying to "
                    + "communicate with S3, "
                    + "such as not being able to access the network.");
            System.out.println("Error Message: " + ace.getMessage());
          } /*catch (InterruptedException ex) {
                Logger.getLogger(Upload.class.getName()).log(Level.SEVERE, null, ex);
            }*/
        } else {
          jLabel2.setText("Uploading failed file empty !");
        }

      } catch (IOException ex) {
        Logger.getLogger(Upload.class.getName()).log(Level.SEVERE, null, ex);
      }
    }
    if (returnVal == JFileChooser.CANCEL_OPTION) {
      System.exit(0);
    }
  } // GEN-LAST:event_jButton1ActionPerformed
 public void uploadFile(String s, String file) throws UnsupportedEncodingException {
   InputStream inStream = new ByteArrayInputStream(s.getBytes("UTF-8"));
   s3client.deleteObject(bucketName, MasterAddrFileName);
   s3client.putObject(new PutObjectRequest(bucketName, file, inStream, new ObjectMetadata()));
 }
예제 #18
0
  public static void main(String[] args) throws IOException {
    /*
     * This credentials provider implementation loads your AWS credentials
     * from a properties file at the root of your classpath.
     *
     * Important: Be sure to fill in your AWS access credentials in the
     *            AwsCredentials.properties file before you try to run this
     *            sample.
     * http://aws.amazon.com/security-credentials
     */
    AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());

    String bucketName = "my-first-s3-bucket-" + UUID.randomUUID();
    String key = "MyObjectKey";

    System.out.println("===========================================");
    System.out.println("Getting Started with Amazon S3");
    System.out.println("===========================================\n");

    try {
      /*
       * Create a new S3 bucket - Amazon S3 bucket names are globally unique,
       * so once a bucket name has been taken by any user, you can't create
       * another bucket with that same name.
       *
       * You can optionally specify a location for your bucket if you want to
       * keep your data closer to your applications or users.
       */
      System.out.println("Creating bucket " + bucketName + "\n");
      s3.createBucket(bucketName);

      /*
       * List the buckets in your account
       */
      System.out.println("Listing buckets");
      for (Bucket bucket : s3.listBuckets()) {
        System.out.println(" - " + bucket.getName());
      }
      System.out.println();

      /*
       * Upload an object to your bucket - You can easily upload a file to
       * S3, or upload directly an InputStream if you know the length of
       * the data in the stream. You can also specify your own metadata
       * when uploading to S3, which allows you set a variety of options
       * like content-type and content-encoding, plus additional metadata
       * specific to your applications.
       */
      System.out.println("Uploading a new object to S3 from a file\n");
      s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile()));

      /*
       * Download an object - When you download an object, you get all of
       * the object's metadata and a stream from which to read the contents.
       * It's important to read the contents of the stream as quickly as
       * possibly since the data is streamed directly from Amazon S3 and your
       * network connection will remain open until you read all the data or
       * close the input stream.
       *
       * GetObjectRequest also supports several other options, including
       * conditional downloading of objects based on modification times,
       * ETags, and selectively downloading a range of an object.
       */
      System.out.println("Downloading an object");
      S3Object object = s3.getObject(new GetObjectRequest(bucketName, key));
      System.out.println("Content-Type: " + object.getObjectMetadata().getContentType());
      displayTextInputStream(object.getObjectContent());

      /*
       * List objects in your bucket by prefix - There are many options for
       * listing the objects in your bucket.  Keep in mind that buckets with
       * many objects might truncate their results when listing their objects,
       * so be sure to check if the returned object listing is truncated, and
       * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve
       * additional results.
       */
      System.out.println("Listing objects");
      ObjectListing objectListing =
          s3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My"));
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        System.out.println(
            " - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")");
      }
      System.out.println();

      /*
       * Delete an object - Unless versioning has been turned on for your bucket,
       * there is no way to undelete an object, so use caution when deleting objects.
       */
      System.out.println("Deleting an object\n");
      s3.deleteObject(bucketName, key);

      /*
       * Delete a bucket - A bucket must be completely empty before it can be
       * deleted, so remember to delete any objects from your buckets before
       * you try to delete them.
       */
      System.out.println("Deleting bucket " + bucketName + "\n");
      s3.deleteBucket(bucketName);
    } catch (AmazonServiceException ase) {
      System.out.println(
          "Caught an AmazonServiceException, which means your request made it "
              + "to Amazon S3, but was rejected with an error response for some reason.");
      System.out.println("Error Message:    " + ase.getMessage());
      System.out.println("HTTP Status Code: " + ase.getStatusCode());
      System.out.println("AWS Error Code:   " + ase.getErrorCode());
      System.out.println("Error Type:       " + ase.getErrorType());
      System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println(
          "Caught an AmazonClientException, which means the client encountered "
              + "a serious internal problem while trying to communicate with S3, "
              + "such as not being able to access the network.");
      System.out.println("Error Message: " + ace.getMessage());
    }
  }
 public void storeFile(final String fileName, final File file) {
   s3client.putObject(new PutObjectRequest(S3_BUCKET, fileName, file));
 }