コード例 #1
0
  /** Data cleansing method */
  public void cleanseData(AmazonS3 client) throws Exception {

    AwsDataLoader loader = new AwsDataLoader();
    CSVReader reader = null;
    String prefix = loader.getS3Prefix(source);
    client.setEndpoint(S3_ENDPOINT);
    S3Object object = client.getObject(new GetObjectRequest(BUCKET_NM, prefix));
    reader =
        new CSVReader(
            new BufferedReader(new InputStreamReader(object.getObjectContent())),
            CSVParser.DEFAULT_SEPARATOR,
            CSVParser.DEFAULT_QUOTE_CHARACTER,
            CSVParser.DEFAULT_ESCAPE_CHARACTER,
            HEADERS_LINE);
    ColumnPositionMappingStrategy<ProductLanguage> strat =
        new ColumnPositionMappingStrategy<ProductLanguage>();
    strat.setType(ProductLanguage.class);
    String[] columns =
        new String[] {"refId", "code", "name", "locale", "state", "format", "displayNameLanguage"};
    strat.setColumnMapping(columns);
    CsvToBean<ProductLanguage> csv = new CsvToBean<ProductLanguage>();
    list = csv.parse(strat, reader);

    System.out.println("ProductLanguageCleanser input size: " + list.size());

    this.updateDataset(list);

    BeanToCsv<ProductLanguage> csvWriter = new BeanToCsv<ProductLanguage>();
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    CSVWriter writer = new CSVWriter(new OutputStreamWriter(os), ',', '"');
    // strat.setColumnMapping(columns);
    log.info("List size: " + list.size());
    csvWriter.write(strat, writer, list);
    writer.flush();
    String dataset = os.toString();
    String outPrefix = PREFIX + OUTPUT_KEY + source + ".csv";

    client.setEndpoint(S3_ENDPOINT);
    ObjectMetadata omd = new ObjectMetadata();

    try {
      byte[] content = dataset.getBytes(StandardCharsets.UTF_8);
      ByteArrayInputStream input = new ByteArrayInputStream(content);
      BufferedReader buffReader = new BufferedReader(new InputStreamReader(input));
      buffReader.readLine();
      InputStream inputObj = new ReaderInputStream(buffReader);
      // omd.setContentLength(IOUtils.toByteArray(input).length);
      client.putObject(BUCKET_NM, outPrefix, inputObj, omd);
      input.close();
    } catch (IOException e) {
      log.error("Axon data write to s3 failed: " + e.getMessage());
    }
  }
コード例 #2
0
ファイル: ceph.java プロジェクト: XiaoqingWang/MovieSite
 // 初始化
 public static AmazonS3 init() {
   String accessKey = "R3949EV1BHELWIZXKGJR";
   String secretKey = "fmimw6JPDApNC5r+iYknX7JXsvBu8DON3J24Sk1A";
   AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
   conn = new AmazonS3Client(credentials);
   conn.setEndpoint("http://192.168.0.19");
   return conn;
 }
コード例 #3
0
ファイル: S3Storage.java プロジェクト: carriercomm/cosbench
  @Override
  public void init(Config config, Logger logger) {
    super.init(config, logger);

    timeout = config.getInt(CONN_TIMEOUT_KEY, CONN_TIMEOUT_DEFAULT);

    parms.put(CONN_TIMEOUT_KEY, timeout);

    endpoint = config.get(ENDPOINT_KEY, ENDPOINT_DEFAULT);
    accessKey = config.get(AUTH_USERNAME_KEY, AUTH_USERNAME_DEFAULT);
    secretKey = config.get(AUTH_PASSWORD_KEY, AUTH_PASSWORD_DEFAULT);

    boolean pathStyleAccess = config.getBoolean(PATH_STYLE_ACCESS_KEY, PATH_STYLE_ACCESS_DEFAULT);

    String proxyHost = config.get(PROXY_HOST_KEY, "");
    String proxyPort = config.get(PROXY_PORT_KEY, "");

    parms.put(ENDPOINT_KEY, endpoint);
    parms.put(AUTH_USERNAME_KEY, accessKey);
    parms.put(AUTH_PASSWORD_KEY, secretKey);
    parms.put(PATH_STYLE_ACCESS_KEY, pathStyleAccess);
    parms.put(PROXY_HOST_KEY, proxyHost);
    parms.put(PROXY_PORT_KEY, proxyPort);

    logger.debug("using storage config: {}", parms);

    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setConnectionTimeout(timeout);
    clientConf.setSocketTimeout(timeout);
    clientConf.withUseExpectContinue(false);
    clientConf.withSignerOverride("S3SignerType");
    //        clientConf.setProtocol(Protocol.HTTP);
    if ((!proxyHost.equals("")) && (!proxyPort.equals(""))) {
      clientConf.setProxyHost(proxyHost);
      clientConf.setProxyPort(Integer.parseInt(proxyPort));
    }

    AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey);
    client = new AmazonS3Client(myCredentials, clientConf);
    client.setEndpoint(endpoint);
    client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(pathStyleAccess));

    logger.debug("S3 client has been initialized");
  }
コード例 #4
0
  public static String xs3_generate_url(String xs3_objname, String content_type) {
    AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
    ClientConfiguration xs3_clientconfig = new ClientConfiguration();
    xs3_clientconfig.setProtocol(Protocol.HTTP);

    S3ClientOptions xs3_client_options = new S3ClientOptions();
    xs3_client_options.setPathStyleAccess(true);

    xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
    xs3_client.setEndpoint(xs3_endpoint);
    xs3_client.setS3ClientOptions(xs3_client_options);

    try {
      java.util.Date expiration = new java.util.Date();

      long milliSeconds = expiration.getTime();
      milliSeconds += 1000 * 60 * 5;
      expiration.setTime(milliSeconds);

      GeneratePresignedUrlRequest xs3_genurl_req =
          new GeneratePresignedUrlRequest(xs3_bucketname, xs3_objname);
      xs3_genurl_req.setMethod(HttpMethod.PUT);
      xs3_genurl_req.setExpiration(expiration);
      xs3_genurl_req.setContentType(content_type);
      xs3_genurl_req.addRequestParameter("x-amz-acl", "public-read");

      URL url = xs3_client.generatePresignedUrl(xs3_genurl_req);

      System.out.println(url.toString());
      return url.toString();

    } catch (AmazonServiceException ase) {
      System.out.println("xs3_svr_error_message:" + ase.getMessage());
      System.out.println("xs3_svr_status_code:  " + ase.getStatusCode());
      System.out.println("xs3_svr_error_code:   " + ase.getErrorCode());
      System.out.println("xs3_svr_error_type:   " + ase.getErrorType());
      System.out.println("xs3_svr_request_id:   " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println("xs3_clt_error_message:" + ace.getMessage());
    }

    return null;
  }
コード例 #5
0
  public static String xs3_init_multi_upload(String xs3_objname, int file_size, String file_type) {
    AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
    ClientConfiguration xs3_clientconfig = new ClientConfiguration();
    xs3_clientconfig.setProtocol(Protocol.HTTP);

    S3ClientOptions xs3_client_options = new S3ClientOptions();
    xs3_client_options.setPathStyleAccess(true);

    xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
    xs3_client.setEndpoint(xs3_endpoint);
    xs3_client.setS3ClientOptions(xs3_client_options);

    try {
      InitiateMultipartUploadRequest xs3_multi_req =
          new InitiateMultipartUploadRequest(xs3_bucketname, xs3_objname);
      xs3_multi_req.setCannedACL(CannedAccessControlList.PublicRead);
      ObjectMetadata xs3_meta = new ObjectMetadata();
      xs3_meta.setContentType(file_type);
      xs3_multi_req.setObjectMetadata(xs3_meta);

      InitiateMultipartUploadResult xs3_multi_res =
          xs3_client.initiateMultipartUpload(xs3_multi_req);

      String xs3_multi_uploadid = xs3_multi_res.getUploadId();

      String json_urls = gen_part_url(xs3_multi_uploadid, file_size, xs3_objname, file_type);
      return json_urls;

    } catch (AmazonServiceException ase) {
      System.out.println("xs3_svr_error_message:" + ase.getMessage());
      System.out.println("xs3_svr_status_code:  " + ase.getStatusCode());
      System.out.println("xs3_svr_error_code:   " + ase.getErrorCode());
      System.out.println("xs3_svr_error_type:   " + ase.getErrorType());
      System.out.println("xs3_svr_request_id:   " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println("xs3_clt_error_message:" + ace.getMessage());
    }
    return null;
  }
コード例 #6
0
  public static void xs3_coplete_multi_upload(String xs3_objname, String uploadId) {
    AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
    ClientConfiguration xs3_clientconfig = new ClientConfiguration();
    xs3_clientconfig.setProtocol(Protocol.HTTP);

    S3ClientOptions xs3_client_options = new S3ClientOptions();
    xs3_client_options.setPathStyleAccess(true);

    xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
    xs3_client.setEndpoint(xs3_endpoint);
    xs3_client.setS3ClientOptions(xs3_client_options);

    try {
      List<PartETag> rest_parts = listPartsXml(xs3_objname, uploadId);
      if (null == rest_parts) {
        return;
      }
      for (PartETag item : rest_parts) {
        System.out.println(item.getETag() + " -> " + item.getPartNumber());
      }

      CompleteMultipartUploadRequest comp_req =
          new CompleteMultipartUploadRequest(xs3_bucketname, xs3_objname, uploadId, rest_parts);

      CompleteMultipartUploadResult comp_result = xs3_client.completeMultipartUpload(comp_req);

      System.out.println(comp_result.getETag());
      System.out.println(comp_result.getKey());
    } catch (AmazonServiceException ase) {
      System.out.println("xs3_svr_error_message:" + ase.getMessage());
      System.out.println("xs3_svr_status_code:  " + ase.getStatusCode());
      System.out.println("xs3_svr_error_code:   " + ase.getErrorCode());
      System.out.println("xs3_svr_error_type:   " + ase.getErrorType());
      System.out.println("xs3_svr_request_id:   " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println("xs3_clt_error_message:" + ace.getMessage());
    }
  }
コード例 #7
0
ファイル: S3NotebookRepo.java プロジェクト: conker84/zeppelin
  public S3NotebookRepo(ZeppelinConfiguration conf) throws IOException {
    this.conf = conf;
    bucketName = conf.getBucketName();
    user = conf.getUser();

    // always use the default provider chain
    AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
    CryptoConfiguration cryptoConf = null;
    String keyRegion = conf.getS3KMSKeyRegion();

    if (StringUtils.isNotBlank(keyRegion)) {
      cryptoConf = new CryptoConfiguration();
      cryptoConf.setAwsKmsRegion(Region.getRegion(Regions.fromName(keyRegion)));
    }

    // see if we should be encrypting data in S3
    String kmsKeyID = conf.getS3KMSKeyID();
    if (kmsKeyID != null) {
      // use the AWS KMS to encrypt data
      KMSEncryptionMaterialsProvider emp = new KMSEncryptionMaterialsProvider(kmsKeyID);
      if (cryptoConf != null) {
        this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cryptoConf);
      } else {
        this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
      }
    } else if (conf.getS3EncryptionMaterialsProviderClass() != null) {
      // use a custom encryption materials provider class
      EncryptionMaterialsProvider emp = createCustomProvider(conf);
      this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
    } else {
      // regular S3
      this.s3client = new AmazonS3Client(credentialsProvider);
    }

    // set S3 endpoint to use
    s3client.setEndpoint(conf.getEndpoint());
  }
コード例 #8
0
  public static String gen_part_url(
      String uploadId, int file_size, String file_name, String file_type) {
    AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
    ClientConfiguration xs3_clientconfig = new ClientConfiguration();
    xs3_clientconfig.setProtocol(Protocol.HTTP);

    S3ClientOptions xs3_client_options = new S3ClientOptions();
    xs3_client_options.setPathStyleAccess(true);

    xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
    xs3_client.setEndpoint(xs3_endpoint);
    xs3_client.setS3ClientOptions(xs3_client_options);

    try {

      final int xs3_part_size = 1024 * 1024 * 5;

      int xs3_part_count = (int) Math.ceil((double) (file_size) / (double) xs3_part_size);
      JSONArray jsonArray = new JSONArray();
      JSONObject jsonObject_1 = new JSONObject();
      jsonObject_1.put("total_num", xs3_part_count);
      jsonObject_1.put("upload_id", uploadId);

      JSONArray jsonArray_sub = new JSONArray();
      for (int part_no = 0; part_no < xs3_part_count; part_no++) {

        long xs3_offset_bytes = xs3_part_size * part_no;

        long part_size =
            xs3_part_size < (file_size - xs3_offset_bytes)
                ? xs3_part_size
                : (file_size - xs3_offset_bytes);

        java.util.Date expiration = new java.util.Date();
        long milliSeconds = expiration.getTime();
        milliSeconds += 1000 * 60 * 5;
        expiration.setTime(milliSeconds);

        GeneratePresignedUrlRequest xs3_genurl_req =
            new GeneratePresignedUrlRequest(xs3_bucketname, file_name);
        xs3_genurl_req.setMethod(HttpMethod.PUT);
        xs3_genurl_req.setExpiration(expiration);
        xs3_genurl_req.setContentType(file_type);
        xs3_genurl_req.addRequestParameter("uploadId", uploadId);
        xs3_genurl_req.addRequestParameter("partNumber", String.valueOf(part_no + 1));

        URL url = xs3_client.generatePresignedUrl(xs3_genurl_req);
        System.out.println(url.toString());

        JSONObject jsonObject = new JSONObject();
        jsonObject.put("part_idx", part_no);
        jsonObject.put("part_url", url.toString());
        jsonObject.put("upload_len", part_size);
        jsonObject.put("part_begin", xs3_offset_bytes);
        jsonObject.put("part_end", xs3_offset_bytes + part_size);

        jsonArray_sub.add(jsonObject);
      }

      jsonObject_1.put("multi_list", jsonArray_sub);
      jsonArray.add(jsonObject_1);

      return jsonArray.toString();

    } catch (AmazonServiceException ase) {
      System.out.println("xs3_svr_error_message:" + ase.getMessage());
      System.out.println("xs3_svr_status_code:  " + ase.getStatusCode());
      System.out.println("xs3_svr_error_code:   " + ase.getErrorCode());
      System.out.println("xs3_svr_error_type:   " + ase.getErrorType());
      System.out.println("xs3_svr_request_id:   " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println("xs3_clt_error_message:" + ace.getMessage());
    }

    return null;
  }
コード例 #9
0
ファイル: App.java プロジェクト: Shahbaz03/java-amazonS3
 /**
  * The method builds the Amazon S3 client using the initialized Aws_Credentials and the Amazon S3
  * endpoint. The Amazon S3 client is used for all the interaction with Amazon S3 storage system
  *
  * @return AmazonS3 client
  */
 private static AmazonS3 getS3Client() {
   AmazonS3 s3Client = new AmazonS3Client(AWS_CREDENTIALS);
   s3Client.setEndpoint("AMAZON_S3_ENDPOINT");
   return s3Client;
 }