/** Data cleansing method */
public void cleanseData(AmazonS3 client) throws Exception {
AwsDataLoader loader = new AwsDataLoader();
CSVReader reader = null;
String prefix = loader.getS3Prefix(source);
client.setEndpoint(S3_ENDPOINT);
S3Object object = client.getObject(new GetObjectRequest(BUCKET_NM, prefix));
reader =
new CSVReader(
new BufferedReader(new InputStreamReader(object.getObjectContent())),
CSVParser.DEFAULT_SEPARATOR,
CSVParser.DEFAULT_QUOTE_CHARACTER,
CSVParser.DEFAULT_ESCAPE_CHARACTER,
HEADERS_LINE);
ColumnPositionMappingStrategy<ProductLanguage> strat =
new ColumnPositionMappingStrategy<ProductLanguage>();
strat.setType(ProductLanguage.class);
String[] columns =
new String[] {"refId", "code", "name", "locale", "state", "format", "displayNameLanguage"};
strat.setColumnMapping(columns);
CsvToBean<ProductLanguage> csv = new CsvToBean<ProductLanguage>();
list = csv.parse(strat, reader);
System.out.println("ProductLanguageCleanser input size: " + list.size());
this.updateDataset(list);
BeanToCsv<ProductLanguage> csvWriter = new BeanToCsv<ProductLanguage>();
ByteArrayOutputStream os = new ByteArrayOutputStream();
CSVWriter writer = new CSVWriter(new OutputStreamWriter(os), ',', '"');
// strat.setColumnMapping(columns);
log.info("List size: " + list.size());
csvWriter.write(strat, writer, list);
writer.flush();
String dataset = os.toString();
String outPrefix = PREFIX + OUTPUT_KEY + source + ".csv";
client.setEndpoint(S3_ENDPOINT);
ObjectMetadata omd = new ObjectMetadata();
try {
byte[] content = dataset.getBytes(StandardCharsets.UTF_8);
ByteArrayInputStream input = new ByteArrayInputStream(content);
BufferedReader buffReader = new BufferedReader(new InputStreamReader(input));
buffReader.readLine();
InputStream inputObj = new ReaderInputStream(buffReader);
// omd.setContentLength(IOUtils.toByteArray(input).length);
client.putObject(BUCKET_NM, outPrefix, inputObj, omd);
input.close();
} catch (IOException e) {
log.error("Axon data write to s3 failed: " + e.getMessage());
}
}
// 初始化
public static AmazonS3 init() {
String accessKey = "R3949EV1BHELWIZXKGJR";
String secretKey = "fmimw6JPDApNC5r+iYknX7JXsvBu8DON3J24Sk1A";
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
conn = new AmazonS3Client(credentials);
conn.setEndpoint("http://192.168.0.19");
return conn;
}
@Override
public void init(Config config, Logger logger) {
super.init(config, logger);
timeout = config.getInt(CONN_TIMEOUT_KEY, CONN_TIMEOUT_DEFAULT);
parms.put(CONN_TIMEOUT_KEY, timeout);
endpoint = config.get(ENDPOINT_KEY, ENDPOINT_DEFAULT);
accessKey = config.get(AUTH_USERNAME_KEY, AUTH_USERNAME_DEFAULT);
secretKey = config.get(AUTH_PASSWORD_KEY, AUTH_PASSWORD_DEFAULT);
boolean pathStyleAccess = config.getBoolean(PATH_STYLE_ACCESS_KEY, PATH_STYLE_ACCESS_DEFAULT);
String proxyHost = config.get(PROXY_HOST_KEY, "");
String proxyPort = config.get(PROXY_PORT_KEY, "");
parms.put(ENDPOINT_KEY, endpoint);
parms.put(AUTH_USERNAME_KEY, accessKey);
parms.put(AUTH_PASSWORD_KEY, secretKey);
parms.put(PATH_STYLE_ACCESS_KEY, pathStyleAccess);
parms.put(PROXY_HOST_KEY, proxyHost);
parms.put(PROXY_PORT_KEY, proxyPort);
logger.debug("using storage config: {}", parms);
ClientConfiguration clientConf = new ClientConfiguration();
clientConf.setConnectionTimeout(timeout);
clientConf.setSocketTimeout(timeout);
clientConf.withUseExpectContinue(false);
clientConf.withSignerOverride("S3SignerType");
// clientConf.setProtocol(Protocol.HTTP);
if ((!proxyHost.equals("")) && (!proxyPort.equals(""))) {
clientConf.setProxyHost(proxyHost);
clientConf.setProxyPort(Integer.parseInt(proxyPort));
}
AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey);
client = new AmazonS3Client(myCredentials, clientConf);
client.setEndpoint(endpoint);
client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(pathStyleAccess));
logger.debug("S3 client has been initialized");
}
public static String xs3_generate_url(String xs3_objname, String content_type) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
java.util.Date expiration = new java.util.Date();
long milliSeconds = expiration.getTime();
milliSeconds += 1000 * 60 * 5;
expiration.setTime(milliSeconds);
GeneratePresignedUrlRequest xs3_genurl_req =
new GeneratePresignedUrlRequest(xs3_bucketname, xs3_objname);
xs3_genurl_req.setMethod(HttpMethod.PUT);
xs3_genurl_req.setExpiration(expiration);
xs3_genurl_req.setContentType(content_type);
xs3_genurl_req.addRequestParameter("x-amz-acl", "public-read");
URL url = xs3_client.generatePresignedUrl(xs3_genurl_req);
System.out.println(url.toString());
return url.toString();
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
return null;
}
public static String xs3_init_multi_upload(String xs3_objname, int file_size, String file_type) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
InitiateMultipartUploadRequest xs3_multi_req =
new InitiateMultipartUploadRequest(xs3_bucketname, xs3_objname);
xs3_multi_req.setCannedACL(CannedAccessControlList.PublicRead);
ObjectMetadata xs3_meta = new ObjectMetadata();
xs3_meta.setContentType(file_type);
xs3_multi_req.setObjectMetadata(xs3_meta);
InitiateMultipartUploadResult xs3_multi_res =
xs3_client.initiateMultipartUpload(xs3_multi_req);
String xs3_multi_uploadid = xs3_multi_res.getUploadId();
String json_urls = gen_part_url(xs3_multi_uploadid, file_size, xs3_objname, file_type);
return json_urls;
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
return null;
}
public static void xs3_coplete_multi_upload(String xs3_objname, String uploadId) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
List<PartETag> rest_parts = listPartsXml(xs3_objname, uploadId);
if (null == rest_parts) {
return;
}
for (PartETag item : rest_parts) {
System.out.println(item.getETag() + " -> " + item.getPartNumber());
}
CompleteMultipartUploadRequest comp_req =
new CompleteMultipartUploadRequest(xs3_bucketname, xs3_objname, uploadId, rest_parts);
CompleteMultipartUploadResult comp_result = xs3_client.completeMultipartUpload(comp_req);
System.out.println(comp_result.getETag());
System.out.println(comp_result.getKey());
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
}
public S3NotebookRepo(ZeppelinConfiguration conf) throws IOException {
this.conf = conf;
bucketName = conf.getBucketName();
user = conf.getUser();
// always use the default provider chain
AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
CryptoConfiguration cryptoConf = null;
String keyRegion = conf.getS3KMSKeyRegion();
if (StringUtils.isNotBlank(keyRegion)) {
cryptoConf = new CryptoConfiguration();
cryptoConf.setAwsKmsRegion(Region.getRegion(Regions.fromName(keyRegion)));
}
// see if we should be encrypting data in S3
String kmsKeyID = conf.getS3KMSKeyID();
if (kmsKeyID != null) {
// use the AWS KMS to encrypt data
KMSEncryptionMaterialsProvider emp = new KMSEncryptionMaterialsProvider(kmsKeyID);
if (cryptoConf != null) {
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cryptoConf);
} else {
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
}
} else if (conf.getS3EncryptionMaterialsProviderClass() != null) {
// use a custom encryption materials provider class
EncryptionMaterialsProvider emp = createCustomProvider(conf);
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
} else {
// regular S3
this.s3client = new AmazonS3Client(credentialsProvider);
}
// set S3 endpoint to use
s3client.setEndpoint(conf.getEndpoint());
}
public static String gen_part_url(
String uploadId, int file_size, String file_name, String file_type) {
AWSCredentials xs3_credentials = new BasicAWSCredentials(xs3_access_key, xs3_secret_key);
ClientConfiguration xs3_clientconfig = new ClientConfiguration();
xs3_clientconfig.setProtocol(Protocol.HTTP);
S3ClientOptions xs3_client_options = new S3ClientOptions();
xs3_client_options.setPathStyleAccess(true);
xs3_client = new AmazonS3Client(xs3_credentials, xs3_clientconfig);
xs3_client.setEndpoint(xs3_endpoint);
xs3_client.setS3ClientOptions(xs3_client_options);
try {
final int xs3_part_size = 1024 * 1024 * 5;
int xs3_part_count = (int) Math.ceil((double) (file_size) / (double) xs3_part_size);
JSONArray jsonArray = new JSONArray();
JSONObject jsonObject_1 = new JSONObject();
jsonObject_1.put("total_num", xs3_part_count);
jsonObject_1.put("upload_id", uploadId);
JSONArray jsonArray_sub = new JSONArray();
for (int part_no = 0; part_no < xs3_part_count; part_no++) {
long xs3_offset_bytes = xs3_part_size * part_no;
long part_size =
xs3_part_size < (file_size - xs3_offset_bytes)
? xs3_part_size
: (file_size - xs3_offset_bytes);
java.util.Date expiration = new java.util.Date();
long milliSeconds = expiration.getTime();
milliSeconds += 1000 * 60 * 5;
expiration.setTime(milliSeconds);
GeneratePresignedUrlRequest xs3_genurl_req =
new GeneratePresignedUrlRequest(xs3_bucketname, file_name);
xs3_genurl_req.setMethod(HttpMethod.PUT);
xs3_genurl_req.setExpiration(expiration);
xs3_genurl_req.setContentType(file_type);
xs3_genurl_req.addRequestParameter("uploadId", uploadId);
xs3_genurl_req.addRequestParameter("partNumber", String.valueOf(part_no + 1));
URL url = xs3_client.generatePresignedUrl(xs3_genurl_req);
System.out.println(url.toString());
JSONObject jsonObject = new JSONObject();
jsonObject.put("part_idx", part_no);
jsonObject.put("part_url", url.toString());
jsonObject.put("upload_len", part_size);
jsonObject.put("part_begin", xs3_offset_bytes);
jsonObject.put("part_end", xs3_offset_bytes + part_size);
jsonArray_sub.add(jsonObject);
}
jsonObject_1.put("multi_list", jsonArray_sub);
jsonArray.add(jsonObject_1);
return jsonArray.toString();
} catch (AmazonServiceException ase) {
System.out.println("xs3_svr_error_message:" + ase.getMessage());
System.out.println("xs3_svr_status_code: " + ase.getStatusCode());
System.out.println("xs3_svr_error_code: " + ase.getErrorCode());
System.out.println("xs3_svr_error_type: " + ase.getErrorType());
System.out.println("xs3_svr_request_id: " + ase.getRequestId());
} catch (AmazonClientException ace) {
System.out.println("xs3_clt_error_message:" + ace.getMessage());
}
return null;
}
/**
* The method builds the Amazon S3 client using the initialized Aws_Credentials and the Amazon S3
* endpoint. The Amazon S3 client is used for all the interaction with Amazon S3 storage system
*
* @return AmazonS3 client
*/
private static AmazonS3 getS3Client() {
AmazonS3 s3Client = new AmazonS3Client(AWS_CREDENTIALS);
s3Client.setEndpoint("AMAZON_S3_ENDPOINT");
return s3Client;
}
