/** Data cleansing method */
public void cleanseData(AmazonS3 client) throws Exception {
AwsDataLoader loader = new AwsDataLoader();
CSVReader reader = null;
String prefix = loader.getS3Prefix(source);
client.setEndpoint(S3_ENDPOINT);
S3Object object = client.getObject(new GetObjectRequest(BUCKET_NM, prefix));
reader =
new CSVReader(
new BufferedReader(new InputStreamReader(object.getObjectContent())),
CSVParser.DEFAULT_SEPARATOR,
CSVParser.DEFAULT_QUOTE_CHARACTER,
CSVParser.DEFAULT_ESCAPE_CHARACTER,
HEADERS_LINE);
ColumnPositionMappingStrategy<ProductLanguage> strat =
new ColumnPositionMappingStrategy<ProductLanguage>();
strat.setType(ProductLanguage.class);
String[] columns =
new String[] {"refId", "code", "name", "locale", "state", "format", "displayNameLanguage"};
strat.setColumnMapping(columns);
CsvToBean<ProductLanguage> csv = new CsvToBean<ProductLanguage>();
list = csv.parse(strat, reader);
System.out.println("ProductLanguageCleanser input size: " + list.size());
this.updateDataset(list);
BeanToCsv<ProductLanguage> csvWriter = new BeanToCsv<ProductLanguage>();
ByteArrayOutputStream os = new ByteArrayOutputStream();
CSVWriter writer = new CSVWriter(new OutputStreamWriter(os), ',', '"');
// strat.setColumnMapping(columns);
log.info("List size: " + list.size());
csvWriter.write(strat, writer, list);
writer.flush();
String dataset = os.toString();
String outPrefix = PREFIX + OUTPUT_KEY + source + ".csv";
client.setEndpoint(S3_ENDPOINT);
ObjectMetadata omd = new ObjectMetadata();
try {
byte[] content = dataset.getBytes(StandardCharsets.UTF_8);
ByteArrayInputStream input = new ByteArrayInputStream(content);
BufferedReader buffReader = new BufferedReader(new InputStreamReader(input));
buffReader.readLine();
InputStream inputObj = new ReaderInputStream(buffReader);
// omd.setContentLength(IOUtils.toByteArray(input).length);
client.putObject(BUCKET_NM, outPrefix, inputObj, omd);
input.close();
} catch (IOException e) {
log.error("Axon data write to s3 failed: " + e.getMessage());
}
}
/**
* Method use OpenCsv Library for
*
* @param columnMapping Map allow the user to pass the column Names to a Field Names of the Class.
* @param fileInputCsv the File CSV to parse.
* @param <T> the generic variable.
* @return the List of Bean parsed from the CSV file.
*/
public static <T> List<T> parseCSVFileAsList(
Map<String, String> columnMapping, File fileInputCsv) {
try {
HeaderColumnNameTranslateMappingStrategy<T> beanStrategy =
new HeaderColumnNameTranslateMappingStrategy<>();
// beanStrategy.setType(clazz); //deprecated
/*Map<String, String> columnMapping = new HashMap<>();
columnMapping.put("ID", "id");
columnMapping.put("Name", "name");
columnMapping.put("Role", "role");*/
beanStrategy.setColumnMapping(columnMapping);
CsvToBean<T> csvToBean = new CsvToBean<>();
CSVReader reader = new CSVReader(new FileReader(fileInputCsv));
return csvToBean.parse(beanStrategy, reader);
} catch (IOException e) {
logger.error(
"Can't parse the CSV file:" + fileInputCsv.getAbsolutePath() + " -> " + e.getMessage(),
e);
return new ArrayList<>();
}
}