@Override
public void exportCSV(String outputFile) {
List<Schedule> data = getSchedule();
FileWriter fileWriter = null;
CSVPrinter csvFilePrinter = null;
try {
fileWriter = new FileWriter(outputFile);
csvFilePrinter = new CSVPrinter(fileWriter, CSVFormat.DEFAULT.withRecordSeparator("\n"));
for (int i = 0; i < data.size(); i++) {
List<Object> line = new ArrayList<>();
for (Field field : Schedule.class.getDeclaredFields()) {
field.setAccessible(true);
Object value = field.get(data.get(i));
line.add(value);
}
csvFilePrinter.printRecord(line);
}
} catch (IOException | IllegalAccessException e) {
e.printStackTrace();
} finally {
try {
if (fileWriter != null) {
fileWriter.flush();
fileWriter.close();
}
if (csvFilePrinter != null) {
csvFilePrinter.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* default settings delimiter = ',' quoteChar = '"', escape = null recordSeparator = CRLF, CR, or
* LF ignore empty lines allows the last data line to have a recordSeparator
*
* @return CSVFormat based on constructor settings.
*/
private CSVFormat buildFormat() {
CSVFormat format =
CSVFormat.DEFAULT
.withIgnoreEmptyLines(true)
.withDelimiter(asControlCharacter(fieldDelimiter))
.withQuoteChar(asControlCharacter(quoteCharacter));
if (escapeCharacter != null) {
format = format.withEscape(asControlCharacter(escapeCharacter));
}
switch (headerSource) {
case FROM_TABLE:
// obtain headers from table, so format should not expect a header.
break;
case IN_LINE:
// an empty string array triggers csv loader to grab the first line as the header
format = format.withHeader(new String[0]);
break;
case SUPPLIED_BY_USER:
// a populated string array supplied by the user
format = format.withHeader(columns.toArray(new String[columns.size()]));
break;
default:
throw new RuntimeException("Header source was unable to be inferred.");
}
return format;
}
/**
* CSV reader that waits for a 2 columns csv files with or without a header. If less than 2
* columns ==> exception, otherwise, the 3rd and following columns are ignored
*
* @param in
* @param hasHeader
* @param inputDatetimeFormat input date format
* @return
* @throws IOException
* @throws IllegalArgumentException
* @throws ArrayIndexOutOfBoundsException
*/
public static List<Record> load(
Reader in, boolean hasHeader, DateTimeFormatter inputDatetimeFormat) throws IOException {
List<Record> records = new ArrayList<>();
for (CSVRecord record : CSVFormat.DEFAULT.parse(in)) {
try {
if (!hasHeader) {
StandardRecord event = new StandardRecord("sensors");
event.setField(
TIMESTAMP_KEY,
FieldType.LONG,
inputDatetimeFormat
.withZone(DateTimeZone.UTC)
.parseDateTime(record.get(0))
.getMillis());
event.setField(VALUE_KEY, FieldType.DOUBLE, Double.parseDouble(record.get(1)));
records.add(event);
} else {
TIMESTAMP_KEY = record.get(0);
VALUE_KEY = record.get(1);
}
hasHeader = false;
} catch (Exception e) {
logger.error("Parsing error " + e.getMessage());
throw new RuntimeException("parsing error", e);
}
}
return records;
}
CsvLineParser(char fieldDelimiter, char quote, char escape) {
this.csvFormat =
CSVFormat.DEFAULT
.withIgnoreEmptyLines(true)
.withDelimiter(fieldDelimiter)
.withEscape(escape)
.withQuote(quote);
}
@Test
public void testWithHeader() throws Exception {
final String[] header = new String[] {"one", "two", "three"};
// withHeader() makes a copy of the header array.
final CSVFormat formatWithHeader = CSVFormat.DEFAULT.withHeader(header);
assertArrayEquals(header, formatWithHeader.getHeader());
assertNotSame(header, formatWithHeader.getHeader());
}
@SuppressWarnings("boxing") // no need to worry about boxing here
@Test
public void testSerialization() throws Exception {
final ByteArrayOutputStream out = new ByteArrayOutputStream();
try (final ObjectOutputStream oos = new ObjectOutputStream(out)) {
oos.writeObject(CSVFormat.DEFAULT);
oos.flush();
}
final ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(out.toByteArray()));
final CSVFormat format = (CSVFormat) in.readObject();
assertNotNull(format);
assertEquals("delimiter", CSVFormat.DEFAULT.getDelimiter(), format.getDelimiter());
assertEquals("encapsulator", CSVFormat.DEFAULT.getQuoteCharacter(), format.getQuoteCharacter());
assertEquals("comment start", CSVFormat.DEFAULT.getCommentMarker(), format.getCommentMarker());
assertEquals(
"record separator", CSVFormat.DEFAULT.getRecordSeparator(), format.getRecordSeparator());
assertEquals("escape", CSVFormat.DEFAULT.getEscapeCharacter(), format.getEscapeCharacter());
assertEquals(
"trim",
CSVFormat.DEFAULT.getIgnoreSurroundingSpaces(),
format.getIgnoreSurroundingSpaces());
assertEquals(
"empty lines", CSVFormat.DEFAULT.getIgnoreEmptyLines(), format.getIgnoreEmptyLines());
}
public CSVFormat getCSVFormat() {
return CSVFormat.DEFAULT
.withDelimiter(delimiter)
.withRecordSeparator(recordSeparator)
.withNullString(nullString)
.withEscape(escapeCharacter)
.withQuote(quoteCharacter)
.withQuoteMode(quoteMode);
}
public static void main(String[] args) {
// 要导入数据库的文档
List<Document> docList = new ArrayList<Document>();
File inFile = new File(IN_PATH);
CSVFormat format = CSVFormat.DEFAULT.withRecordSeparator('\n'); // 每条记录间隔符
if (!inFile.exists()) {
throw new RuntimeException("原始输入文件不存在！");
}
Reader reader = null;
CSVParser parser = null;
try {
reader = new InputStreamReader(new FileInputStream(inFile), "utf-8");
parser = new CSVParser(reader, format);
} catch (IOException e) {
throw new RuntimeException("Csv File output preparing fails", e);
}
Iterator<CSVRecord> iterator = parser.iterator();
while (iterator.hasNext()) {
CSVRecord record = iterator.next();
Iterator<String> iterator2 = record.iterator();
boolean isTarget = true;
String target = null;
while (iterator2.hasNext()) {
if (isTarget) {
target = iterator2.next();
isTarget = false;
continue;
}
String[] recAndScore = iterator2.next().trim().split("\\|");
if (recAndScore.length == 3) {
String rec = recAndScore[0];
String score = recAndScore[1];
String cosine = recAndScore[2];
Map<String, Object> map = new HashMap<String, Object>();
map.put("target", target);
map.put("rec", rec);
map.put("score", Double.valueOf(score));
map.put("cosine", Double.valueOf(cosine));
docList.add(new Document(map));
}
}
}
System.out.println(docList.size());
try {
parser.close();
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
importToMongoDB(MONGO_HOST, MONGO_PORT, DATABASE, COLLECTION, docList);
}
public TradingDataResource(final URL locatorBase, final String strategy) {
super("Trading Data", locatorBase);
this.strategy = strategy;
this.csvFormat =
CSVFormat.DEFAULT
.withRecordSeparator(
ApplicationContext.getInstance()
.getConfigurationResource()
.getDefaultCsvRecordSeparator())
.withHeader();
}
@Test
public void testGetHeader() throws Exception {
final String[] header = new String[] {"one", "two", "three"};
final CSVFormat formatWithHeader = CSVFormat.DEFAULT.withHeader(header);
// getHeader() makes a copy of the header array.
final String[] headerCopy = formatWithHeader.getHeader();
headerCopy[0] = "A";
headerCopy[1] = "B";
headerCopy[2] = "C";
assertFalse(Arrays.equals(formatWithHeader.getHeader(), headerCopy));
assertNotSame(formatWithHeader.getHeader(), headerCopy);
}
private List<Measurement> readMeasurements(String measurementCsvLocation) {
List<Measurement> measurements = new ArrayList<>();
try (Reader in = new FileReader(measurementCsvLocation)) {
Iterable<CSVRecord> records = CSVFormat.DEFAULT.withDelimiter(';').parse(in);
for (CSVRecord record : records) {
Long timeStamp = Long.parseLong(record.get(0));
Double temperature = Double.parseDouble(record.get(1));
measurements.add(new Measurement(timeStamp, temperature));
}
} catch (IOException e) {
e.printStackTrace();
}
return measurements;
}
@Test
public void testApacheCommonCsvMapping() throws Exception {
StringReader stringReader = new StringReader("foo,bar,15,true");
CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader("firstName", "lastName", "age", "married");
ApacheCommonCsvRecord record = getApacheCommonCsvRecord(stringReader, csvFormat);
Foo foo = mapper.mapRecord(record);
assertThat(foo).isNotNull();
assertThat(foo.getFirstName()).isEqualTo("foo");
assertThat(foo.getLastName()).isEqualTo("bar");
assertThat(foo.getAge()).isEqualTo(15);
assertThat(foo.isMarried()).isTrue();
}
@Test
public void testApacheCommonCsvQualifier() throws Exception {
StringReader stringReader = new StringReader("'foo,s','bar,n'");
CSVFormat csvFormat =
CSVFormat.DEFAULT.withQuote('\'').withHeader("firstName", "lastName", "age", "married");
ApacheCommonCsvRecord record = getApacheCommonCsvRecord(stringReader, csvFormat);
Foo foo = mapper.mapRecord(record);
assertThat(foo).isNotNull();
assertThat(foo.getFirstName()).isEqualTo("foo,s");
assertThat(foo.getLastName()).isEqualTo("bar,n");
assertThat(foo.getAge()).isEqualTo(0);
assertThat(foo.isMarried()).isFalse();
}
public static DoubleMatrix readFileIntoMatrix(String filename, char delimiter)
throws IOException {
File csvData = new File(filename);
CSVFormat format = CSVFormat.DEFAULT.withHeader().withDelimiter(delimiter);
CSVParser parser = CSVParser.parse(csvData, StandardCharsets.UTF_8, format);
int row = 0;
List<CSVRecord> records = parser.getRecords();
DoubleMatrix x = new DoubleMatrix(records.size(), records.get(0).size());
for (CSVRecord csvRecord : records) {
for (int column = 0; column < csvRecord.size(); column++) {
String s = csvRecord.get(column);
x.put(row, column, Double.valueOf(s));
}
row++;
}
return x;
}
@Override
public String outputReportResult(Report report) {
FileWriter fileWriter = null;
CSVPrinter csvFilePrinter = null;
CSVFormat csvFileFormat = CSVFormat.DEFAULT.withRecordSeparator(NEW_LINE_SEPARATOR);
try {
SimpleDateFormat dateFormat = new SimpleDateFormat("ddmmyyHHmmss");
String outputFileName = String.format(OUTPUT_FILE_NAME, dateFormat.format(new Date()));
fileWriter = new FileWriter(outputFileName);
csvFilePrinter = new CSVPrinter(fileWriter, csvFileFormat);
csvFilePrinter.printRecord(FILE_HEADER);
for (ReportList reportList : report.getReportLists()) {
csvFilePrinter.printRecord(
reportList.getPerson(),
reportList.getPaymentMonth(),
reportList.getGrossIncome(),
reportList.getIncomeTax(),
reportList.getNetIncome(),
reportList.getSuperannuation());
}
return outputFileName;
} catch (Exception e) {
e.printStackTrace();
return null;
} finally {
try {
fileWriter.flush();
fileWriter.close();
csvFilePrinter.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
public void crearCSV(LinkedList<Mapeo> rsList) throws IOException {
FileWriter escritorArchivo = null;
CSVPrinter csvPrinter = null;
CSVFormat csvFileFormat = CSVFormat.DEFAULT.withRecordSeparator(SALTO_LINEA);
escritorArchivo = new FileWriter(PATH_VOLCADO_DB);
csvPrinter = new CSVPrinter(escritorArchivo, csvFileFormat);
csvPrinter.printRecord(CABECERAS);
for (Mapeo elemento : rsList) {
List<String> camposCsv = new ArrayList<String>();
camposCsv.add(elemento.getIdERP());
camposCsv.add(elemento.getIdNimbus());
camposCsv.add(elemento.getDescERP());
camposCsv.add(elemento.getDescNimbus());
csvPrinter.printRecord(camposCsv);
}
escritorArchivo.flush();
escritorArchivo.close();
csvPrinter.close();
}
@Override
public void importCSV(String inputFile) {
try {
String csvData = new String(Files.readAllBytes(FileSystems.getDefault().getPath(inputFile)));
csvData = csvData.replaceAll("\\r", "");
CSVParser parser = CSVParser.parse(csvData, CSVFormat.DEFAULT.withRecordSeparator("\n"));
for (CSVRecord record : parser) {
Schedule schedule = new Schedule();
schedule.setId(Integer.parseInt(record.get(0)));
schedule.setMatchNum(Integer.parseInt(record.get(1)));
schedule.setB1(Integer.parseInt(record.get(2)));
schedule.setB2(Integer.parseInt(record.get(3)));
schedule.setB3(Integer.parseInt(record.get(4)));
schedule.setR1(Integer.parseInt(record.get(5)));
schedule.setR2(Integer.parseInt(record.get(6)));
schedule.setR3(Integer.parseInt(record.get(7)));
if (checkForMatch(schedule)) update(schedule);
else create(schedule);
}
} catch (IOException e) {
e.printStackTrace();
}
}
@Test
public void testWithIgnoreSurround() throws Exception {
assertFalse(CSVFormat.DEFAULT.withIgnoreSurroundingSpaces(false).getIgnoreSurroundingSpaces());
assertTrue(CSVFormat.DEFAULT.withIgnoreSurroundingSpaces().getIgnoreSurroundingSpaces());
}
@Test(expected = IllegalArgumentException.class)
public void testDuplicateHeaderElements() {
CSVFormat.DEFAULT.withHeader("A", "A");
}
@Test(expected = IllegalArgumentException.class)
public void testDelimiterSameAsEscapeThrowsException() {
CSVFormat.DEFAULT.withDelimiter('!').withEscape('!');
}
@Test(expected = IllegalArgumentException.class)
public void testDelimiterSameAsCommentStartThrowsException() {
CSVFormat.DEFAULT.withDelimiter('!').withCommentMarker('!');
}
@Test
public void testWithFirstRecordAsHeader() throws Exception {
final CSVFormat formatWithFirstRecordAsHeader = CSVFormat.DEFAULT.withFirstRecordAsHeader();
assertTrue(formatWithFirstRecordAsHeader.getSkipHeaderRecord());
assertTrue(formatWithFirstRecordAsHeader.getHeader().length == 0);
}
@Test
public void testWithRecordSeparatorCRLF() throws Exception {
final CSVFormat formatWithRecordSeparator = CSVFormat.DEFAULT.withRecordSeparator(CRLF);
assertEquals(CRLF, formatWithRecordSeparator.getRecordSeparator());
}
@Test
public void testWithRecordSeparatorLF() throws Exception {
final CSVFormat formatWithRecordSeparator = CSVFormat.DEFAULT.withRecordSeparator(LF);
assertEquals(String.valueOf(LF), formatWithRecordSeparator.getRecordSeparator());
}
@Test
public void testWithQuotePolicy() throws Exception {
final CSVFormat formatWithQuotePolicy = CSVFormat.DEFAULT.withQuoteMode(QuoteMode.ALL);
assertEquals(QuoteMode.ALL, formatWithQuotePolicy.getQuoteMode());
}
@Test(expected = IllegalArgumentException.class)
public void testWithQuoteLFThrowsException() {
CSVFormat.DEFAULT.withQuote(LF);
}
@Test
public void testWithQuoteChar() throws Exception {
final CSVFormat formatWithQuoteChar = CSVFormat.DEFAULT.withQuote('"');
assertEquals(Character.valueOf('"'), formatWithQuoteChar.getQuoteCharacter());
}
@Test
public void testWithNullString() throws Exception {
final CSVFormat formatWithNullString = CSVFormat.DEFAULT.withNullString("null");
assertEquals("null", formatWithNullString.getNullString());
}
@Test
public void testWithIgnoreEmptyLines() throws Exception {
assertFalse(CSVFormat.DEFAULT.withIgnoreEmptyLines(false).getIgnoreEmptyLines());
assertTrue(CSVFormat.DEFAULT.withIgnoreEmptyLines().getIgnoreEmptyLines());
}
@Test
public void testWithEmptyEnum() throws Exception {
final CSVFormat formatWithHeader = CSVFormat.DEFAULT.withHeader(EmptyEnum.class);
Assert.assertTrue(formatWithHeader.getHeader().length == 0);
}