private void test(boolean hasHeader, String input, List<String[]> expOutput) throws Exception {
StringReader r = new StringReader(input);
CSVParser p = new CSVParser(r, hasHeader, CSVParser.DELIMINATOR, "\n");
List<String[]> actual = p.readAll();
assertEquals(CSVTestUtils.toString(expOutput), CSVTestUtils.toString(actual));
}
@Override
@SuppressWarnings("unchecked")
public <T> T fromString(String content, Class<T> classOfT) {
if (!classOfT.isArray()) {
if (Collection.class.isAssignableFrom(classOfT)) {
// Collections are NOT supported for deserialization from CSV
throw new RuntimeException(
"Collection types are not supported. Please specify an array[] type.");
}
throw new RuntimeException(
String.format("Array[] types are required. Please specify %s[]", classOfT.getName()));
}
Class<?> objectType = classOfT.getComponentType();
int currentLine = 0;
try (CSVParser parser = new CSVParser(new StringReader(content), getCSVFormat().withHeader())) {
Set<String> columns = parser.getHeaderMap().keySet();
Map<String, Field> fieldMap = getFieldMap(objectType);
Constructor<?> objectConstructor;
try {
objectConstructor = objectType.getConstructor();
} catch (NoSuchMethodException e) {
throw new RuntimeException("A default constructor is required for " + objectType.getName());
}
List objects = new ArrayList<>();
for (CSVRecord record : parser) {
currentLine++;
Object o = objectConstructor.newInstance();
for (String column : columns) {
Field field = fieldMap.get(caseSensitiveFieldNames ? column : column.toLowerCase());
String value = record.get(column);
Object object = objectFromString(value, field.getType());
field.set(o, object);
}
objects.add(o);
}
Object array = Array.newInstance(objectType, objects.size());
for (int i = 0; i < objects.size(); i++) {
Array.set(array, i, objects.get(i));
}
return (T) array;
} catch (Exception e) {
throw new RuntimeException("Failed to parse CSV near line #" + currentLine, e);
}
}
@Override
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
try {
CSVParser parser = new CSVParser();
String[] fields = parser.parseLine(value.toString());
BrowerLog BrowerLog = new BrowerLog(fields.length);
BrowerLog.setFileds(fields);
newValue.set(BrowerLog.toString());
context.write(NullWritable.get(), newValue);
} catch (Exception e) {
// newValue.set(e.getMessage() + "\t" + value.toString());
// multipleOutputs.write(NullWritable.get(), newValue, errorOutputDir);
}
}
/**
* Reads the next line from the buffer and converts to a string array.
*
* @return a string array with each comma-separated element as a separate entry.
* @throws IOException if bad things happen during the read
*/
@Nullable
public List<String> readNext() throws IOException {
List<String> ret = null;
do {
final String sNextLine = _getNextLine();
if (!m_bHasNext) {
// should throw if still pending?
return ret;
}
final List<String> r = m_aParser.parseLineMulti(sNextLine);
if (ret == null) ret = r;
else ret.addAll(r);
} while (m_aParser.isPending());
return ret;
}
@Test
public void testParse() throws Exception {
SortedSet<MatchResult> results = parser.parse(LogFileHelper.getValidCsvLogFile("log.csv"));
assertEquals(results.size(), 2);
assertEquals(results.first().getSkillMean(), -1f);
assertEquals(results.first().getSkillSigma(), -1f);
}
@Test
public void testParsePlayListNumber() throws Exception {
SortedSet<MatchResult> results =
parser.parse(LogFileHelper.getValidCsvLogFile("playlistAsNumber.csv"));
assertEquals(results.size(), 1);
assertEquals(results.first().getPlayList(), MatchResult.SOLO_RANKED_3V3);
}
/** Created by schls1 on 24.11.2014. */
public class CSVDataLocationTest {
URL url = JVMEnviroment.getNotNullResource("dummy-data.csv");
private final CSVParser parser = new CSVParser(true, ',', true);
private final CSVTable referenceTable = parser.parseWithRuntimeException(url);
@Test
public void testCSVTable() throws IOException {
CSVDataReference inMemory = referenceTable;
CSVDataReference urlCon = new CSVDataLocation(url, null, true, null, true);
try (CSVDataConnection refCon = inMemory.openConnection();
CSVDataConnection testCon = urlCon.openConnection()) {
int count = 0;
do {
CSVRecord reference = refCon.readNext();
CSVRecord test = testCon.readNext();
if (reference == null) {
Assert.assertNull(test);
break;
} else if (test == null) {
Assert.assertNull(reference);
break;
} else {
count++;
Assert.assertNotSame(reference, test);
Assert.assertEquals(reference, test);
}
} while (true);
Assert.assertEquals(37, count);
}
}
private void printConnectionInfo(CSVDataConnection connection) throws IOException {
System.out.println("Charset: " + connection.getCharset());
System.out.println("Headers present?: " + connection.hasHeaders());
System.out.println("Headers: " + connection.getHeaders());
}
private int countAndPrintRecords(CSVDataConnection connection) throws IOException {
CSVRecord record;
int count = 0;
while ((record = connection.readNext()) != null) {
System.out.println(++count + ": " + record);
}
return count;
}
}
/** @return The default quotation character for this parser. */
public char getQuoteChar() {
return m_aParser.getQuoteChar();
}
/**
* Sets the ignore quotations mode - if <code>true</code>, quotations are ignored.
*
* @param bIgnoreQuotations if <code>true</code>, quotations are ignored
* @return this
*/
@Nonnull
public CSVReader setIgnoreQuotations(final boolean bIgnoreQuotations) {
m_aParser.setIgnoreQuotations(bIgnoreQuotations);
return this;
}
@Test
public void testParseMissingHeader() throws Exception {
SortedSet<MatchResult> results =
parser.parse(LogFileHelper.getInvalidCsvLogFile("noheader.csv"));
assertEquals(results.size(), 0);
}
/**
* Sets the ignore leading whitespace setting - if true, white space in front of a quote in a
* field is ignored.
*
* @param bIgnoreLeadingWhiteSpace if <code>true</code>, white space in front of a quote in a
* field is ignored
* @return this
*/
@Nonnull
public CSVReader setIgnoreLeadingWhiteSpace(final boolean bIgnoreLeadingWhiteSpace) {
m_aParser.setIgnoreLeadingWhiteSpace(bIgnoreLeadingWhiteSpace);
return this;
}
/** @return the default ignoreQuotation setting for this parser. */
public boolean isIgnoreQuotations() {
return m_aParser.isIgnoreQuotations();
}
/**
* Sets the strict quotes setting - if true, characters outside the quotes are ignored.
*
* @param bStrictQuotes if <code>true</code>, characters outside the quotes are ignored
* @return this
*/
@Nonnull
public CSVReader setStrictQuotes(final boolean bStrictQuotes) {
m_aParser.setStrictQuotes(bStrictQuotes);
return this;
}
/** @return The default ignoreLeadingWhiteSpace setting for this parser. */
public boolean isIgnoreLeadingWhiteSpace() {
return m_aParser.isIgnoreLeadingWhiteSpace();
}
/**
* Sets the character to use for escaping a separator or quote.
*
* @param cEscapeChar the character to use for escaping a separator or quote.
* @return this
*/
@Nonnull
public CSVReader setEscapeChar(final char cEscapeChar) {
m_aParser.setEscapeChar(cEscapeChar);
return this;
}
/** @return The default strictQuotes setting for this parser. */
public boolean isStrictQuotes() {
return m_aParser.isStrictQuotes();
}
/**
* Sets the character to use for quoted elements.
*
* @param cQuoteChar the character to use for quoted element.
* @return this
*/
@Nonnull
public CSVReader setQuoteChar(final char cQuoteChar) {
m_aParser.setQuoteChar(cQuoteChar);
return this;
}
/** @return The default escape character for this parser. */
public char getEscapeChar() {
return m_aParser.getEscapeChar();
}
@Test
public void testParseBadValues() throws Exception {
SortedSet<MatchResult> results =
parser.parse(LogFileHelper.getInvalidCsvLogFile("badvalues.csv"));
assertEquals(results.size(), 0);
}
/**
* Sets the delimiter to use for separating entries.
*
* @param cSeparator the delimiter to use for separating entries
* @return this
*/
@Nonnull
public CSVReader setSeparatorChar(final char cSeparator) {
m_aParser.setSeparatorChar(cSeparator);
return this;
}
/** @return The default separator for this parser. */
public char getSeparatorChar() {
return m_aParser.getSeparatorChar();
}
@Test
public void testParseBadPlayListNumber() throws Exception {
SortedSet<MatchResult> results =
parser.parse(LogFileHelper.getInvalidCsvLogFile("playlistAsBadNumber.csv"));
assertEquals(results.size(), 0);
}
// rdfInputFile is actually a 3 column csv file. The returned output will
// also be a csv file but in property table rep.
// header not expected in input file
// if headeropt is true, header will also be printed
// if opt is 2 multiset will be collected in single line
// if opt is 1, then multi-sets will be collapsed to 'last' value of property
// if opt is 0 then all combinatorial possibilities will be explored
public MakePropertyTable(String rdfInputFile, String csvOutputFile, int opt, boolean headeropt) {
ArrayList<String[]> triples = new ArrayList<String[]>();
int count = 0;
HashSet<String> properties = new HashSet<String>();
String total = null;
try {
Scanner in = new Scanner(new FileReader(rdfInputFile));
while (in.hasNextLine()) {
total = in.nextLine();
CSVParser parse = new CSVParser();
String[] triple = parse.parseLine(total);
for (int i = 0; i < triple.length; i++) triple[i] = triple[i].replace(",", ";");
triples.add(triple);
if (!properties.contains(triple[1])) {
count++;
properties.add(triple[1]);
}
}
in.close();
} catch (IOException E) {
System.out.println(total);
}
ArrayList<String> props = new ArrayList<String>(properties);
Collections.sort(props);
ArrayList<String[]> csvRecords = new ArrayList<String[]>();
HashMap<String, Integer> subjectIndex = new HashMap<String, Integer>();
for (String[] triple : triples) {
if (!subjectIndex.containsKey(triple[0])) {
String[] row = new String[count + 1];
row[0] = triple[0];
csvRecords.add(row);
subjectIndex.put(triple[0], csvRecords.size() - 1);
}
int propPosition = props.indexOf(triple[1]) + 1;
int subjectPosition = subjectIndex.get(triple[0]);
if (csvRecords.get(subjectPosition)[propPosition] == null)
csvRecords.get(subjectPosition)[propPosition] = triple[2];
else
csvRecords.get(subjectPosition)[propPosition] =
csvRecords.get(subjectPosition)[propPosition] + "," + triple[2];
}
csvRecords = preprocess(csvRecords, opt);
try {
PrintWriter out = new PrintWriter(new File(csvOutputFile));
String header = "subject,";
for (String prop : props) header += (prop + ",");
header = header.substring(0, header.length() - 1);
header = correct(header);
if (headeropt) out.println(header);
for (String[] record : csvRecords) {
String l = correct(getCSVLine(record));
out.println(l);
}
out.close();
} catch (Exception E) {
}
}
