@Test
public void testReadDoubleMultiFiles() throws Exception {
String dirPath = "src/test/resources/multi_file_double";
ArrayList<String> correctAnswer = readAllFiles(dirPath);
Collections.sort(
correctAnswer,
new Comparator<String>() {
@Override
public int compare(String o1, String o2) {
try {
Double d1 = Double.parseDouble(o1);
Double d2 = Double.parseDouble(o2);
return d1.compareTo(d2);
} catch (NumberFormatException e) {
e.printStackTrace();
return 0;
}
}
});
SortedColumnDFSFile column =
new SortedColumnDFSFile(qualify(dirPath + "/"), DataType.getType("double"));
IDictionaryValueEnumerator e = new TableColumnValueEnumerator(column.getReader(), -1);
ArrayList<String> output = new ArrayList<>();
while (e.moveNext()) {
System.out.println(new String(e.current()));
output.add(new String(e.current()));
}
System.out.println(correctAnswer.size());
assertTrue(correctAnswer.size() == output.size());
for (int i = 0; i < correctAnswer.size(); i++) {
assertEquals(correctAnswer.get(i), output.get(i));
}
}
@Test
public void testEmptyDir() throws Exception {
String dirPath = "src/test/resources/empty_dir";
new File(dirPath).mkdirs();
SortedColumnDFSFile column =
new SortedColumnDFSFile(qualify(dirPath + "/"), DataType.getType("varchar"));
IDictionaryValueEnumerator e = new TableColumnValueEnumerator(column.getReader(), -1);
ArrayList<String> output = new ArrayList<>();
while (e.moveNext()) {
System.out.println(new String(e.current()));
output.add(new String(e.current()));
}
System.out.println(output.size());
}
@Test
public void testReadWrite() throws Exception {
// string dict
Dictionary<String> dict =
getDict(DataType.getType("string"), Arrays.asList(new String[] {"a", "b"}).iterator());
readWriteTest(dict);
// number dict
Dictionary<String> dict2 =
getDict(DataType.getType("long"), Arrays.asList(new String[] {"1", "2"}).iterator());
readWriteTest(dict2);
// date dict
Dictionary<String> dict3 =
getDict(
DataType.getType("datetime"),
Arrays.asList(new String[] {"20161122", "20161123"}).iterator());
readWriteTest(dict3);
// date dict
Dictionary<String> dict4 =
getDict(
DataType.getType("datetime"),
Arrays.asList(new String[] {"2016-11-22", "2016-11-23"}).iterator());
readWriteTest(dict4);
// date dict
try {
Dictionary<String> dict5 =
getDict(
DataType.getType("date"),
Arrays.asList(new String[] {"2016-11-22", "20161122"}).iterator());
readWriteTest(dict5);
fail("Date format not correct.Should throw exception");
} catch (IllegalArgumentException e) {
// correct
}
}
@Test
public void testReadStringMultiFile() throws Exception {
String dirPath = "src/test/resources/multi_file_str";
ArrayList<String> correctAnswer = readAllFiles(dirPath);
Collections.sort(correctAnswer, new ByteComparator<String>(new StringBytesConverter()));
SortedColumnDFSFile column =
new SortedColumnDFSFile(qualify(dirPath + "/"), DataType.getType("varchar"));
IDictionaryValueEnumerator e = new TableColumnValueEnumerator(column.getReader(), -1);
ArrayList<String> output = new ArrayList<>();
while (e.moveNext()) {
output.add(new String(e.current()));
}
System.out.println(correctAnswer.size());
assertTrue(correctAnswer.size() == output.size());
for (int i = 0; i < correctAnswer.size(); i++) {
assertEquals(correctAnswer.get(i), output.get(i));
}
}
