@Test
public void testSkippingHeaderWithText() throws IOException {
TableMeta meta = CatalogUtil.newTableMeta("TEXT");
meta.putOption(StorageConstants.TEXT_SKIP_HEADER_LINE, "1");
meta.putOption(StorageConstants.TEXT_DELIMITER, ",");
FileFragment fragment = getFileFragment("testSkip.txt");
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
scanner.init();
int lines = 0;
try {
while (true) {
Tuple tuple = scanner.next();
if (tuple != null) {
assertEquals(17 + lines, tuple.getInt2(2));
lines++;
} else break;
}
} finally {
assertEquals(6, lines);
scanner.close();
}
}
@Test
public void testIgnoreTruncatedValueErrorTolerance() throws IOException {
TajoConf conf = new TajoConf();
TableMeta meta = CatalogUtil.newTableMeta("JSON");
meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "1");
FileFragment fragment = getFileFragment("testErrorTolerance3.json");
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
scanner.init();
try {
Tuple tuple = scanner.next();
assertNull(tuple);
} finally {
scanner.close();
}
}
@Test
public void testNoErrorTolerance() throws IOException {
TajoConf conf = new TajoConf();
TableMeta meta = CatalogUtil.newTableMeta("JSON");
meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "0");
FileFragment fragment = getFileFragment("testErrorTolerance2.json");
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
scanner.init();
try {
scanner.next();
} catch (IOException ioe) {
return;
} finally {
scanner.close();
}
fail();
}
@Test
public void testIgnoreAllErrors() throws IOException {
TajoConf conf = new TajoConf();
TableMeta meta = CatalogUtil.newTableMeta("JSON");
meta.putOption(StorageUtil.TEXT_ERROR_TOLERANCE_MAXNUM, "-1");
FileFragment fragment = getFileFragment("testErrorTolerance1.json");
Scanner scanner = TablespaceManager.getLocalFs().getScanner(meta, schema, fragment);
scanner.init();
Tuple tuple;
int i = 0;
while ((tuple = scanner.next()) != null) {
assertEquals(baseTuple, tuple);
i++;
}
assertEquals(3, i);
scanner.close();
}