@Test
public void testZeroSizeFile() throws Exception {
ByteArrayOutputStream log = new ByteArrayOutputStream();
helper.redirectLog(log, LogLevel.BASIC);
try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.json");
LocaleChange enUs = new LocaleChange(Locale.US); ) {
fileObj.createFile();
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
JsonInputMeta meta = createSimpleMeta("in file", price);
meta.setIsAFile(true);
meta.setRemoveSourceField(true);
meta.setIgnoreEmptyFile(false);
JsonInput jsonInput =
createJsonInput(
"in file", meta, new Object[][] {new Object[] {BASE_RAM_DIR + "test.json"}});
processRows(jsonInput, 1);
String logMsgs = log.toString();
assertTrue(logMsgs, logMsgs.contains("is empty!"));
} finally {
deleteFiles();
}
}
@Test
public void testErrorRedirect() throws Exception {
JsonInputField field = new JsonInputField("value");
field.setPath("$.value");
field.setType(ValueMetaInterface.TYPE_STRING);
String input1 = "{{";
String input2 = "{ \"value\": \"ok\" }";
JsonInputMeta meta = createSimpleMeta("json", field);
meta.setRemoveSourceField(true);
when(helper.stepMeta.isDoingErrorHandling()).thenReturn(true);
JsonInput jsonInput =
createJsonInput("json", meta, new Object[] {input1}, new Object[] {input2});
StepErrorMeta errMeta = new StepErrorMeta(jsonInput, helper.stepMeta);
errMeta.setEnabled(true);
errMeta.setErrorFieldsValuename("err field");
when(helper.stepMeta.getStepErrorMeta()).thenReturn(errMeta);
final List<Object[]> errorLines = new ArrayList<>();
jsonInput.addRowListener(
new RowComparatorListener(new Object[] {"ok"}) {
@Override
public void errorRowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
throws KettleStepException {
errorLines.add(row);
}
});
processRows(jsonInput, 3);
Assert.assertEquals("fwd error", 1, errorLines.size());
Assert.assertEquals("input in err line", input1, errorLines.get(0)[0]);
Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
}
@Test
public void testDualExpMismatchPathLeafToNull() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField isbn = new JsonInputField("isbn");
isbn.setPath("$..book[*].isbn");
isbn.setType(ValueMetaInterface.TYPE_STRING);
JsonInputField price = new JsonInputField("price");
price.setPath("$..book[*].price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
JsonInputMeta meta = createSimpleMeta("json", isbn, price);
meta.setRemoveSourceField(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {null, 8.95d},
new Object[] {null, 12.99d},
new Object[] {"0-553-21311-3", 8.99d},
new Object[] {"0-395-19395-8", 22.99d});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
Assert.assertEquals("rows written", 4, jsonInput.getLinesWritten());
}
@Test
public void testRowLimit() throws Exception {
final String inCol = "json";
JsonInputField jpath = new JsonInputField("isbn");
jpath.setPath("$..book[*].isbn");
jpath.setType(ValueMetaInterface.TYPE_STRING);
JsonInputMeta meta = createSimpleMeta(inCol, jpath);
meta.setRemoveSourceField(true);
meta.setIgnoreMissingPath(true);
meta.setRowLimit(2);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
processRows(jsonInput, 4);
Assert.assertEquals("errors", 0, jsonInput.getErrors());
Assert.assertEquals("lines written", 2, jsonInput.getLinesWritten());
}
@Test
public void testZipFileInput() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input = getBasicTestJson();
try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.zip")) {
fileObj.createFile();
try (OutputStream out = fileObj.getContent().getOutputStream()) {
try (ZipOutputStream zipOut = new ZipOutputStream(out)) {
ZipEntry jsonFile = new ZipEntry("test.json");
zipOut.putNextEntry(jsonFile);
zipOut.write(input.getBytes());
zipOut.closeEntry();
zipOut.flush();
}
}
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
JsonInputMeta meta = createSimpleMeta("in file", price);
meta.setIsAFile(true);
meta.setRemoveSourceField(true);
JsonInput jsonInput =
createJsonInput(
"in file",
meta,
new Object[][] {new Object[] {"zip:" + BASE_RAM_DIR + "test.zip!/test.json"}});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {8.95d},
new Object[] {12.99d},
new Object[] {8.99d},
new Object[] {22.99d});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
Assert.assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
@Test
public void testDefaultLeafToNull() throws Exception {
JsonInputField noPath = new JsonInputField("price");
noPath.setPath("$..price");
noPath.setType(ValueMetaInterface.TYPE_STRING);
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputMeta meta = createSimpleMeta("json", noPath);
meta.setIgnoreMissingPath(true);
meta.setRemoveSourceField(true);
final String input = getBasicTestJson();
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {input}, new Object[] {input});
processRows(jsonInput, 7);
disposeJsonInput(jsonInput);
Assert.assertEquals(5, jsonInput.getLinesWritten());
}
@Test
public void testSingleObjPred() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField bic = new JsonInputField("color");
bic.setPath("$.store.bicycle[?(@.price)].color");
bic.setType(ValueMetaInterface.TYPE_STRING);
JsonInputMeta meta = createSimpleMeta("json", bic);
meta.setRemoveSourceField(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
RowComparatorListener rowComparator = new RowComparatorListener(new Object[] {"red"});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 2);
Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
}
@Test
public void testArrayOut() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField byc = new JsonInputField("books (array)");
byc.setPath("$.store.book");
byc.setType(ValueMetaInterface.TYPE_STRING);
JsonInputMeta meta = createSimpleMeta("json", byc);
meta.setRemoveSourceField(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {
"[{\"category\":\"reference\",\"author\":\"Nigel Rees\",\"title\":\"Sayings of the Century\",\"price\":8.95},{\"category\":\"fiction\",\"author\":\"Evelyn Waugh\",\"title\":\"Sword of Honour\",\"price\":12.99},{\"category\":\"fiction\",\"author\":\"Herman Melville\",\"title\":\"Moby Dick\",\"isbn\":\"0-553-21311-3\",\"price\":8.99},{\"category\":\"fiction\",\"author\":\"J. R. R. Tolkien\",\"title\":\"The Lord of the Rings\",\"isbn\":\"0-395-19395-8\",\"price\":22.99}]"
});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 2);
Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
}
@Test
public void testExtraFileFields() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input1 = getBasicTestJson();
final String input2 = "{ \"store\": { \"bicycle\": { \"color\": \"blue\" } } }";
final String path1 = BASE_RAM_DIR + "test1.json";
final String path2 = BASE_RAM_DIR + "test2.js";
try (FileObject fileObj1 = KettleVFS.getFileObject(path1);
FileObject fileObj2 = KettleVFS.getFileObject(path2)) {
try (OutputStream out = fileObj1.getContent().getOutputStream()) {
out.write(input1.getBytes());
}
try (OutputStream out = fileObj2.getContent().getOutputStream()) {
out.write(input2.getBytes());
}
JsonInputField color = new JsonInputField();
color.setName("color");
color.setType(ValueMetaInterface.TYPE_STRING);
color.setPath("$.store.bicycle.color");
JsonInputMeta meta = createSimpleMeta("in file", color);
meta.setInFields(true);
meta.setIsAFile(true);
meta.setRemoveSourceField(true);
meta.setExtensionField("extension");
meta.setPathField("dir path");
meta.setSizeField("size");
meta.setIsHiddenField("hidden?");
meta.setLastModificationDateField("last modified");
meta.setUriField("URI");
meta.setRootUriField("root URI");
// custom checkers for size and last modified
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {
"red",
"json",
"ram:///jsonInputTest",
-1L,
false,
new Date(0),
"ram:///jsonInputTest/test1.json",
"ram:///"
},
new Object[] {
"blue",
"js",
"ram:///jsonInputTest",
-1L,
false,
new Date(0),
"ram:///jsonInputTest/test2.js",
"ram:///"
});
rowComparator.setComparator(
3,
new RowComparatorListener.Comparison<Object>() {
@Override
public boolean equals(Object expected, Object actual) throws Exception {
// just want a valid size
return ((long) actual) > 0L;
}
});
rowComparator.setComparator(
5,
new RowComparatorListener.Comparison<Object>() {
@Override
public boolean equals(Object expected, Object actual) throws Exception {
return ((Date) actual).after(new Date(0));
}
});
JsonInput jsonInput =
createJsonInput(
"in file", meta, new Object[][] {new Object[] {path1}, new Object[] {path2}});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 3);
Assert.assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
