@Test
public void testErrorRedirect() throws Exception {
JsonInputField field = new JsonInputField("value");
field.setPath("$.value");
field.setType(ValueMetaInterface.TYPE_STRING);
String input1 = "{{";
String input2 = "{ \"value\": \"ok\" }";
JsonInputMeta meta = createSimpleMeta("json", field);
meta.setRemoveSourceField(true);
when(helper.stepMeta.isDoingErrorHandling()).thenReturn(true);
JsonInput jsonInput =
createJsonInput("json", meta, new Object[] {input1}, new Object[] {input2});
StepErrorMeta errMeta = new StepErrorMeta(jsonInput, helper.stepMeta);
errMeta.setEnabled(true);
errMeta.setErrorFieldsValuename("err field");
when(helper.stepMeta.getStepErrorMeta()).thenReturn(errMeta);
final List<Object[]> errorLines = new ArrayList<>();
jsonInput.addRowListener(
new RowComparatorListener(new Object[] {"ok"}) {
@Override
public void errorRowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
throws KettleStepException {
errorLines.add(row);
}
});
processRows(jsonInput, 3);
Assert.assertEquals("fwd error", 1, errorLines.size());
Assert.assertEquals("input in err line", input1, errorLines.get(0)[0]);
Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
}
@Test
public void testRepeatFieldSingleObj() throws Exception {
final String input =
" { \"items\": [ "
+ "{ \"a\": 1, \"b\": null }, "
+ "{ \"a\":null, \"b\":2 }, "
+ "{ \"a\":3, \"b\":null }, "
+ "{ \"a\":4, \"b\":4 } ] }";
final String inCol = "input";
JsonInputField aField = new JsonInputField();
aField.setName("a");
aField.setPath("$.items[*].a");
aField.setType(ValueMetaInterface.TYPE_INTEGER);
JsonInputField bField = new JsonInputField();
bField.setName("b");
bField.setPath("$.items[*].b");
bField.setType(ValueMetaInterface.TYPE_INTEGER);
bField.setRepeated(true);
JsonInputMeta meta = createSimpleMeta(inCol, aField, bField);
JsonInput step = createJsonInput(inCol, meta, new Object[] {input});
step.addRowListener(
new RowComparatorListener(
new Object[] {input, 1L, null},
new Object[] {input, null, 2L},
new Object[] {input, 3L, 2L},
new Object[] {input, 4L, 4L}));
processRows(step, 4);
Assert.assertEquals(4, step.getLinesWritten());
}
/** File tests */
@Test
public void testNullFileList() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
try {
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
List<FileObject> fileList = Arrays.asList(null, null);
JsonInputMeta meta = createFileListMeta(fileList);
meta.setInputFields(new JsonInputField[] {price});
meta.setIncludeRowNumber(true);
meta.setRowNumberField("rownbr");
meta.setShortFileNameField("fname");
JsonInput jsonInput = createJsonInput(meta);
processRows(jsonInput, 5);
disposeJsonInput(jsonInput);
assertEquals(err.toString(), 2, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
@Test
public void testDualExpMismatchError() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField isbn = new JsonInputField("isbn");
isbn.setPath("$..book[?(@.isbn)].isbn");
isbn.setType(ValueMetaInterface.TYPE_STRING);
JsonInputField price = new JsonInputField("price");
price.setPath("$..book[*].price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
try (LocaleChange enUS = new LocaleChange(Locale.US)) {
JsonInputMeta meta = createSimpleMeta("json", isbn, price);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
processRows(jsonInput, 3);
Assert.assertEquals("error", 1, jsonInput.getErrors());
Assert.assertEquals("rows written", 0, jsonInput.getLinesWritten());
String errors =
IOUtils.toString(
new ByteArrayInputStream(out.toByteArray()), StandardCharsets.UTF_8.name());
String expectedError =
"The data structure is not the same inside the resource!"
+ " We found 4 values for json path [$..book[*].price],"
+ " which is different that the number returned for path [$..book[?(@.isbn)].isbn] (2 values)."
+ " We MUST have the same number of values for all paths.";
Assert.assertTrue("expected error", errors.contains(expectedError));
}
}
@Test
public void testIndexLastObj() throws Exception {
final String jsonInputField = getBasicTestJson();
JsonInput jsonInput =
createBasicTestJsonInput(
"$..book[-1:]",
new ValueMetaString("last book"),
"json",
new Object[] {jsonInputField});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {
jsonInputField,
"{ \"category\": \"fiction\",\n"
+ " \"author\": \"J. R. R. Tolkien\",\n"
+ " \"title\": \"The Lord of the Rings\",\n"
+ " \"isbn\": \"0-395-19395-8\",\n"
+ " \"price\": 22.99\n"
+ "}\n"
});
rowComparator.setComparator(1, new JsonComparison());
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 2);
Assert.assertEquals(1, jsonInput.getLinesWritten());
}
@Test
public void testDualExpMismatchPathLeafToNull() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField isbn = new JsonInputField("isbn");
isbn.setPath("$..book[*].isbn");
isbn.setType(ValueMetaInterface.TYPE_STRING);
JsonInputField price = new JsonInputField("price");
price.setPath("$..book[*].price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
JsonInputMeta meta = createSimpleMeta("json", isbn, price);
meta.setRemoveSourceField(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {null, 8.95d},
new Object[] {null, 12.99d},
new Object[] {"0-553-21311-3", 8.99d},
new Object[] {"0-395-19395-8", 22.99d});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
Assert.assertEquals("rows written", 4, jsonInput.getLinesWritten());
}
protected JsonInput createJsonInput(JsonInputMeta meta) {
JsonInputData data = new JsonInputData();
JsonInput jsonInput =
new JsonInput(helper.stepMeta, helper.stepDataInterface, 0, helper.transMeta, helper.trans);
jsonInput.init(meta, data);
return jsonInput;
}
@Test
public void testBfsMatchOrder() throws Exception {
// streaming will be dfs..ref impl is bfs
String input = "{ \"a\": { \"a\" : { \"b\" :2 } , \"b\":1 } }";
JsonInput jsonInput =
createBasicTestJsonInput("$..a.b", new ValueMetaInteger("b"), "in", new Object[] {input});
RowComparatorListener rowComparator =
new RowComparatorListener(jsonInput, new Object[] {input, 1L}, new Object[] {input, 2L});
rowComparator.setComparator(0, null);
processRows(jsonInput, 2);
Assert.assertEquals(2, jsonInput.getLinesWritten());
}
@Test
public void testNullProp() throws Exception {
final String input = "{ \"obj\": [ { \"nval\": null, \"val\": 2 }, { \"val\": 1 } ] }";
JsonInput jsonInput =
createBasicTestJsonInput(
"$.obj[?(@.nval)].val", new ValueMetaString("obj"), "json", new Object[] {input});
RowComparatorListener rowComparator = new RowComparatorListener(new Object[] {input, "2"});
rowComparator.setComparator(1, new JsonComparison());
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 2);
// in jsonpath 2.0->2.1, null value properties started being counted as existing
Assert.assertEquals(1, jsonInput.getLinesWritten());
}
protected void testSimpleJsonPath(
String jsonPath, ValueMetaInterface outputMeta, Object[][] inputRows, Object[][] outputRows)
throws Exception {
final String inCol = "in";
JsonInput jsonInput = createBasicTestJsonInput(jsonPath, outputMeta, inCol, inputRows);
RowComparatorListener rowComparator = new RowComparatorListener(outputRows);
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, outputRows.length + 1);
Assert.assertEquals("rows written", outputRows.length, jsonInput.getLinesWritten());
Assert.assertEquals("errors", 0, jsonInput.getErrors());
}
@Test
public void testJgdArray() throws Exception {
final String input =
" { \"arr\": [ [ { \"a\": 1, \"b\": 1}, { \"a\": 1, \"b\": 2} ], [ {\"a\": 3, \"b\": 4 } ] ] }";
JsonInput jsonInput =
createBasicTestJsonInput(
"$.arr", new ValueMetaString("array"), "json", new Object[] {input});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {input, "[[{\"a\":1,\"b\":1},{\"a\":1,\"b\":2}],[{\"a\":3,\"b\":4}]]"});
rowComparator.setComparator(1, new JsonComparison());
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 2);
Assert.assertEquals(1, jsonInput.getLinesWritten());
}
@Test
public void testUrlInput() throws Exception {
JsonInputField field = new JsonInputField("value");
field.setPath("$.value");
field.setType(ValueMetaInterface.TYPE_STRING);
String input1 = "http://localhost/test.json";
JsonInputMeta meta = createSimpleMeta("json", field);
meta.setReadUrl(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {input1});
processRows(jsonInput, 3);
Assert.assertEquals(1, jsonInput.getErrors());
}
@Test
public void testRowLimit() throws Exception {
final String inCol = "json";
JsonInputField jpath = new JsonInputField("isbn");
jpath.setPath("$..book[*].isbn");
jpath.setType(ValueMetaInterface.TYPE_STRING);
JsonInputMeta meta = createSimpleMeta(inCol, jpath);
meta.setRemoveSourceField(true);
meta.setIgnoreMissingPath(true);
meta.setRowLimit(2);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
processRows(jsonInput, 4);
Assert.assertEquals("errors", 0, jsonInput.getErrors());
Assert.assertEquals("lines written", 2, jsonInput.getLinesWritten());
}
protected JsonInput createJsonInput(
final String inCol, JsonInputMeta meta, Object[]... inputRows) {
JsonInputData data = new JsonInputData();
JsonInput jsonInput =
new JsonInput(helper.stepMeta, helper.stepDataInterface, 0, helper.transMeta, helper.trans);
RowSet input = helper.getMockInputRowSet(inputRows);
RowMetaInterface rowMeta = createRowMeta(new ValueMetaString(inCol));
input.setRowMeta(rowMeta);
jsonInput.getInputRowSets().add(input);
jsonInput.setInputRowMeta(rowMeta);
jsonInput.init(meta, data);
jsonInput.initializeVariablesFrom(null);
return jsonInput;
}
@Test
public void testFileList() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input1 = getBasicTestJson();
final String input2 = "{ \"store\": { \"book\": [ { \"price\": 9.99 } ] } }";
try (FileObject fileObj1 = KettleVFS.getFileObject(BASE_RAM_DIR + "test1.json");
FileObject fileObj2 = KettleVFS.getFileObject(BASE_RAM_DIR + "test2.json")) {
try (OutputStream out = fileObj1.getContent().getOutputStream()) {
out.write(input1.getBytes());
}
try (OutputStream out = fileObj2.getContent().getOutputStream()) {
out.write(input2.getBytes());
}
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
List<FileObject> fileList = Arrays.asList(fileObj1, fileObj2);
JsonInputMeta meta = createFileListMeta(fileList);
meta.setInputFields(new JsonInputField[] {price});
meta.setIncludeRowNumber(true);
meta.setRowNumberField("rownbr");
meta.setShortFileNameField("fname");
JsonInput jsonInput = createJsonInput(meta);
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {8.95d, 1L, "test1.json"},
new Object[] {12.99d, 2L, "test1.json"},
new Object[] {8.99d, 3L, "test1.json"},
new Object[] {22.99d, 4L, "test1.json"},
new Object[] {9.99d, 5L, "test2.json"});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
disposeJsonInput(jsonInput);
assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
@Test
public void testZipFileInput() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input = getBasicTestJson();
try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.zip")) {
fileObj.createFile();
try (OutputStream out = fileObj.getContent().getOutputStream()) {
try (ZipOutputStream zipOut = new ZipOutputStream(out)) {
ZipEntry jsonFile = new ZipEntry("test.json");
zipOut.putNextEntry(jsonFile);
zipOut.write(input.getBytes());
zipOut.closeEntry();
zipOut.flush();
}
}
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
JsonInputMeta meta = createSimpleMeta("in file", price);
meta.setIsAFile(true);
meta.setRemoveSourceField(true);
JsonInput jsonInput =
createJsonInput(
"in file",
meta,
new Object[][] {new Object[] {"zip:" + BASE_RAM_DIR + "test.zip!/test.json"}});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {8.95d},
new Object[] {12.99d},
new Object[] {8.99d},
new Object[] {22.99d});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
Assert.assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
@Test
public void testDefaultLeafToNull() throws Exception {
JsonInputField noPath = new JsonInputField("price");
noPath.setPath("$..price");
noPath.setType(ValueMetaInterface.TYPE_STRING);
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputMeta meta = createSimpleMeta("json", noPath);
meta.setIgnoreMissingPath(true);
meta.setRemoveSourceField(true);
final String input = getBasicTestJson();
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {input}, new Object[] {input});
processRows(jsonInput, 7);
disposeJsonInput(jsonInput);
Assert.assertEquals(5, jsonInput.getLinesWritten());
}
@Test
public void testBadExp() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
try (LocaleChange enUS = new LocaleChange(Locale.US)) {
JsonInput jsonInput =
createBasicTestJsonInput(
"$..fail", new ValueMetaString("result"), "json", new Object[] {getBasicTestJson()});
processRows(jsonInput, 2);
Assert.assertEquals("errors", 1, jsonInput.getErrors());
Assert.assertEquals("rows written", 0, jsonInput.getLinesWritten());
String expectedError = "We can not find any data with path [$..fail]";
String errors =
IOUtils.toString(
new ByteArrayInputStream(out.toByteArray()), StandardCharsets.UTF_8.name());
Assert.assertTrue("error", errors.contains(expectedError));
}
}
@Test
public void testSingleObjPred() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField bic = new JsonInputField("color");
bic.setPath("$.store.bicycle[?(@.price)].color");
bic.setType(ValueMetaInterface.TYPE_STRING);
JsonInputMeta meta = createSimpleMeta("json", bic);
meta.setRemoveSourceField(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
RowComparatorListener rowComparator = new RowComparatorListener(new Object[] {"red"});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 2);
Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
}
@Test
public void testDualExp() throws Exception {
JsonInputField isbn = new JsonInputField("isbn");
isbn.setPath("$..book[?(@.isbn)].isbn");
isbn.setType(ValueMetaInterface.TYPE_STRING);
JsonInputField price = new JsonInputField("price");
price.setPath("$..book[?(@.isbn)].price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
JsonInputMeta meta = createSimpleMeta("json", isbn, price);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {null, "0-553-21311-3", 8.99},
new Object[] {null, "0-395-19395-8", 22.99});
rowComparator.setComparator(0, null);
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 3);
Assert.assertEquals("error", 0, jsonInput.getErrors());
Assert.assertEquals("lines written", 2, jsonInput.getLinesWritten());
}
@Test
public void testBadInput() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField isbn = new JsonInputField("isbn");
isbn.setPath("$..book[?(@.isbn)].isbn");
isbn.setType(ValueMetaInterface.TYPE_STRING);
String input = "{{";
try (LocaleChange enUS = new LocaleChange(Locale.US)) {
JsonInputMeta meta = createSimpleMeta("json", isbn);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {input});
processRows(jsonInput, 3);
Assert.assertEquals("error", 1, jsonInput.getErrors());
Assert.assertEquals("rows written", 0, jsonInput.getLinesWritten());
String errors =
IOUtils.toString(
new ByteArrayInputStream(out.toByteArray()), StandardCharsets.UTF_8.name());
Assert.assertTrue("expected error", errors.contains("Error parsing string"));
}
}
@Test
public void testArrayOut() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField byc = new JsonInputField("books (array)");
byc.setPath("$.store.book");
byc.setType(ValueMetaInterface.TYPE_STRING);
JsonInputMeta meta = createSimpleMeta("json", byc);
meta.setRemoveSourceField(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {
"[{\"category\":\"reference\",\"author\":\"Nigel Rees\",\"title\":\"Sayings of the Century\",\"price\":8.95},{\"category\":\"fiction\",\"author\":\"Evelyn Waugh\",\"title\":\"Sword of Honour\",\"price\":12.99},{\"category\":\"fiction\",\"author\":\"Herman Melville\",\"title\":\"Moby Dick\",\"isbn\":\"0-553-21311-3\",\"price\":8.99},{\"category\":\"fiction\",\"author\":\"J. R. R. Tolkien\",\"title\":\"The Lord of the Rings\",\"isbn\":\"0-395-19395-8\",\"price\":22.99}]"
});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 2);
Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
}
protected void disposeJsonInput(JsonInput jsonInput) {
jsonInput.dispose(null, helper.stepDataInterface);
}
@Test
public void testExtraFileFields() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input1 = getBasicTestJson();
final String input2 = "{ \"store\": { \"bicycle\": { \"color\": \"blue\" } } }";
final String path1 = BASE_RAM_DIR + "test1.json";
final String path2 = BASE_RAM_DIR + "test2.js";
try (FileObject fileObj1 = KettleVFS.getFileObject(path1);
FileObject fileObj2 = KettleVFS.getFileObject(path2)) {
try (OutputStream out = fileObj1.getContent().getOutputStream()) {
out.write(input1.getBytes());
}
try (OutputStream out = fileObj2.getContent().getOutputStream()) {
out.write(input2.getBytes());
}
JsonInputField color = new JsonInputField();
color.setName("color");
color.setType(ValueMetaInterface.TYPE_STRING);
color.setPath("$.store.bicycle.color");
JsonInputMeta meta = createSimpleMeta("in file", color);
meta.setInFields(true);
meta.setIsAFile(true);
meta.setRemoveSourceField(true);
meta.setExtensionField("extension");
meta.setPathField("dir path");
meta.setSizeField("size");
meta.setIsHiddenField("hidden?");
meta.setLastModificationDateField("last modified");
meta.setUriField("URI");
meta.setRootUriField("root URI");
// custom checkers for size and last modified
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {
"red",
"json",
"ram:///jsonInputTest",
-1L,
false,
new Date(0),
"ram:///jsonInputTest/test1.json",
"ram:///"
},
new Object[] {
"blue",
"js",
"ram:///jsonInputTest",
-1L,
false,
new Date(0),
"ram:///jsonInputTest/test2.js",
"ram:///"
});
rowComparator.setComparator(
3,
new RowComparatorListener.Comparison<Object>() {
@Override
public boolean equals(Object expected, Object actual) throws Exception {
// just want a valid size
return ((long) actual) > 0L;
}
});
rowComparator.setComparator(
5,
new RowComparatorListener.Comparison<Object>() {
@Override
public boolean equals(Object expected, Object actual) throws Exception {
return ((Date) actual).after(new Date(0));
}
});
JsonInput jsonInput =
createJsonInput(
"in file", meta, new Object[][] {new Object[] {path1}, new Object[] {path2}});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 3);
Assert.assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}