@Test
public void testDualExpMismatchPathLeafToNull() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField isbn = new JsonInputField("isbn");
isbn.setPath("$..book[*].isbn");
isbn.setType(ValueMetaInterface.TYPE_STRING);
JsonInputField price = new JsonInputField("price");
price.setPath("$..book[*].price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
JsonInputMeta meta = createSimpleMeta("json", isbn, price);
meta.setRemoveSourceField(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {null, 8.95d},
new Object[] {null, 12.99d},
new Object[] {"0-553-21311-3", 8.99d},
new Object[] {"0-395-19395-8", 22.99d});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
Assert.assertEquals("rows written", 4, jsonInput.getLinesWritten());
}
/** File tests */
@Test
public void testNullFileList() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
try {
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
List<FileObject> fileList = Arrays.asList(null, null);
JsonInputMeta meta = createFileListMeta(fileList);
meta.setInputFields(new JsonInputField[] {price});
meta.setIncludeRowNumber(true);
meta.setRowNumberField("rownbr");
meta.setShortFileNameField("fname");
JsonInput jsonInput = createJsonInput(meta);
processRows(jsonInput, 5);
disposeJsonInput(jsonInput);
assertEquals(err.toString(), 2, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
@Test
public void testDualExpMismatchError() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField isbn = new JsonInputField("isbn");
isbn.setPath("$..book[?(@.isbn)].isbn");
isbn.setType(ValueMetaInterface.TYPE_STRING);
JsonInputField price = new JsonInputField("price");
price.setPath("$..book[*].price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
try (LocaleChange enUS = new LocaleChange(Locale.US)) {
JsonInputMeta meta = createSimpleMeta("json", isbn, price);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
processRows(jsonInput, 3);
Assert.assertEquals("error", 1, jsonInput.getErrors());
Assert.assertEquals("rows written", 0, jsonInput.getLinesWritten());
String errors =
IOUtils.toString(
new ByteArrayInputStream(out.toByteArray()), StandardCharsets.UTF_8.name());
String expectedError =
"The data structure is not the same inside the resource!"
+ " We found 4 values for json path [$..book[*].price],"
+ " which is different that the number returned for path [$..book[?(@.isbn)].isbn] (2 values)."
+ " We MUST have the same number of values for all paths.";
Assert.assertTrue("expected error", errors.contains(expectedError));
}
}
protected void testSimpleJsonPath(
String jsonPath, ValueMetaInterface outputMeta, Object[][] inputRows, Object[][] outputRows)
throws Exception {
final String inCol = "in";
JsonInput jsonInput = createBasicTestJsonInput(jsonPath, outputMeta, inCol, inputRows);
RowComparatorListener rowComparator = new RowComparatorListener(outputRows);
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, outputRows.length + 1);
Assert.assertEquals("rows written", outputRows.length, jsonInput.getLinesWritten());
Assert.assertEquals("errors", 0, jsonInput.getErrors());
}
@Test
public void testUrlInput() throws Exception {
JsonInputField field = new JsonInputField("value");
field.setPath("$.value");
field.setType(ValueMetaInterface.TYPE_STRING);
String input1 = "http://localhost/test.json";
JsonInputMeta meta = createSimpleMeta("json", field);
meta.setReadUrl(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {input1});
processRows(jsonInput, 3);
Assert.assertEquals(1, jsonInput.getErrors());
}
@Test
public void testRowLimit() throws Exception {
final String inCol = "json";
JsonInputField jpath = new JsonInputField("isbn");
jpath.setPath("$..book[*].isbn");
jpath.setType(ValueMetaInterface.TYPE_STRING);
JsonInputMeta meta = createSimpleMeta(inCol, jpath);
meta.setRemoveSourceField(true);
meta.setIgnoreMissingPath(true);
meta.setRowLimit(2);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
processRows(jsonInput, 4);
Assert.assertEquals("errors", 0, jsonInput.getErrors());
Assert.assertEquals("lines written", 2, jsonInput.getLinesWritten());
}
@Test
public void testFileList() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input1 = getBasicTestJson();
final String input2 = "{ \"store\": { \"book\": [ { \"price\": 9.99 } ] } }";
try (FileObject fileObj1 = KettleVFS.getFileObject(BASE_RAM_DIR + "test1.json");
FileObject fileObj2 = KettleVFS.getFileObject(BASE_RAM_DIR + "test2.json")) {
try (OutputStream out = fileObj1.getContent().getOutputStream()) {
out.write(input1.getBytes());
}
try (OutputStream out = fileObj2.getContent().getOutputStream()) {
out.write(input2.getBytes());
}
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
List<FileObject> fileList = Arrays.asList(fileObj1, fileObj2);
JsonInputMeta meta = createFileListMeta(fileList);
meta.setInputFields(new JsonInputField[] {price});
meta.setIncludeRowNumber(true);
meta.setRowNumberField("rownbr");
meta.setShortFileNameField("fname");
JsonInput jsonInput = createJsonInput(meta);
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {8.95d, 1L, "test1.json"},
new Object[] {12.99d, 2L, "test1.json"},
new Object[] {8.99d, 3L, "test1.json"},
new Object[] {22.99d, 4L, "test1.json"},
new Object[] {9.99d, 5L, "test2.json"});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
disposeJsonInput(jsonInput);
assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
@Test
public void testZipFileInput() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input = getBasicTestJson();
try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.zip")) {
fileObj.createFile();
try (OutputStream out = fileObj.getContent().getOutputStream()) {
try (ZipOutputStream zipOut = new ZipOutputStream(out)) {
ZipEntry jsonFile = new ZipEntry("test.json");
zipOut.putNextEntry(jsonFile);
zipOut.write(input.getBytes());
zipOut.closeEntry();
zipOut.flush();
}
}
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
JsonInputMeta meta = createSimpleMeta("in file", price);
meta.setIsAFile(true);
meta.setRemoveSourceField(true);
JsonInput jsonInput =
createJsonInput(
"in file",
meta,
new Object[][] {new Object[] {"zip:" + BASE_RAM_DIR + "test.zip!/test.json"}});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {8.95d},
new Object[] {12.99d},
new Object[] {8.99d},
new Object[] {22.99d});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
Assert.assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
@Test
public void testSingleObjPred() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField bic = new JsonInputField("color");
bic.setPath("$.store.bicycle[?(@.price)].color");
bic.setType(ValueMetaInterface.TYPE_STRING);
JsonInputMeta meta = createSimpleMeta("json", bic);
meta.setRemoveSourceField(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
RowComparatorListener rowComparator = new RowComparatorListener(new Object[] {"red"});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 2);
Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
}
@Test
public void testBadExp() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
try (LocaleChange enUS = new LocaleChange(Locale.US)) {
JsonInput jsonInput =
createBasicTestJsonInput(
"$..fail", new ValueMetaString("result"), "json", new Object[] {getBasicTestJson()});
processRows(jsonInput, 2);
Assert.assertEquals("errors", 1, jsonInput.getErrors());
Assert.assertEquals("rows written", 0, jsonInput.getLinesWritten());
String expectedError = "We can not find any data with path [$..fail]";
String errors =
IOUtils.toString(
new ByteArrayInputStream(out.toByteArray()), StandardCharsets.UTF_8.name());
Assert.assertTrue("error", errors.contains(expectedError));
}
}
@Test
public void testDualExp() throws Exception {
JsonInputField isbn = new JsonInputField("isbn");
isbn.setPath("$..book[?(@.isbn)].isbn");
isbn.setType(ValueMetaInterface.TYPE_STRING);
JsonInputField price = new JsonInputField("price");
price.setPath("$..book[?(@.isbn)].price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
JsonInputMeta meta = createSimpleMeta("json", isbn, price);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {null, "0-553-21311-3", 8.99},
new Object[] {null, "0-395-19395-8", 22.99});
rowComparator.setComparator(0, null);
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 3);
Assert.assertEquals("error", 0, jsonInput.getErrors());
Assert.assertEquals("lines written", 2, jsonInput.getLinesWritten());
}
@Test
public void testBadInput() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField isbn = new JsonInputField("isbn");
isbn.setPath("$..book[?(@.isbn)].isbn");
isbn.setType(ValueMetaInterface.TYPE_STRING);
String input = "{{";
try (LocaleChange enUS = new LocaleChange(Locale.US)) {
JsonInputMeta meta = createSimpleMeta("json", isbn);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {input});
processRows(jsonInput, 3);
Assert.assertEquals("error", 1, jsonInput.getErrors());
Assert.assertEquals("rows written", 0, jsonInput.getLinesWritten());
String errors =
IOUtils.toString(
new ByteArrayInputStream(out.toByteArray()), StandardCharsets.UTF_8.name());
Assert.assertTrue("expected error", errors.contains("Error parsing string"));
}
}
@Test
public void testArrayOut() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField byc = new JsonInputField("books (array)");
byc.setPath("$.store.book");
byc.setType(ValueMetaInterface.TYPE_STRING);
JsonInputMeta meta = createSimpleMeta("json", byc);
meta.setRemoveSourceField(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {
"[{\"category\":\"reference\",\"author\":\"Nigel Rees\",\"title\":\"Sayings of the Century\",\"price\":8.95},{\"category\":\"fiction\",\"author\":\"Evelyn Waugh\",\"title\":\"Sword of Honour\",\"price\":12.99},{\"category\":\"fiction\",\"author\":\"Herman Melville\",\"title\":\"Moby Dick\",\"isbn\":\"0-553-21311-3\",\"price\":8.99},{\"category\":\"fiction\",\"author\":\"J. R. R. Tolkien\",\"title\":\"The Lord of the Rings\",\"isbn\":\"0-395-19395-8\",\"price\":22.99}]"
});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 2);
Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
}
@Test
public void testExtraFileFields() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input1 = getBasicTestJson();
final String input2 = "{ \"store\": { \"bicycle\": { \"color\": \"blue\" } } }";
final String path1 = BASE_RAM_DIR + "test1.json";
final String path2 = BASE_RAM_DIR + "test2.js";
try (FileObject fileObj1 = KettleVFS.getFileObject(path1);
FileObject fileObj2 = KettleVFS.getFileObject(path2)) {
try (OutputStream out = fileObj1.getContent().getOutputStream()) {
out.write(input1.getBytes());
}
try (OutputStream out = fileObj2.getContent().getOutputStream()) {
out.write(input2.getBytes());
}
JsonInputField color = new JsonInputField();
color.setName("color");
color.setType(ValueMetaInterface.TYPE_STRING);
color.setPath("$.store.bicycle.color");
JsonInputMeta meta = createSimpleMeta("in file", color);
meta.setInFields(true);
meta.setIsAFile(true);
meta.setRemoveSourceField(true);
meta.setExtensionField("extension");
meta.setPathField("dir path");
meta.setSizeField("size");
meta.setIsHiddenField("hidden?");
meta.setLastModificationDateField("last modified");
meta.setUriField("URI");
meta.setRootUriField("root URI");
// custom checkers for size and last modified
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {
"red",
"json",
"ram:///jsonInputTest",
-1L,
false,
new Date(0),
"ram:///jsonInputTest/test1.json",
"ram:///"
},
new Object[] {
"blue",
"js",
"ram:///jsonInputTest",
-1L,
false,
new Date(0),
"ram:///jsonInputTest/test2.js",
"ram:///"
});
rowComparator.setComparator(
3,
new RowComparatorListener.Comparison<Object>() {
@Override
public boolean equals(Object expected, Object actual) throws Exception {
// just want a valid size
return ((long) actual) > 0L;
}
});
rowComparator.setComparator(
5,
new RowComparatorListener.Comparison<Object>() {
@Override
public boolean equals(Object expected, Object actual) throws Exception {
return ((Date) actual).after(new Date(0));
}
});
JsonInput jsonInput =
createJsonInput(
"in file", meta, new Object[][] {new Object[] {path1}, new Object[] {path2}});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 3);
Assert.assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}