@Test public void testZeroSizeFile() throws Exception { ByteArrayOutputStream log = new ByteArrayOutputStream(); helper.redirectLog(log, LogLevel.BASIC); try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.json"); LocaleChange enUs = new LocaleChange(Locale.US); ) { fileObj.createFile(); JsonInputField price = new JsonInputField(); price.setName("price"); price.setType(ValueMetaInterface.TYPE_NUMBER); price.setPath("$..book[*].price"); JsonInputMeta meta = createSimpleMeta("in file", price); meta.setIsAFile(true); meta.setRemoveSourceField(true); meta.setIgnoreEmptyFile(false); JsonInput jsonInput = createJsonInput( "in file", meta, new Object[][] {new Object[] {BASE_RAM_DIR + "test.json"}}); processRows(jsonInput, 1); String logMsgs = log.toString(); assertTrue(logMsgs, logMsgs.contains("is empty!")); } finally { deleteFiles(); } }
@Test public void testErrorRedirect() throws Exception { JsonInputField field = new JsonInputField("value"); field.setPath("$.value"); field.setType(ValueMetaInterface.TYPE_STRING); String input1 = "{{"; String input2 = "{ \"value\": \"ok\" }"; JsonInputMeta meta = createSimpleMeta("json", field); meta.setRemoveSourceField(true); when(helper.stepMeta.isDoingErrorHandling()).thenReturn(true); JsonInput jsonInput = createJsonInput("json", meta, new Object[] {input1}, new Object[] {input2}); StepErrorMeta errMeta = new StepErrorMeta(jsonInput, helper.stepMeta); errMeta.setEnabled(true); errMeta.setErrorFieldsValuename("err field"); when(helper.stepMeta.getStepErrorMeta()).thenReturn(errMeta); final List<Object[]> errorLines = new ArrayList<>(); jsonInput.addRowListener( new RowComparatorListener(new Object[] {"ok"}) { @Override public void errorRowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException { errorLines.add(row); } }); processRows(jsonInput, 3); Assert.assertEquals("fwd error", 1, errorLines.size()); Assert.assertEquals("input in err line", input1, errorLines.get(0)[0]); Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten()); }
@Test public void testDualExpMismatchPathLeafToNull() throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); helper.redirectLog(out, LogLevel.ERROR); JsonInputField isbn = new JsonInputField("isbn"); isbn.setPath("$..book[*].isbn"); isbn.setType(ValueMetaInterface.TYPE_STRING); JsonInputField price = new JsonInputField("price"); price.setPath("$..book[*].price"); price.setType(ValueMetaInterface.TYPE_NUMBER); JsonInputMeta meta = createSimpleMeta("json", isbn, price); meta.setRemoveSourceField(true); JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()}); RowComparatorListener rowComparator = new RowComparatorListener( new Object[] {null, 8.95d}, new Object[] {null, 12.99d}, new Object[] {"0-553-21311-3", 8.99d}, new Object[] {"0-395-19395-8", 22.99d}); jsonInput.addRowListener(rowComparator); processRows(jsonInput, 5); Assert.assertEquals(out.toString(), 0, jsonInput.getErrors()); Assert.assertEquals("rows written", 4, jsonInput.getLinesWritten()); }
@Test public void testRowLimit() throws Exception { final String inCol = "json"; JsonInputField jpath = new JsonInputField("isbn"); jpath.setPath("$..book[*].isbn"); jpath.setType(ValueMetaInterface.TYPE_STRING); JsonInputMeta meta = createSimpleMeta(inCol, jpath); meta.setRemoveSourceField(true); meta.setIgnoreMissingPath(true); meta.setRowLimit(2); JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()}); processRows(jsonInput, 4); Assert.assertEquals("errors", 0, jsonInput.getErrors()); Assert.assertEquals("lines written", 2, jsonInput.getLinesWritten()); }
@Test public void testZipFileInput() throws Exception { ByteArrayOutputStream err = new ByteArrayOutputStream(); helper.redirectLog(err, LogLevel.ERROR); final String input = getBasicTestJson(); try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.zip")) { fileObj.createFile(); try (OutputStream out = fileObj.getContent().getOutputStream()) { try (ZipOutputStream zipOut = new ZipOutputStream(out)) { ZipEntry jsonFile = new ZipEntry("test.json"); zipOut.putNextEntry(jsonFile); zipOut.write(input.getBytes()); zipOut.closeEntry(); zipOut.flush(); } } JsonInputField price = new JsonInputField(); price.setName("price"); price.setType(ValueMetaInterface.TYPE_NUMBER); price.setPath("$..book[*].price"); JsonInputMeta meta = createSimpleMeta("in file", price); meta.setIsAFile(true); meta.setRemoveSourceField(true); JsonInput jsonInput = createJsonInput( "in file", meta, new Object[][] {new Object[] {"zip:" + BASE_RAM_DIR + "test.zip!/test.json"}}); RowComparatorListener rowComparator = new RowComparatorListener( new Object[] {8.95d}, new Object[] {12.99d}, new Object[] {8.99d}, new Object[] {22.99d}); jsonInput.addRowListener(rowComparator); processRows(jsonInput, 5); Assert.assertEquals(err.toString(), 0, jsonInput.getErrors()); } finally { deleteFiles(); } }
@Test public void testDefaultLeafToNull() throws Exception { JsonInputField noPath = new JsonInputField("price"); noPath.setPath("$..price"); noPath.setType(ValueMetaInterface.TYPE_STRING); ByteArrayOutputStream out = new ByteArrayOutputStream(); helper.redirectLog(out, LogLevel.ERROR); JsonInputMeta meta = createSimpleMeta("json", noPath); meta.setIgnoreMissingPath(true); meta.setRemoveSourceField(true); final String input = getBasicTestJson(); JsonInput jsonInput = createJsonInput("json", meta, new Object[] {input}, new Object[] {input}); processRows(jsonInput, 7); disposeJsonInput(jsonInput); Assert.assertEquals(5, jsonInput.getLinesWritten()); }
@Test public void testSingleObjPred() throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); helper.redirectLog(out, LogLevel.ERROR); JsonInputField bic = new JsonInputField("color"); bic.setPath("$.store.bicycle[?(@.price)].color"); bic.setType(ValueMetaInterface.TYPE_STRING); JsonInputMeta meta = createSimpleMeta("json", bic); meta.setRemoveSourceField(true); JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()}); RowComparatorListener rowComparator = new RowComparatorListener(new Object[] {"red"}); jsonInput.addRowListener(rowComparator); processRows(jsonInput, 2); Assert.assertEquals(out.toString(), 0, jsonInput.getErrors()); Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten()); }
@Test public void testArrayOut() throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); helper.redirectLog(out, LogLevel.ERROR); JsonInputField byc = new JsonInputField("books (array)"); byc.setPath("$.store.book"); byc.setType(ValueMetaInterface.TYPE_STRING); JsonInputMeta meta = createSimpleMeta("json", byc); meta.setRemoveSourceField(true); JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()}); RowComparatorListener rowComparator = new RowComparatorListener( new Object[] { "[{\"category\":\"reference\",\"author\":\"Nigel Rees\",\"title\":\"Sayings of the Century\",\"price\":8.95},{\"category\":\"fiction\",\"author\":\"Evelyn Waugh\",\"title\":\"Sword of Honour\",\"price\":12.99},{\"category\":\"fiction\",\"author\":\"Herman Melville\",\"title\":\"Moby Dick\",\"isbn\":\"0-553-21311-3\",\"price\":8.99},{\"category\":\"fiction\",\"author\":\"J. R. R. Tolkien\",\"title\":\"The Lord of the Rings\",\"isbn\":\"0-395-19395-8\",\"price\":22.99}]" }); jsonInput.addRowListener(rowComparator); processRows(jsonInput, 2); Assert.assertEquals(out.toString(), 0, jsonInput.getErrors()); Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten()); }
@Test public void testExtraFileFields() throws Exception { ByteArrayOutputStream err = new ByteArrayOutputStream(); helper.redirectLog(err, LogLevel.ERROR); final String input1 = getBasicTestJson(); final String input2 = "{ \"store\": { \"bicycle\": { \"color\": \"blue\" } } }"; final String path1 = BASE_RAM_DIR + "test1.json"; final String path2 = BASE_RAM_DIR + "test2.js"; try (FileObject fileObj1 = KettleVFS.getFileObject(path1); FileObject fileObj2 = KettleVFS.getFileObject(path2)) { try (OutputStream out = fileObj1.getContent().getOutputStream()) { out.write(input1.getBytes()); } try (OutputStream out = fileObj2.getContent().getOutputStream()) { out.write(input2.getBytes()); } JsonInputField color = new JsonInputField(); color.setName("color"); color.setType(ValueMetaInterface.TYPE_STRING); color.setPath("$.store.bicycle.color"); JsonInputMeta meta = createSimpleMeta("in file", color); meta.setInFields(true); meta.setIsAFile(true); meta.setRemoveSourceField(true); meta.setExtensionField("extension"); meta.setPathField("dir path"); meta.setSizeField("size"); meta.setIsHiddenField("hidden?"); meta.setLastModificationDateField("last modified"); meta.setUriField("URI"); meta.setRootUriField("root URI"); // custom checkers for size and last modified RowComparatorListener rowComparator = new RowComparatorListener( new Object[] { "red", "json", "ram:///jsonInputTest", -1L, false, new Date(0), "ram:///jsonInputTest/test1.json", "ram:///" }, new Object[] { "blue", "js", "ram:///jsonInputTest", -1L, false, new Date(0), "ram:///jsonInputTest/test2.js", "ram:///" }); rowComparator.setComparator( 3, new RowComparatorListener.Comparison<Object>() { @Override public boolean equals(Object expected, Object actual) throws Exception { // just want a valid size return ((long) actual) > 0L; } }); rowComparator.setComparator( 5, new RowComparatorListener.Comparison<Object>() { @Override public boolean equals(Object expected, Object actual) throws Exception { return ((Date) actual).after(new Date(0)); } }); JsonInput jsonInput = createJsonInput( "in file", meta, new Object[][] {new Object[] {path1}, new Object[] {path2}}); jsonInput.addRowListener(rowComparator); processRows(jsonInput, 3); Assert.assertEquals(err.toString(), 0, jsonInput.getErrors()); } finally { deleteFiles(); } }