@Test
  public void testErrorRedirect() throws Exception {
    JsonInputField field = new JsonInputField("value");
    field.setPath("$.value");
    field.setType(ValueMetaInterface.TYPE_STRING);

    String input1 = "{{";
    String input2 = "{ \"value\": \"ok\" }";

    JsonInputMeta meta = createSimpleMeta("json", field);
    meta.setRemoveSourceField(true);
    when(helper.stepMeta.isDoingErrorHandling()).thenReturn(true);
    JsonInput jsonInput =
        createJsonInput("json", meta, new Object[] {input1}, new Object[] {input2});
    StepErrorMeta errMeta = new StepErrorMeta(jsonInput, helper.stepMeta);
    errMeta.setEnabled(true);
    errMeta.setErrorFieldsValuename("err field");
    when(helper.stepMeta.getStepErrorMeta()).thenReturn(errMeta);
    final List<Object[]> errorLines = new ArrayList<>();
    jsonInput.addRowListener(
        new RowComparatorListener(new Object[] {"ok"}) {
          @Override
          public void errorRowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
              throws KettleStepException {
            errorLines.add(row);
          }
        });
    processRows(jsonInput, 3);
    Assert.assertEquals("fwd error", 1, errorLines.size());
    Assert.assertEquals("input in err line", input1, errorLines.get(0)[0]);
    Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
  }
  @Test
  public void testDualExpMismatchPathLeafToNull() throws Exception {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    helper.redirectLog(out, LogLevel.ERROR);

    JsonInputField isbn = new JsonInputField("isbn");
    isbn.setPath("$..book[*].isbn");
    isbn.setType(ValueMetaInterface.TYPE_STRING);
    JsonInputField price = new JsonInputField("price");
    price.setPath("$..book[*].price");
    price.setType(ValueMetaInterface.TYPE_NUMBER);

    JsonInputMeta meta = createSimpleMeta("json", isbn, price);
    meta.setRemoveSourceField(true);

    JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
    RowComparatorListener rowComparator =
        new RowComparatorListener(
            new Object[] {null, 8.95d},
            new Object[] {null, 12.99d},
            new Object[] {"0-553-21311-3", 8.99d},
            new Object[] {"0-395-19395-8", 22.99d});
    jsonInput.addRowListener(rowComparator);

    processRows(jsonInput, 5);

    Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
    Assert.assertEquals("rows written", 4, jsonInput.getLinesWritten());
  }
  @Test
  public void testRepeatFieldSingleObj() throws Exception {
    final String input =
        " { \"items\": [ "
            + "{ \"a\": 1, \"b\": null }, "
            + "{ \"a\":null, \"b\":2 }, "
            + "{ \"a\":3, \"b\":null }, "
            + "{ \"a\":4, \"b\":4 } ] }";
    final String inCol = "input";

    JsonInputField aField = new JsonInputField();
    aField.setName("a");
    aField.setPath("$.items[*].a");
    aField.setType(ValueMetaInterface.TYPE_INTEGER);
    JsonInputField bField = new JsonInputField();
    bField.setName("b");
    bField.setPath("$.items[*].b");
    bField.setType(ValueMetaInterface.TYPE_INTEGER);
    bField.setRepeated(true);

    JsonInputMeta meta = createSimpleMeta(inCol, aField, bField);
    JsonInput step = createJsonInput(inCol, meta, new Object[] {input});
    step.addRowListener(
        new RowComparatorListener(
            new Object[] {input, 1L, null},
            new Object[] {input, null, 2L},
            new Object[] {input, 3L, 2L},
            new Object[] {input, 4L, 4L}));
    processRows(step, 4);
    Assert.assertEquals(4, step.getLinesWritten());
  }
 @Test
 public void testIndexLastObj() throws Exception {
   final String jsonInputField = getBasicTestJson();
   JsonInput jsonInput =
       createBasicTestJsonInput(
           "$..book[-1:]",
           new ValueMetaString("last book"),
           "json",
           new Object[] {jsonInputField});
   RowComparatorListener rowComparator =
       new RowComparatorListener(
           new Object[] {
             jsonInputField,
             "{ \"category\": \"fiction\",\n"
                 + "  \"author\": \"J. R. R. Tolkien\",\n"
                 + "  \"title\": \"The Lord of the Rings\",\n"
                 + "  \"isbn\": \"0-395-19395-8\",\n"
                 + "  \"price\": 22.99\n"
                 + "}\n"
           });
   rowComparator.setComparator(1, new JsonComparison());
   jsonInput.addRowListener(rowComparator);
   processRows(jsonInput, 2);
   Assert.assertEquals(1, jsonInput.getLinesWritten());
 }
 @Test
 public void testNullProp() throws Exception {
   final String input = "{ \"obj\": [ { \"nval\": null, \"val\": 2 }, { \"val\": 1 } ] }";
   JsonInput jsonInput =
       createBasicTestJsonInput(
           "$.obj[?(@.nval)].val", new ValueMetaString("obj"), "json", new Object[] {input});
   RowComparatorListener rowComparator = new RowComparatorListener(new Object[] {input, "2"});
   rowComparator.setComparator(1, new JsonComparison());
   jsonInput.addRowListener(rowComparator);
   processRows(jsonInput, 2);
   // in jsonpath 2.0->2.1, null value properties started being counted as existing
   Assert.assertEquals(1, jsonInput.getLinesWritten());
 }
  protected void testSimpleJsonPath(
      String jsonPath, ValueMetaInterface outputMeta, Object[][] inputRows, Object[][] outputRows)
      throws Exception {
    final String inCol = "in";

    JsonInput jsonInput = createBasicTestJsonInput(jsonPath, outputMeta, inCol, inputRows);

    RowComparatorListener rowComparator = new RowComparatorListener(outputRows);

    jsonInput.addRowListener(rowComparator);
    processRows(jsonInput, outputRows.length + 1);
    Assert.assertEquals("rows written", outputRows.length, jsonInput.getLinesWritten());
    Assert.assertEquals("errors", 0, jsonInput.getErrors());
  }
 @Test
 public void testJgdArray() throws Exception {
   final String input =
       " { \"arr\": [ [ { \"a\": 1, \"b\": 1}, { \"a\": 1, \"b\": 2} ], [ {\"a\": 3, \"b\": 4 } ] ] }";
   JsonInput jsonInput =
       createBasicTestJsonInput(
           "$.arr", new ValueMetaString("array"), "json", new Object[] {input});
   RowComparatorListener rowComparator =
       new RowComparatorListener(
           new Object[] {input, "[[{\"a\":1,\"b\":1},{\"a\":1,\"b\":2}],[{\"a\":3,\"b\":4}]]"});
   rowComparator.setComparator(1, new JsonComparison());
   jsonInput.addRowListener(rowComparator);
   processRows(jsonInput, 2);
   Assert.assertEquals(1, jsonInput.getLinesWritten());
 }
  @Test
  public void testFileList() throws Exception {
    ByteArrayOutputStream err = new ByteArrayOutputStream();
    helper.redirectLog(err, LogLevel.ERROR);

    final String input1 = getBasicTestJson();
    final String input2 = "{ \"store\": { \"book\": [ { \"price\": 9.99 } ] } }";
    try (FileObject fileObj1 = KettleVFS.getFileObject(BASE_RAM_DIR + "test1.json");
        FileObject fileObj2 = KettleVFS.getFileObject(BASE_RAM_DIR + "test2.json")) {
      try (OutputStream out = fileObj1.getContent().getOutputStream()) {
        out.write(input1.getBytes());
      }
      try (OutputStream out = fileObj2.getContent().getOutputStream()) {
        out.write(input2.getBytes());
      }
      JsonInputField price = new JsonInputField();
      price.setName("price");
      price.setType(ValueMetaInterface.TYPE_NUMBER);
      price.setPath("$..book[*].price");
      List<FileObject> fileList = Arrays.asList(fileObj1, fileObj2);
      JsonInputMeta meta = createFileListMeta(fileList);
      meta.setInputFields(new JsonInputField[] {price});

      meta.setIncludeRowNumber(true);
      meta.setRowNumberField("rownbr");

      meta.setShortFileNameField("fname");

      JsonInput jsonInput = createJsonInput(meta);
      RowComparatorListener rowComparator =
          new RowComparatorListener(
              new Object[] {8.95d, 1L, "test1.json"},
              new Object[] {12.99d, 2L, "test1.json"},
              new Object[] {8.99d, 3L, "test1.json"},
              new Object[] {22.99d, 4L, "test1.json"},
              new Object[] {9.99d, 5L, "test2.json"});
      jsonInput.addRowListener(rowComparator);

      processRows(jsonInput, 5);
      disposeJsonInput(jsonInput);
      assertEquals(err.toString(), 0, jsonInput.getErrors());
    } finally {
      deleteFiles();
    }
  }
  @Test
  public void testRemoveSourceField() throws Exception {
    final String inCol = "json";
    JsonInputField jpath = new JsonInputField("isbn");
    jpath.setPath("$..book[*].isbn");
    jpath.setType(ValueMetaInterface.TYPE_STRING);

    JsonInputMeta meta = createSimpleMeta(inCol, jpath);
    meta.setRemoveSourceField(true);
    meta.setIgnoreMissingPath(true);
    JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
    RowComparatorListener rowComparator =
        new RowComparatorListener(new Object[] {"0-553-21311-3"}, new Object[] {"0-395-19395-8"});
    jsonInput.addRowListener(rowComparator);
    processRows(jsonInput, 4);
    Assert.assertEquals("errors", 0, jsonInput.getErrors());
    Assert.assertEquals("lines written", 2, jsonInput.getLinesWritten());
  }
  @Test
  public void testZipFileInput() throws Exception {
    ByteArrayOutputStream err = new ByteArrayOutputStream();
    helper.redirectLog(err, LogLevel.ERROR);

    final String input = getBasicTestJson();
    try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.zip")) {
      fileObj.createFile();
      try (OutputStream out = fileObj.getContent().getOutputStream()) {
        try (ZipOutputStream zipOut = new ZipOutputStream(out)) {
          ZipEntry jsonFile = new ZipEntry("test.json");
          zipOut.putNextEntry(jsonFile);
          zipOut.write(input.getBytes());
          zipOut.closeEntry();
          zipOut.flush();
        }
      }
      JsonInputField price = new JsonInputField();
      price.setName("price");
      price.setType(ValueMetaInterface.TYPE_NUMBER);
      price.setPath("$..book[*].price");

      JsonInputMeta meta = createSimpleMeta("in file", price);
      meta.setIsAFile(true);
      meta.setRemoveSourceField(true);
      JsonInput jsonInput =
          createJsonInput(
              "in file",
              meta,
              new Object[][] {new Object[] {"zip:" + BASE_RAM_DIR + "test.zip!/test.json"}});
      RowComparatorListener rowComparator =
          new RowComparatorListener(
              new Object[] {8.95d},
              new Object[] {12.99d},
              new Object[] {8.99d},
              new Object[] {22.99d});
      jsonInput.addRowListener(rowComparator);
      processRows(jsonInput, 5);
      Assert.assertEquals(err.toString(), 0, jsonInput.getErrors());
    } finally {
      deleteFiles();
    }
  }
  @Test
  public void testSingleObjPred() throws Exception {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    helper.redirectLog(out, LogLevel.ERROR);

    JsonInputField bic = new JsonInputField("color");
    bic.setPath("$.store.bicycle[?(@.price)].color");
    bic.setType(ValueMetaInterface.TYPE_STRING);

    JsonInputMeta meta = createSimpleMeta("json", bic);
    meta.setRemoveSourceField(true);

    JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
    RowComparatorListener rowComparator = new RowComparatorListener(new Object[] {"red"});
    jsonInput.addRowListener(rowComparator);

    processRows(jsonInput, 2);
    Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
    Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
  }
  @Test
  public void testDualExp() throws Exception {
    JsonInputField isbn = new JsonInputField("isbn");
    isbn.setPath("$..book[?(@.isbn)].isbn");
    isbn.setType(ValueMetaInterface.TYPE_STRING);
    JsonInputField price = new JsonInputField("price");
    price.setPath("$..book[?(@.isbn)].price");
    price.setType(ValueMetaInterface.TYPE_NUMBER);

    JsonInputMeta meta = createSimpleMeta("json", isbn, price);
    JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
    RowComparatorListener rowComparator =
        new RowComparatorListener(
            new Object[] {null, "0-553-21311-3", 8.99},
            new Object[] {null, "0-395-19395-8", 22.99});
    rowComparator.setComparator(0, null);
    jsonInput.addRowListener(rowComparator);
    processRows(jsonInput, 3);
    Assert.assertEquals("error", 0, jsonInput.getErrors());
    Assert.assertEquals("lines written", 2, jsonInput.getLinesWritten());
  }
  @Test
  public void testArrayOut() throws Exception {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    helper.redirectLog(out, LogLevel.ERROR);

    JsonInputField byc = new JsonInputField("books (array)");
    byc.setPath("$.store.book");
    byc.setType(ValueMetaInterface.TYPE_STRING);

    JsonInputMeta meta = createSimpleMeta("json", byc);
    meta.setRemoveSourceField(true);

    JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
    RowComparatorListener rowComparator =
        new RowComparatorListener(
            new Object[] {
              "[{\"category\":\"reference\",\"author\":\"Nigel Rees\",\"title\":\"Sayings of the Century\",\"price\":8.95},{\"category\":\"fiction\",\"author\":\"Evelyn Waugh\",\"title\":\"Sword of Honour\",\"price\":12.99},{\"category\":\"fiction\",\"author\":\"Herman Melville\",\"title\":\"Moby Dick\",\"isbn\":\"0-553-21311-3\",\"price\":8.99},{\"category\":\"fiction\",\"author\":\"J. R. R. Tolkien\",\"title\":\"The Lord of the Rings\",\"isbn\":\"0-395-19395-8\",\"price\":22.99}]"
            });
    jsonInput.addRowListener(rowComparator);

    processRows(jsonInput, 2);
    Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
    Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
  }
  @Test
  public void testExtraFileFields() throws Exception {
    ByteArrayOutputStream err = new ByteArrayOutputStream();
    helper.redirectLog(err, LogLevel.ERROR);

    final String input1 = getBasicTestJson();
    final String input2 = "{ \"store\": { \"bicycle\": { \"color\": \"blue\" } } }";
    final String path1 = BASE_RAM_DIR + "test1.json";
    final String path2 = BASE_RAM_DIR + "test2.js";
    try (FileObject fileObj1 = KettleVFS.getFileObject(path1);
        FileObject fileObj2 = KettleVFS.getFileObject(path2)) {
      try (OutputStream out = fileObj1.getContent().getOutputStream()) {
        out.write(input1.getBytes());
      }
      try (OutputStream out = fileObj2.getContent().getOutputStream()) {
        out.write(input2.getBytes());
      }

      JsonInputField color = new JsonInputField();
      color.setName("color");
      color.setType(ValueMetaInterface.TYPE_STRING);
      color.setPath("$.store.bicycle.color");

      JsonInputMeta meta = createSimpleMeta("in file", color);
      meta.setInFields(true);
      meta.setIsAFile(true);
      meta.setRemoveSourceField(true);

      meta.setExtensionField("extension");
      meta.setPathField("dir path");
      meta.setSizeField("size");
      meta.setIsHiddenField("hidden?");
      meta.setLastModificationDateField("last modified");
      meta.setUriField("URI");
      meta.setRootUriField("root URI");

      // custom checkers for size and last modified
      RowComparatorListener rowComparator =
          new RowComparatorListener(
              new Object[] {
                "red",
                "json",
                "ram:///jsonInputTest",
                -1L,
                false,
                new Date(0),
                "ram:///jsonInputTest/test1.json",
                "ram:///"
              },
              new Object[] {
                "blue",
                "js",
                "ram:///jsonInputTest",
                -1L,
                false,
                new Date(0),
                "ram:///jsonInputTest/test2.js",
                "ram:///"
              });
      rowComparator.setComparator(
          3,
          new RowComparatorListener.Comparison<Object>() {
            @Override
            public boolean equals(Object expected, Object actual) throws Exception {
              // just want a valid size
              return ((long) actual) > 0L;
            }
          });
      rowComparator.setComparator(
          5,
          new RowComparatorListener.Comparison<Object>() {
            @Override
            public boolean equals(Object expected, Object actual) throws Exception {
              return ((Date) actual).after(new Date(0));
            }
          });
      JsonInput jsonInput =
          createJsonInput(
              "in file", meta, new Object[][] {new Object[] {path1}, new Object[] {path2}});
      jsonInput.addRowListener(rowComparator);
      processRows(jsonInput, 3);
      Assert.assertEquals(err.toString(), 0, jsonInput.getErrors());
    } finally {
      deleteFiles();
    }
  }