/** File tests */
@Test
public void testNullFileList() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
try {
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
List<FileObject> fileList = Arrays.asList(null, null);
JsonInputMeta meta = createFileListMeta(fileList);
meta.setInputFields(new JsonInputField[] {price});
meta.setIncludeRowNumber(true);
meta.setRowNumberField("rownbr");
meta.setShortFileNameField("fname");
JsonInput jsonInput = createJsonInput(meta);
processRows(jsonInput, 5);
disposeJsonInput(jsonInput);
assertEquals(err.toString(), 2, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
@Test
public void testErrorRedirect() throws Exception {
JsonInputField field = new JsonInputField("value");
field.setPath("$.value");
field.setType(ValueMetaInterface.TYPE_STRING);
String input1 = "{{";
String input2 = "{ \"value\": \"ok\" }";
JsonInputMeta meta = createSimpleMeta("json", field);
meta.setRemoveSourceField(true);
when(helper.stepMeta.isDoingErrorHandling()).thenReturn(true);
JsonInput jsonInput =
createJsonInput("json", meta, new Object[] {input1}, new Object[] {input2});
StepErrorMeta errMeta = new StepErrorMeta(jsonInput, helper.stepMeta);
errMeta.setEnabled(true);
errMeta.setErrorFieldsValuename("err field");
when(helper.stepMeta.getStepErrorMeta()).thenReturn(errMeta);
final List<Object[]> errorLines = new ArrayList<>();
jsonInput.addRowListener(
new RowComparatorListener(new Object[] {"ok"}) {
@Override
public void errorRowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
throws KettleStepException {
errorLines.add(row);
}
});
processRows(jsonInput, 3);
Assert.assertEquals("fwd error", 1, errorLines.size());
Assert.assertEquals("input in err line", input1, errorLines.get(0)[0]);
Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
}
@Test
public void testDualExpMismatchPathLeafToNull() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField isbn = new JsonInputField("isbn");
isbn.setPath("$..book[*].isbn");
isbn.setType(ValueMetaInterface.TYPE_STRING);
JsonInputField price = new JsonInputField("price");
price.setPath("$..book[*].price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
JsonInputMeta meta = createSimpleMeta("json", isbn, price);
meta.setRemoveSourceField(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {null, 8.95d},
new Object[] {null, 12.99d},
new Object[] {"0-553-21311-3", 8.99d},
new Object[] {"0-395-19395-8", 22.99d});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
Assert.assertEquals("rows written", 4, jsonInput.getLinesWritten());
}
@Test
public void testZeroSizeFile() throws Exception {
ByteArrayOutputStream log = new ByteArrayOutputStream();
helper.redirectLog(log, LogLevel.BASIC);
try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.json");
LocaleChange enUs = new LocaleChange(Locale.US); ) {
fileObj.createFile();
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
JsonInputMeta meta = createSimpleMeta("in file", price);
meta.setIsAFile(true);
meta.setRemoveSourceField(true);
meta.setIgnoreEmptyFile(false);
JsonInput jsonInput =
createJsonInput(
"in file", meta, new Object[][] {new Object[] {BASE_RAM_DIR + "test.json"}});
processRows(jsonInput, 1);
String logMsgs = log.toString();
assertTrue(logMsgs, logMsgs.contains("is empty!"));
} finally {
deleteFiles();
}
}
public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
meta = (JsonInputMeta) smi;
data = (JsonInputData) sdi;
if (super.init(smi, sdi)) {
data.rownr = 1L;
data.nrInputFields = meta.getInputFields().length;
// Take care of variable substitution
for (int i = 0; i < data.nrInputFields; i++) {
JsonInputField field = meta.getInputFields()[i];
field.setPath(environmentSubstitute(field.getPath()));
}
try {
// Init a new JSON reader
data.jsonReader = new JsonReader();
data.jsonReader.SetIgnoreMissingPath(meta.isIgnoreMissingPath());
} catch (KettleException e) {
logError(e.getMessage());
return false;
}
return true;
}
return false;
}
private void parseJson() throws Exception {
// Read JSON source
if (data.file != null) {
data.jsonReader.readFile(data.filename);
} else {
if (meta.isReadUrl()) {
data.jsonReader.readUrl(data.stringToParse);
} else {
// read string
data.jsonReader.readString(data.stringToParse);
}
}
List<NJSONArray> resultList = new ArrayList<NJSONArray>();
data.nrrecords = -1;
data.recordnr = 0;
String prevPath = "";
for (int i = 0; i < data.nrInputFields; i++) {
String path = meta.getInputFields()[i].getPath();
NJSONArray ja = data.jsonReader.getPath(path);
if (data.nrrecords != -1 && data.nrrecords != ja.size() && !ja.isNull()) {
throw new KettleException(
BaseMessages.getString(
PKG, "JsonInput.Error.BadStructure", ja.size(), path, prevPath, data.nrrecords));
}
resultList.add(ja);
if (data.nrrecords == -1 && !ja.isNull()) {
data.nrrecords = ja.size();
}
prevPath = path;
}
data.resultList = new ArrayList<NJSONArray>();
Iterator<NJSONArray> it = resultList.iterator();
while (it.hasNext()) {
NJSONArray j = it.next();
if (j.isNull()) {
if (data.nrrecords == -1) {
data.nrrecords = 1;
}
// The object is empty means that we do not
// find Json path
// We need here to create a dummy structure
j = new NJSONArray();
for (int i = 0; i < data.nrrecords; i++) {
j.add(null);
}
}
data.resultList.add(j);
}
resultList = null;
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JsonInput.Log.NrRecords", data.nrrecords));
}
}
@Test
public void testUrlInput() throws Exception {
JsonInputField field = new JsonInputField("value");
field.setPath("$.value");
field.setType(ValueMetaInterface.TYPE_STRING);
String input1 = "http://localhost/test.json";
JsonInputMeta meta = createSimpleMeta("json", field);
meta.setReadUrl(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {input1});
processRows(jsonInput, 3);
Assert.assertEquals(1, jsonInput.getErrors());
}
private boolean openNextFile() {
try {
if (data.filenr >= data.files.nrOfFiles()) {
if (log.isDetailed())
logDetailed(BaseMessages.getString(PKG, "JsonInput.Log.FinishedProcessing"));
return false;
}
// Close previous file if needed
if (data.file != null) data.file.close();
// get file
data.file = (FileObject) data.files.getFile(data.filenr);
if (meta.isIgnoreEmptyFile() && data.file.getContent().getSize() == 0) {
// log only basic as a warning (was before logError)
logBasic(
BaseMessages.getString(PKG, "JsonInput.Error.FileSizeZero", "" + data.file.getName()));
openNextFile();
}
readFileOrString();
} catch (Exception e) {
logError(
BaseMessages.getString(
PKG,
"JsonInput.Log.UnableToOpenFile",
"" + data.filenr,
data.file.toString(),
e.toString()));
stopAll();
setErrors(1);
return false;
}
return true;
}
@Test
public void testRowLimit() throws Exception {
final String inCol = "json";
JsonInputField jpath = new JsonInputField("isbn");
jpath.setPath("$..book[*].isbn");
jpath.setType(ValueMetaInterface.TYPE_STRING);
JsonInputMeta meta = createSimpleMeta(inCol, jpath);
meta.setRemoveSourceField(true);
meta.setIgnoreMissingPath(true);
meta.setRowLimit(2);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
processRows(jsonInput, 4);
Assert.assertEquals("errors", 0, jsonInput.getErrors());
Assert.assertEquals("lines written", 2, jsonInput.getLinesWritten());
}
@Test
public void testFileList() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input1 = getBasicTestJson();
final String input2 = "{ \"store\": { \"book\": [ { \"price\": 9.99 } ] } }";
try (FileObject fileObj1 = KettleVFS.getFileObject(BASE_RAM_DIR + "test1.json");
FileObject fileObj2 = KettleVFS.getFileObject(BASE_RAM_DIR + "test2.json")) {
try (OutputStream out = fileObj1.getContent().getOutputStream()) {
out.write(input1.getBytes());
}
try (OutputStream out = fileObj2.getContent().getOutputStream()) {
out.write(input2.getBytes());
}
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
List<FileObject> fileList = Arrays.asList(fileObj1, fileObj2);
JsonInputMeta meta = createFileListMeta(fileList);
meta.setInputFields(new JsonInputField[] {price});
meta.setIncludeRowNumber(true);
meta.setRowNumberField("rownbr");
meta.setShortFileNameField("fname");
JsonInput jsonInput = createJsonInput(meta);
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {8.95d, 1L, "test1.json"},
new Object[] {12.99d, 2L, "test1.json"},
new Object[] {8.99d, 3L, "test1.json"},
new Object[] {22.99d, 4L, "test1.json"},
new Object[] {9.99d, 5L, "test2.json"});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
disposeJsonInput(jsonInput);
assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
@Test
public void testDefaultLeafToNull() throws Exception {
JsonInputField noPath = new JsonInputField("price");
noPath.setPath("$..price");
noPath.setType(ValueMetaInterface.TYPE_STRING);
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputMeta meta = createSimpleMeta("json", noPath);
meta.setIgnoreMissingPath(true);
meta.setRemoveSourceField(true);
final String input = getBasicTestJson();
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {input}, new Object[] {input});
processRows(jsonInput, 7);
disposeJsonInput(jsonInput);
Assert.assertEquals(5, jsonInput.getLinesWritten());
}
private void addFileToResultFilesname(FileObject file) throws Exception {
if (meta.addResultFile()) {
// Add this to the result file names...
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_GENERAL, file, getTransMeta().getName(), getStepname());
resultFile.setComment(BaseMessages.getString(PKG, "JsonInput.Log.FileAddedResult"));
addResultFile(resultFile);
}
}
@Test
public void testZipFileInput() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input = getBasicTestJson();
try (FileObject fileObj = KettleVFS.getFileObject(BASE_RAM_DIR + "test.zip")) {
fileObj.createFile();
try (OutputStream out = fileObj.getContent().getOutputStream()) {
try (ZipOutputStream zipOut = new ZipOutputStream(out)) {
ZipEntry jsonFile = new ZipEntry("test.json");
zipOut.putNextEntry(jsonFile);
zipOut.write(input.getBytes());
zipOut.closeEntry();
zipOut.flush();
}
}
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
JsonInputMeta meta = createSimpleMeta("in file", price);
meta.setIsAFile(true);
meta.setRemoveSourceField(true);
JsonInput jsonInput =
createJsonInput(
"in file",
meta,
new Object[][] {new Object[] {"zip:" + BASE_RAM_DIR + "test.zip!/test.json"}});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {8.95d},
new Object[] {12.99d},
new Object[] {8.99d},
new Object[] {22.99d});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 5);
Assert.assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
@Test
public void testNoFilesInListError() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
JsonInputMeta meta = createFileListMeta(Collections.<FileObject>emptyList());
meta.setDoNotFailIfNoFile(false);
JsonInputField price = new JsonInputField();
price.setName("price");
price.setType(ValueMetaInterface.TYPE_NUMBER);
price.setPath("$..book[*].price");
meta.setInputFields(new JsonInputField[] {price});
try (LocaleChange enUS = new LocaleChange(Locale.US)) {
JsonInput jsonInput = createJsonInput(meta);
processRows(jsonInput, 1);
}
String errMsgs = err.toString();
assertTrue(errMsgs, errMsgs.contains("No file(s) specified!"));
}
@Test
public void testSingleObjPred() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField bic = new JsonInputField("color");
bic.setPath("$.store.bicycle[?(@.price)].color");
bic.setType(ValueMetaInterface.TYPE_STRING);
JsonInputMeta meta = createSimpleMeta("json", bic);
meta.setRemoveSourceField(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
RowComparatorListener rowComparator = new RowComparatorListener(new Object[] {"red"});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 2);
Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
}
protected JsonInputMeta createFileListMeta(final List<FileObject> files) {
JsonInputMeta meta =
new JsonInputMeta() {
@Override
public FileInputList getFileInputList(VariableSpace space) {
return new FileInputList() {
@Override
public List<FileObject> getFiles() {
return files;
}
@Override
public int nrOfFiles() {
return files.size();
}
};
}
};
meta.setDefault();
meta.setInFields(false);
meta.setIgnoreMissingPath(false);
return meta;
}
protected JsonInputMeta createSimpleMeta(String inputColumn, JsonInputField... jsonPathFields) {
JsonInputMeta jsonInputMeta = new JsonInputMeta();
jsonInputMeta.setDefault();
jsonInputMeta.setInFields(true);
jsonInputMeta.setFieldValue(inputColumn);
jsonInputMeta.setInputFields(jsonPathFields);
jsonInputMeta.setIgnoreMissingPath(false);
return jsonInputMeta;
}
@Test
public void testArrayOut() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
helper.redirectLog(out, LogLevel.ERROR);
JsonInputField byc = new JsonInputField("books (array)");
byc.setPath("$.store.book");
byc.setType(ValueMetaInterface.TYPE_STRING);
JsonInputMeta meta = createSimpleMeta("json", byc);
meta.setRemoveSourceField(true);
JsonInput jsonInput = createJsonInput("json", meta, new Object[] {getBasicTestJson()});
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {
"[{\"category\":\"reference\",\"author\":\"Nigel Rees\",\"title\":\"Sayings of the Century\",\"price\":8.95},{\"category\":\"fiction\",\"author\":\"Evelyn Waugh\",\"title\":\"Sword of Honour\",\"price\":12.99},{\"category\":\"fiction\",\"author\":\"Herman Melville\",\"title\":\"Moby Dick\",\"isbn\":\"0-553-21311-3\",\"price\":8.99},{\"category\":\"fiction\",\"author\":\"J. R. R. Tolkien\",\"title\":\"The Lord of the Rings\",\"isbn\":\"0-395-19395-8\",\"price\":22.99}]"
});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 2);
Assert.assertEquals(out.toString(), 0, jsonInput.getErrors());
Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
}
private Object[] getOneRow() throws KettleException {
if (!meta.isInFields()) {
while ((data.recordnr >= data.nrrecords || data.file == null)) {
if (!openNextFile()) {
return null;
}
}
} else {
while ((data.recordnr >= data.nrrecords || data.readrow == null)) {
if (!ReadNextString()) {
return null;
}
if (data.readrow == null) {
return null;
}
}
}
return buildRow();
}
private boolean ReadNextString() {
try {
data.readrow = getRow(); // Grab another row ...
if (data.readrow == null) {
// finished processing!
if (log.isDetailed())
logDetailed(BaseMessages.getString(PKG, "JsonInput.Log.FinishedProcessing"));
return false;
}
if (first) {
first = false;
data.inputRowMeta = getInputRowMeta();
data.outputRowMeta = data.inputRowMeta.clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
// Get total previous fields
data.totalpreviousfields = data.inputRowMeta.size();
// Create convert meta-data objects that will contain Date & Number formatters
data.convertRowMeta = data.outputRowMeta.clone();
for (int i = 0; i < data.convertRowMeta.size(); i++)
data.convertRowMeta.getValueMeta(i).setType(ValueMetaInterface.TYPE_STRING);
// For String to <type> conversions, we allocate a conversion meta data row as well...
//
data.convertRowMeta = data.outputRowMeta.clone();
for (int i = 0; i < data.convertRowMeta.size(); i++) {
data.convertRowMeta.getValueMeta(i).setType(ValueMetaInterface.TYPE_STRING);
}
// Check if source field is provided
if (Const.isEmpty(meta.getFieldValue())) {
logError(BaseMessages.getString(PKG, "JsonInput.Log.NoField"));
throw new KettleException(BaseMessages.getString(PKG, "JsonInput.Log.NoField"));
}
// cache the position of the field
if (data.indexSourceField < 0) {
data.indexSourceField = getInputRowMeta().indexOfValue(meta.getFieldValue());
if (data.indexSourceField < 0) {
// The field is unreachable !
logError(
BaseMessages.getString(
PKG,
"JsonInput.Log.ErrorFindingField",
meta.getFieldValue())); // $NON-NLS-1$ //$NON-NLS-2$
throw new KettleException(
BaseMessages.getString(
PKG,
"JsonInput.Exception.CouldnotFindField",
meta.getFieldValue())); // $NON-NLS-1$ //$NON-NLS-2$
}
}
}
// get source field value
String fieldValue = getInputRowMeta().getString(data.readrow, data.indexSourceField);
if (log.isDetailed())
logDetailed(
BaseMessages.getString(
PKG, "JsonInput.Log.SourceValue", meta.getFieldValue(), fieldValue));
if (meta.getIsAFile()) {
// source is a file.
data.file = KettleVFS.getFileObject(fieldValue, getTransMeta());
if (meta.isIgnoreEmptyFile() && data.file.getContent().getSize() == 0) {
// log only basic as a warning (was before logError)
logBasic(
BaseMessages.getString(PKG, "JsonInput.Error.FileSizeZero", data.file.getName()));
ReadNextString();
}
} else {
// read string
data.stringToParse = fieldValue;
}
readFileOrString();
} catch (Exception e) {
logError(BaseMessages.getString(PKG, "JsonInput.Log.UnexpectedError", e.toString()));
stopAll();
logError(Const.getStackTracker(e));
setErrors(1);
return false;
}
return true;
}
private Object[] buildRow() throws KettleException {
// Create new row...
Object[] outputRowData = null;
if (data.readrow != null) outputRowData = data.readrow.clone();
else outputRowData = buildEmptyRow();
// Read fields...
for (int i = 0; i < data.nrInputFields; i++) {
// Get field
JsonInputField field = meta.getInputFields()[i];
// get json array for field
JSONArray jsona = data.resultList.get(i).getJSONArray();
String nodevalue = null;
if (jsona != null) {
Object jo = (Object) jsona.get(data.recordnr);
if (jo != null) {
nodevalue = jo.toString();
}
}
// Do trimming
switch (field.getTrimType()) {
case JsonInputField.TYPE_TRIM_LEFT:
nodevalue = Const.ltrim(nodevalue);
break;
case JsonInputField.TYPE_TRIM_RIGHT:
nodevalue = Const.rtrim(nodevalue);
break;
case JsonInputField.TYPE_TRIM_BOTH:
nodevalue = Const.trim(nodevalue);
break;
default:
break;
}
if (meta.isInFields()) {
// Add result field to input stream
outputRowData =
RowDataUtil.addValueData(outputRowData, data.totalpreviousfields + i, nodevalue);
}
// Do conversions
//
ValueMetaInterface targetValueMeta =
data.outputRowMeta.getValueMeta(data.totalpreviousfields + i);
ValueMetaInterface sourceValueMeta =
data.convertRowMeta.getValueMeta(data.totalpreviousfields + i);
outputRowData[data.totalpreviousfields + i] =
targetValueMeta.convertData(sourceValueMeta, nodevalue);
// Do we need to repeat this field if it is null?
if (meta.getInputFields()[i].isRepeated()) {
if (data.previousRow != null && Const.isEmpty(nodevalue)) {
outputRowData[data.totalpreviousfields + i] =
data.previousRow[data.totalpreviousfields + i];
}
}
} // End of loop over fields...
// When we have an input stream
// the row index take care of previous fields
int rowIndex = data.totalpreviousfields + data.nrInputFields;
// See if we need to add the filename to the row...
if (meta.includeFilename() && !Const.isEmpty(meta.getFilenameField())) {
outputRowData[rowIndex++] = data.filename;
}
// See if we need to add the row number to the row...
if (meta.includeRowNumber() && !Const.isEmpty(meta.getRowNumberField())) {
outputRowData[rowIndex++] = new Long(data.rownr);
}
// Possibly add short filename...
if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
outputRowData[rowIndex++] = data.shortFilename;
}
// Add Extension
if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
outputRowData[rowIndex++] = data.extension;
}
// add path
if (meta.getPathField() != null && meta.getPathField().length() > 0) {
outputRowData[rowIndex++] = data.path;
}
// Add Size
if (meta.getSizeField() != null && meta.getSizeField().length() > 0) {
outputRowData[rowIndex++] = new Long(data.size);
}
// add Hidden
if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
outputRowData[rowIndex++] = new Boolean(data.path);
}
// Add modification date
if (meta.getLastModificationDateField() != null
&& meta.getLastModificationDateField().length() > 0) {
outputRowData[rowIndex++] = data.lastModificationDateTime;
}
// Add Uri
if (meta.getUriField() != null && meta.getUriField().length() > 0) {
outputRowData[rowIndex++] = data.uriName;
}
// Add RootUri
if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
outputRowData[rowIndex++] = data.rootUriName;
}
data.recordnr++;
RowMetaInterface irow = getInputRowMeta();
data.previousRow =
irow == null ? outputRowData : (Object[]) irow.cloneRow(outputRowData); // copy it to make
// surely the next step doesn't change it in between...
return outputRowData;
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
if (first && !meta.isInFields()) {
first = false;
data.files = meta.getFiles(this);
if (!meta.isdoNotFailIfNoFile() && data.files.nrOfFiles() == 0) {
throw new KettleException(BaseMessages.getString(PKG, "JsonInput.Log.NoFiles"));
}
handleMissingFiles();
// Create the output row meta-data
data.outputRowMeta = new RowMeta();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
// Create convert meta-data objects that will contain Date & Number formatters
data.convertRowMeta = data.outputRowMeta.clone();
for (int i = 0; i < data.convertRowMeta.size(); i++)
data.convertRowMeta.getValueMeta(i).setType(ValueMetaInterface.TYPE_STRING);
// For String to <type> conversions, we allocate a conversion meta data row as well...
//
data.convertRowMeta = data.outputRowMeta.clone();
for (int i = 0; i < data.convertRowMeta.size(); i++) {
data.convertRowMeta.getValueMeta(i).setType(ValueMetaInterface.TYPE_STRING);
}
}
Object[] r = null;
try {
// Grab a row
r = getOneRow();
if (r == null) {
setOutputDone(); // signal end to receiver(s)
return false; // end of data or error.
}
if (log.isRowLevel())
logRowlevel(
BaseMessages.getString(PKG, "JsonInput.Log.ReadRow", data.outputRowMeta.getString(r)));
incrementLinesInput();
data.rownr++;
putRow(data.outputRowMeta, r); // copy row to output rowset(s);
if (meta.getRowLimit() > 0 && data.rownr > meta.getRowLimit()) {
// limit has been reached: stop now.
setOutputDone();
return false;
}
} catch (Exception e) {
boolean sendToErrorRow = false;
String errorMessage = null;
if (getStepMeta().isDoingErrorHandling()) {
sendToErrorRow = true;
errorMessage = e.toString();
} else {
logError(
BaseMessages.getString(
PKG, "JsonInput.ErrorInStepRunning", e.getMessage())); // $NON-NLS-1$
setErrors(1);
stopAll();
setOutputDone(); // signal end to receiver(s)
return false;
}
if (sendToErrorRow) {
// Simply add this row to the error row
putError(getInputRowMeta(), r, 1, errorMessage, null, "JsonInput001");
}
}
return true;
}
private void readFileOrString() throws Exception {
if (data.file != null) {
data.filename = KettleVFS.getFilename(data.file);
// Add additional fields?
if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
data.shortFilename = data.file.getName().getBaseName();
}
if (meta.getPathField() != null && meta.getPathField().length() > 0) {
data.path = KettleVFS.getFilename(data.file.getParent());
}
if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
data.hidden = data.file.isHidden();
}
if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
data.extension = data.file.getName().getExtension();
}
if (meta.getLastModificationDateField() != null
&& meta.getLastModificationDateField().length() > 0) {
data.lastModificationDateTime = new Date(data.file.getContent().getLastModifiedTime());
}
if (meta.getUriField() != null && meta.getUriField().length() > 0) {
data.uriName = data.file.getName().getURI();
}
if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
data.rootUriName = data.file.getName().getRootURI();
}
// Check if file is empty
long fileSize = data.file.getContent().getSize();
if (meta.getSizeField() != null && meta.getSizeField().length() > 0) {
data.size = fileSize;
}
// Move file pointer ahead!
data.filenr++;
if (log.isDetailed())
logDetailed(BaseMessages.getString(PKG, "JsonInput.Log.OpeningFile", data.file.toString()));
addFileToResultFilesname(data.file);
}
parseJson();
}
@Test
public void testExtraFileFields() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
helper.redirectLog(err, LogLevel.ERROR);
final String input1 = getBasicTestJson();
final String input2 = "{ \"store\": { \"bicycle\": { \"color\": \"blue\" } } }";
final String path1 = BASE_RAM_DIR + "test1.json";
final String path2 = BASE_RAM_DIR + "test2.js";
try (FileObject fileObj1 = KettleVFS.getFileObject(path1);
FileObject fileObj2 = KettleVFS.getFileObject(path2)) {
try (OutputStream out = fileObj1.getContent().getOutputStream()) {
out.write(input1.getBytes());
}
try (OutputStream out = fileObj2.getContent().getOutputStream()) {
out.write(input2.getBytes());
}
JsonInputField color = new JsonInputField();
color.setName("color");
color.setType(ValueMetaInterface.TYPE_STRING);
color.setPath("$.store.bicycle.color");
JsonInputMeta meta = createSimpleMeta("in file", color);
meta.setInFields(true);
meta.setIsAFile(true);
meta.setRemoveSourceField(true);
meta.setExtensionField("extension");
meta.setPathField("dir path");
meta.setSizeField("size");
meta.setIsHiddenField("hidden?");
meta.setLastModificationDateField("last modified");
meta.setUriField("URI");
meta.setRootUriField("root URI");
// custom checkers for size and last modified
RowComparatorListener rowComparator =
new RowComparatorListener(
new Object[] {
"red",
"json",
"ram:///jsonInputTest",
-1L,
false,
new Date(0),
"ram:///jsonInputTest/test1.json",
"ram:///"
},
new Object[] {
"blue",
"js",
"ram:///jsonInputTest",
-1L,
false,
new Date(0),
"ram:///jsonInputTest/test2.js",
"ram:///"
});
rowComparator.setComparator(
3,
new RowComparatorListener.Comparison<Object>() {
@Override
public boolean equals(Object expected, Object actual) throws Exception {
// just want a valid size
return ((long) actual) > 0L;
}
});
rowComparator.setComparator(
5,
new RowComparatorListener.Comparison<Object>() {
@Override
public boolean equals(Object expected, Object actual) throws Exception {
return ((Date) actual).after(new Date(0));
}
});
JsonInput jsonInput =
createJsonInput(
"in file", meta, new Object[][] {new Object[] {path1}, new Object[] {path2}});
jsonInput.addRowListener(rowComparator);
processRows(jsonInput, 3);
Assert.assertEquals(err.toString(), 0, jsonInput.getErrors());
} finally {
deleteFiles();
}
}
