private Object[] getOneRow() throws KettleException {
if (!openNextFile()) {
return null;
}
// Build an empty row based on the meta-data
Object[] outputRowData = buildEmptyRow();
try {
// Create new row or clone
if (meta.getIsInFields())
System.arraycopy(data.readrow, 0, outputRowData, 0, data.readrow.length);
// Read fields...
for (int i = 0; i < data.nrInputFields; i++) {
// Get field
LoadFileInputField loadFileInputField = meta.getInputFields()[i];
String o = null;
switch (loadFileInputField.getElementType()) {
case LoadFileInputField.ELEMENT_TYPE_FILECONTENT:
// DO Trimming!
switch (loadFileInputField.getTrimType()) {
case LoadFileInputField.TYPE_TRIM_LEFT:
data.filecontent = Const.ltrim(data.filecontent);
break;
case LoadFileInputField.TYPE_TRIM_RIGHT:
data.filecontent = Const.rtrim(data.filecontent);
break;
case LoadFileInputField.TYPE_TRIM_BOTH:
data.filecontent = Const.trim(data.filecontent);
break;
default:
break;
}
o = data.filecontent;
break;
case LoadFileInputField.ELEMENT_TYPE_FILESIZE:
o = String.valueOf(data.fileSize);
break;
default:
break;
}
int indexField = data.totalpreviousfields + i;
// Do conversions
//
ValueMetaInterface targetValueMeta = data.outputRowMeta.getValueMeta(indexField);
ValueMetaInterface sourceValueMeta = data.convertRowMeta.getValueMeta(indexField);
outputRowData[indexField] = targetValueMeta.convertData(sourceValueMeta, o);
// Do we need to repeat this field if it is null?
if (loadFileInputField.isRepeated()) {
if (data.previousRow != null && o == null) {
outputRowData[indexField] = data.previousRow[indexField];
}
}
} // End of loop over fields...
int rowIndex = data.totalpreviousfields + data.nrInputFields;
// See if we need to add the filename to the row...
if (meta.includeFilename()
&& meta.getFilenameField() != null
&& meta.getFilenameField().length() > 0) {
outputRowData[rowIndex++] = data.filename;
}
// See if we need to add the row number to the row...
if (meta.includeRowNumber()
&& meta.getRowNumberField() != null
&& meta.getRowNumberField().length() > 0) {
outputRowData[rowIndex++] = new Long(data.rownr);
}
// Possibly add short filename...
if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
outputRowData[rowIndex++] = data.shortFilename;
}
// Add Extension
if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
outputRowData[rowIndex++] = data.extension;
}
// add path
if (meta.getPathField() != null && meta.getPathField().length() > 0) {
outputRowData[rowIndex++] = data.path;
}
// add Hidden
if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
outputRowData[rowIndex++] = new Boolean(data.hidden);
}
// Add modification date
if (meta.getLastModificationDateField() != null
&& meta.getLastModificationDateField().length() > 0) {
outputRowData[rowIndex++] = data.lastModificationDateTime;
}
// Add Uri
if (meta.getUriField() != null && meta.getUriField().length() > 0) {
outputRowData[rowIndex++] = data.uriName;
}
// Add RootUri
if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
outputRowData[rowIndex++] = data.rootUriName;
}
RowMetaInterface irow = getInputRowMeta();
data.previousRow =
irow == null ? outputRowData : (Object[]) irow.cloneRow(outputRowData); // copy it to make
// surely the next step doesn't change it in between...
incrementLinesInput();
data.rownr++;
} catch (Exception e) {
throw new KettleException("Impossible de charger le fichier", e);
}
return outputRowData;
}
/**
* Create the command line for an sqlldr process depending on the meta information supplied.
*
* @param meta The meta data to create the command line from
* @param password Use the real password or not
* @return The string to execute.
* @throws KettleException Upon any exception
*/
public String createCommandLine(OraBulkLoaderMeta meta, boolean password) throws KettleException {
StringBuilder sb = new StringBuilder(300);
if (meta.getSqlldr() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getSqlldr()), getTransMeta());
String sqlldr = KettleVFS.getFilename(fileObject);
sb.append(sqlldr);
} catch (KettleFileException ex) {
throw new KettleException("Error retrieving sqlldr string", ex);
}
} else {
throw new KettleException("No sqlldr application specified");
}
if (meta.getControlFile() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getControlFile()), getTransMeta());
sb.append(" control=\'");
sb.append(KettleVFS.getFilename(fileObject));
sb.append("\'");
} catch (KettleFileException ex) {
throw new KettleException("Error retrieving controlfile string", ex);
}
} else {
throw new KettleException("No control file specified");
}
if (OraBulkLoaderMeta.METHOD_AUTO_CONCURRENT.equals(meta.getLoadMethod())) {
sb.append(" data=\'-\'");
}
if (meta.getLogFile() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getLogFile()), getTransMeta());
sb.append(" log=\'");
sb.append(KettleVFS.getFilename(fileObject));
sb.append("\'");
} catch (KettleFileException ex) {
throw new KettleException("Error retrieving logfile string", ex);
}
}
if (meta.getBadFile() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getBadFile()), getTransMeta());
sb.append(" bad=\'");
sb.append(KettleVFS.getFilename(fileObject));
sb.append("\'");
} catch (KettleFileException ex) {
throw new KettleException("Error retrieving badfile string", ex);
}
}
if (meta.getDiscardFile() != null) {
try {
FileObject fileObject =
KettleVFS.getFileObject(environmentSubstitute(meta.getDiscardFile()), getTransMeta());
sb.append(" discard=\'");
sb.append(KettleVFS.getFilename(fileObject));
sb.append("\'");
} catch (KettleFileException ex) {
throw new KettleException("Error retrieving discardfile string", ex);
}
}
DatabaseMeta dm = meta.getDatabaseMeta();
if (dm != null) {
String user = Const.NVL(dm.getUsername(), "");
String pass =
Const.NVL(
Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(dm.getPassword())), "");
if (!password) {
pass = "******";
}
String dns = Const.NVL(dm.getDatabaseName(), "");
sb.append(" userid=")
.append(environmentSubstitute(user))
.append("/")
.append(environmentSubstitute(pass))
.append("@");
String overrideName = meta.getDbNameOverride();
if (Utils.isEmpty(Const.rtrim(overrideName))) {
sb.append(environmentSubstitute(dns));
} else {
// if the database name override is filled in, do that one.
sb.append(environmentSubstitute(overrideName));
}
} else {
throw new KettleException("No connection specified");
}
if (meta.isDirectPath()) {
sb.append(" DIRECT=TRUE");
if (getStepMeta().getCopies() > 1 || meta.isParallel()) {
sb.append(" PARALLEL=TRUE");
}
}
return sb.toString();
}
private Object[] buildRow() throws KettleException {
// Create new row...
Object[] outputRowData = null;
if (data.readrow != null) outputRowData = data.readrow.clone();
else outputRowData = buildEmptyRow();
// Read fields...
for (int i = 0; i < data.nrInputFields; i++) {
// Get field
JsonInputField field = meta.getInputFields()[i];
// get json array for field
JSONArray jsona = data.resultList.get(i).getJSONArray();
String nodevalue = null;
if (jsona != null) {
Object jo = (Object) jsona.get(data.recordnr);
if (jo != null) {
nodevalue = jo.toString();
}
}
// Do trimming
switch (field.getTrimType()) {
case JsonInputField.TYPE_TRIM_LEFT:
nodevalue = Const.ltrim(nodevalue);
break;
case JsonInputField.TYPE_TRIM_RIGHT:
nodevalue = Const.rtrim(nodevalue);
break;
case JsonInputField.TYPE_TRIM_BOTH:
nodevalue = Const.trim(nodevalue);
break;
default:
break;
}
if (meta.isInFields()) {
// Add result field to input stream
outputRowData =
RowDataUtil.addValueData(outputRowData, data.totalpreviousfields + i, nodevalue);
}
// Do conversions
//
ValueMetaInterface targetValueMeta =
data.outputRowMeta.getValueMeta(data.totalpreviousfields + i);
ValueMetaInterface sourceValueMeta =
data.convertRowMeta.getValueMeta(data.totalpreviousfields + i);
outputRowData[data.totalpreviousfields + i] =
targetValueMeta.convertData(sourceValueMeta, nodevalue);
// Do we need to repeat this field if it is null?
if (meta.getInputFields()[i].isRepeated()) {
if (data.previousRow != null && Const.isEmpty(nodevalue)) {
outputRowData[data.totalpreviousfields + i] =
data.previousRow[data.totalpreviousfields + i];
}
}
} // End of loop over fields...
// When we have an input stream
// the row index take care of previous fields
int rowIndex = data.totalpreviousfields + data.nrInputFields;
// See if we need to add the filename to the row...
if (meta.includeFilename() && !Const.isEmpty(meta.getFilenameField())) {
outputRowData[rowIndex++] = data.filename;
}
// See if we need to add the row number to the row...
if (meta.includeRowNumber() && !Const.isEmpty(meta.getRowNumberField())) {
outputRowData[rowIndex++] = new Long(data.rownr);
}
// Possibly add short filename...
if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) {
outputRowData[rowIndex++] = data.shortFilename;
}
// Add Extension
if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) {
outputRowData[rowIndex++] = data.extension;
}
// add path
if (meta.getPathField() != null && meta.getPathField().length() > 0) {
outputRowData[rowIndex++] = data.path;
}
// Add Size
if (meta.getSizeField() != null && meta.getSizeField().length() > 0) {
outputRowData[rowIndex++] = new Long(data.size);
}
// add Hidden
if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) {
outputRowData[rowIndex++] = new Boolean(data.path);
}
// Add modification date
if (meta.getLastModificationDateField() != null
&& meta.getLastModificationDateField().length() > 0) {
outputRowData[rowIndex++] = data.lastModificationDateTime;
}
// Add Uri
if (meta.getUriField() != null && meta.getUriField().length() > 0) {
outputRowData[rowIndex++] = data.uriName;
}
// Add RootUri
if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) {
outputRowData[rowIndex++] = data.rootUriName;
}
data.recordnr++;
RowMetaInterface irow = getInputRowMeta();
data.previousRow =
irow == null ? outputRowData : (Object[]) irow.cloneRow(outputRowData); // copy it to make
// surely the next step doesn't change it in between...
return outputRowData;
}
private void executeShell(Result result, List<RowMetaAndData> cmdRows, String[] args) {
FileObject fileObject = null;
String realScript = null;
FileObject tempFile = null;
try {
// What's the exact command?
String[] base = null;
List<String> cmds = new ArrayList<String>();
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobShell.RunningOn", Const.getOS()));
}
if (insertScript) {
realScript = environmentSubstitute(script);
} else {
String realFilename = environmentSubstitute(getFilename());
fileObject = KettleVFS.getFileObject(realFilename, this);
}
if (Const.getOS().equals("Windows 95")) {
base = new String[] {"command.com", "/C"};
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
} else if (Const.getOS().startsWith("Windows")) {
base = new String[] {"cmd.exe", "/C"};
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
} else {
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
base = new String[] {KettleVFS.getFilename(fileObject)};
}
// Construct the arguments...
if (argFromPrevious && cmdRows != null) {
// Add the base command...
for (int i = 0; i < base.length; i++) {
cmds.add(base[i]);
}
if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
// for windows all arguments including the command itself
// need to be
// included in 1 argument to cmd/command.
StringBuffer cmdline = new StringBuffer(300);
cmdline.append('"');
cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
// Add the arguments from previous results...
for (int i = 0; i < cmdRows.size(); i++) {
// Normally just one row, but once in a while to remain compatible we have multiple.
RowMetaAndData r = cmdRows.get(i);
for (int j = 0; j < r.size(); j++) {
cmdline.append(' ');
cmdline.append(Const.optionallyQuoteStringByOS(r.getString(j, null)));
}
}
cmdline.append('"');
cmds.add(cmdline.toString());
} else {
// Add the arguments from previous results...
for (int i = 0; i < cmdRows.size(); i++) {
// Normally just one row, but once in a while to remain compatible we have multiple.
RowMetaAndData r = cmdRows.get(i);
for (int j = 0; j < r.size(); j++) {
cmds.add(Const.optionallyQuoteStringByOS(r.getString(j, null)));
}
}
}
} else if (args != null) {
// Add the base command...
for (int i = 0; i < base.length; i++) {
cmds.add(base[i]);
}
if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
// for windows all arguments including the command itself
// need to be
// included in 1 argument to cmd/command.
StringBuffer cmdline = new StringBuffer(300);
cmdline.append('"');
cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
for (int i = 0; i < args.length; i++) {
cmdline.append(' ');
cmdline.append(Const.optionallyQuoteStringByOS(args[i]));
}
cmdline.append('"');
cmds.add(cmdline.toString());
} else {
for (int i = 0; i < args.length; i++) {
cmds.add(args[i]);
}
}
}
StringBuffer command = new StringBuffer();
Iterator<String> it = cmds.iterator();
boolean first = true;
while (it.hasNext()) {
if (!first) {
command.append(' ');
} else {
first = false;
}
command.append(it.next());
}
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobShell.ExecCommand", command.toString()));
}
// Build the environment variable list...
ProcessBuilder procBuilder = new ProcessBuilder(cmds);
Map<String, String> env = procBuilder.environment();
String[] variables = listVariables();
for (int i = 0; i < variables.length; i++) {
env.put(variables[i], getVariable(variables[i]));
}
if (getWorkDirectory() != null && !Const.isEmpty(Const.rtrim(getWorkDirectory()))) {
String vfsFilename = environmentSubstitute(getWorkDirectory());
File file = new File(KettleVFS.getFilename(KettleVFS.getFileObject(vfsFilename, this)));
procBuilder.directory(file);
}
Process proc = procBuilder.start();
// any error message?
StreamLogger errorLogger = new StreamLogger(log, proc.getErrorStream(), "(stderr)", true);
// any output?
StreamLogger outputLogger = new StreamLogger(log, proc.getInputStream(), "(stdout)");
// kick them off
Thread errorLoggerThread = new Thread(errorLogger);
errorLoggerThread.start();
Thread outputLoggerThread = new Thread(outputLogger);
outputLoggerThread.start();
proc.waitFor();
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobShell.CommandFinished", command.toString()));
}
// What's the exit status?
result.setExitStatus(proc.exitValue());
if (result.getExitStatus() != 0) {
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobShell.ExitStatus",
environmentSubstitute(getFilename()),
"" + result.getExitStatus()));
}
result.setNrErrors(1);
}
// wait until loggers read all data from stdout and stderr
errorLoggerThread.join();
outputLoggerThread.join();
// close the streams
// otherwise you get "Too many open files, java.io.IOException" after a lot of iterations
proc.getErrorStream().close();
proc.getOutputStream().close();
} catch (IOException ioe) {
logError(
BaseMessages.getString(
PKG,
"JobShell.ErrorRunningShell",
environmentSubstitute(getFilename()),
ioe.toString()),
ioe);
result.setNrErrors(1);
} catch (InterruptedException ie) {
logError(
BaseMessages.getString(
PKG, "JobShell.Shellinterupted", environmentSubstitute(getFilename()), ie.toString()),
ie);
result.setNrErrors(1);
} catch (Exception e) {
logError(
BaseMessages.getString(
PKG, "JobShell.UnexpectedError", environmentSubstitute(getFilename()), e.toString()),
e);
result.setNrErrors(1);
} finally {
// If we created a temporary file, remove it...
//
if (tempFile != null) {
try {
tempFile.delete();
} catch (Exception e) {
BaseMessages.getString(
PKG, "JobShell.UnexpectedError", tempFile.toString(), e.toString());
}
}
}
if (result.getNrErrors() > 0) {
result.setResult(false);
} else {
result.setResult(true);
}
}
