private Object[] getOneRow() throws KettleException { if (!openNextFile()) { return null; } // Build an empty row based on the meta-data Object[] outputRowData = buildEmptyRow(); try { // Create new row or clone if (meta.getIsInFields()) System.arraycopy(data.readrow, 0, outputRowData, 0, data.readrow.length); // Read fields... for (int i = 0; i < data.nrInputFields; i++) { // Get field LoadFileInputField loadFileInputField = meta.getInputFields()[i]; String o = null; switch (loadFileInputField.getElementType()) { case LoadFileInputField.ELEMENT_TYPE_FILECONTENT: // DO Trimming! switch (loadFileInputField.getTrimType()) { case LoadFileInputField.TYPE_TRIM_LEFT: data.filecontent = Const.ltrim(data.filecontent); break; case LoadFileInputField.TYPE_TRIM_RIGHT: data.filecontent = Const.rtrim(data.filecontent); break; case LoadFileInputField.TYPE_TRIM_BOTH: data.filecontent = Const.trim(data.filecontent); break; default: break; } o = data.filecontent; break; case LoadFileInputField.ELEMENT_TYPE_FILESIZE: o = String.valueOf(data.fileSize); break; default: break; } int indexField = data.totalpreviousfields + i; // Do conversions // ValueMetaInterface targetValueMeta = data.outputRowMeta.getValueMeta(indexField); ValueMetaInterface sourceValueMeta = data.convertRowMeta.getValueMeta(indexField); outputRowData[indexField] = targetValueMeta.convertData(sourceValueMeta, o); // Do we need to repeat this field if it is null? if (loadFileInputField.isRepeated()) { if (data.previousRow != null && o == null) { outputRowData[indexField] = data.previousRow[indexField]; } } } // End of loop over fields... int rowIndex = data.totalpreviousfields + data.nrInputFields; // See if we need to add the filename to the row... if (meta.includeFilename() && meta.getFilenameField() != null && meta.getFilenameField().length() > 0) { outputRowData[rowIndex++] = data.filename; } // See if we need to add the row number to the row... if (meta.includeRowNumber() && meta.getRowNumberField() != null && meta.getRowNumberField().length() > 0) { outputRowData[rowIndex++] = new Long(data.rownr); } // Possibly add short filename... if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) { outputRowData[rowIndex++] = data.shortFilename; } // Add Extension if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) { outputRowData[rowIndex++] = data.extension; } // add path if (meta.getPathField() != null && meta.getPathField().length() > 0) { outputRowData[rowIndex++] = data.path; } // add Hidden if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) { outputRowData[rowIndex++] = new Boolean(data.hidden); } // Add modification date if (meta.getLastModificationDateField() != null && meta.getLastModificationDateField().length() > 0) { outputRowData[rowIndex++] = data.lastModificationDateTime; } // Add Uri if (meta.getUriField() != null && meta.getUriField().length() > 0) { outputRowData[rowIndex++] = data.uriName; } // Add RootUri if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) { outputRowData[rowIndex++] = data.rootUriName; } RowMetaInterface irow = getInputRowMeta(); data.previousRow = irow == null ? outputRowData : (Object[]) irow.cloneRow(outputRowData); // copy it to make // surely the next step doesn't change it in between... incrementLinesInput(); data.rownr++; } catch (Exception e) { throw new KettleException("Impossible de charger le fichier", e); } return outputRowData; }
/** * Create the command line for an sqlldr process depending on the meta information supplied. * * @param meta The meta data to create the command line from * @param password Use the real password or not * @return The string to execute. * @throws KettleException Upon any exception */ public String createCommandLine(OraBulkLoaderMeta meta, boolean password) throws KettleException { StringBuilder sb = new StringBuilder(300); if (meta.getSqlldr() != null) { try { FileObject fileObject = KettleVFS.getFileObject(environmentSubstitute(meta.getSqlldr()), getTransMeta()); String sqlldr = KettleVFS.getFilename(fileObject); sb.append(sqlldr); } catch (KettleFileException ex) { throw new KettleException("Error retrieving sqlldr string", ex); } } else { throw new KettleException("No sqlldr application specified"); } if (meta.getControlFile() != null) { try { FileObject fileObject = KettleVFS.getFileObject(environmentSubstitute(meta.getControlFile()), getTransMeta()); sb.append(" control=\'"); sb.append(KettleVFS.getFilename(fileObject)); sb.append("\'"); } catch (KettleFileException ex) { throw new KettleException("Error retrieving controlfile string", ex); } } else { throw new KettleException("No control file specified"); } if (OraBulkLoaderMeta.METHOD_AUTO_CONCURRENT.equals(meta.getLoadMethod())) { sb.append(" data=\'-\'"); } if (meta.getLogFile() != null) { try { FileObject fileObject = KettleVFS.getFileObject(environmentSubstitute(meta.getLogFile()), getTransMeta()); sb.append(" log=\'"); sb.append(KettleVFS.getFilename(fileObject)); sb.append("\'"); } catch (KettleFileException ex) { throw new KettleException("Error retrieving logfile string", ex); } } if (meta.getBadFile() != null) { try { FileObject fileObject = KettleVFS.getFileObject(environmentSubstitute(meta.getBadFile()), getTransMeta()); sb.append(" bad=\'"); sb.append(KettleVFS.getFilename(fileObject)); sb.append("\'"); } catch (KettleFileException ex) { throw new KettleException("Error retrieving badfile string", ex); } } if (meta.getDiscardFile() != null) { try { FileObject fileObject = KettleVFS.getFileObject(environmentSubstitute(meta.getDiscardFile()), getTransMeta()); sb.append(" discard=\'"); sb.append(KettleVFS.getFilename(fileObject)); sb.append("\'"); } catch (KettleFileException ex) { throw new KettleException("Error retrieving discardfile string", ex); } } DatabaseMeta dm = meta.getDatabaseMeta(); if (dm != null) { String user = Const.NVL(dm.getUsername(), ""); String pass = Const.NVL( Encr.decryptPasswordOptionallyEncrypted(environmentSubstitute(dm.getPassword())), ""); if (!password) { pass = "******"; } String dns = Const.NVL(dm.getDatabaseName(), ""); sb.append(" userid=") .append(environmentSubstitute(user)) .append("/") .append(environmentSubstitute(pass)) .append("@"); String overrideName = meta.getDbNameOverride(); if (Utils.isEmpty(Const.rtrim(overrideName))) { sb.append(environmentSubstitute(dns)); } else { // if the database name override is filled in, do that one. sb.append(environmentSubstitute(overrideName)); } } else { throw new KettleException("No connection specified"); } if (meta.isDirectPath()) { sb.append(" DIRECT=TRUE"); if (getStepMeta().getCopies() > 1 || meta.isParallel()) { sb.append(" PARALLEL=TRUE"); } } return sb.toString(); }
private Object[] buildRow() throws KettleException { // Create new row... Object[] outputRowData = null; if (data.readrow != null) outputRowData = data.readrow.clone(); else outputRowData = buildEmptyRow(); // Read fields... for (int i = 0; i < data.nrInputFields; i++) { // Get field JsonInputField field = meta.getInputFields()[i]; // get json array for field JSONArray jsona = data.resultList.get(i).getJSONArray(); String nodevalue = null; if (jsona != null) { Object jo = (Object) jsona.get(data.recordnr); if (jo != null) { nodevalue = jo.toString(); } } // Do trimming switch (field.getTrimType()) { case JsonInputField.TYPE_TRIM_LEFT: nodevalue = Const.ltrim(nodevalue); break; case JsonInputField.TYPE_TRIM_RIGHT: nodevalue = Const.rtrim(nodevalue); break; case JsonInputField.TYPE_TRIM_BOTH: nodevalue = Const.trim(nodevalue); break; default: break; } if (meta.isInFields()) { // Add result field to input stream outputRowData = RowDataUtil.addValueData(outputRowData, data.totalpreviousfields + i, nodevalue); } // Do conversions // ValueMetaInterface targetValueMeta = data.outputRowMeta.getValueMeta(data.totalpreviousfields + i); ValueMetaInterface sourceValueMeta = data.convertRowMeta.getValueMeta(data.totalpreviousfields + i); outputRowData[data.totalpreviousfields + i] = targetValueMeta.convertData(sourceValueMeta, nodevalue); // Do we need to repeat this field if it is null? if (meta.getInputFields()[i].isRepeated()) { if (data.previousRow != null && Const.isEmpty(nodevalue)) { outputRowData[data.totalpreviousfields + i] = data.previousRow[data.totalpreviousfields + i]; } } } // End of loop over fields... // When we have an input stream // the row index take care of previous fields int rowIndex = data.totalpreviousfields + data.nrInputFields; // See if we need to add the filename to the row... if (meta.includeFilename() && !Const.isEmpty(meta.getFilenameField())) { outputRowData[rowIndex++] = data.filename; } // See if we need to add the row number to the row... if (meta.includeRowNumber() && !Const.isEmpty(meta.getRowNumberField())) { outputRowData[rowIndex++] = new Long(data.rownr); } // Possibly add short filename... if (meta.getShortFileNameField() != null && meta.getShortFileNameField().length() > 0) { outputRowData[rowIndex++] = data.shortFilename; } // Add Extension if (meta.getExtensionField() != null && meta.getExtensionField().length() > 0) { outputRowData[rowIndex++] = data.extension; } // add path if (meta.getPathField() != null && meta.getPathField().length() > 0) { outputRowData[rowIndex++] = data.path; } // Add Size if (meta.getSizeField() != null && meta.getSizeField().length() > 0) { outputRowData[rowIndex++] = new Long(data.size); } // add Hidden if (meta.isHiddenField() != null && meta.isHiddenField().length() > 0) { outputRowData[rowIndex++] = new Boolean(data.path); } // Add modification date if (meta.getLastModificationDateField() != null && meta.getLastModificationDateField().length() > 0) { outputRowData[rowIndex++] = data.lastModificationDateTime; } // Add Uri if (meta.getUriField() != null && meta.getUriField().length() > 0) { outputRowData[rowIndex++] = data.uriName; } // Add RootUri if (meta.getRootUriField() != null && meta.getRootUriField().length() > 0) { outputRowData[rowIndex++] = data.rootUriName; } data.recordnr++; RowMetaInterface irow = getInputRowMeta(); data.previousRow = irow == null ? outputRowData : (Object[]) irow.cloneRow(outputRowData); // copy it to make // surely the next step doesn't change it in between... return outputRowData; }
private void executeShell(Result result, List<RowMetaAndData> cmdRows, String[] args) { FileObject fileObject = null; String realScript = null; FileObject tempFile = null; try { // What's the exact command? String[] base = null; List<String> cmds = new ArrayList<String>(); if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "JobShell.RunningOn", Const.getOS())); } if (insertScript) { realScript = environmentSubstitute(script); } else { String realFilename = environmentSubstitute(getFilename()); fileObject = KettleVFS.getFileObject(realFilename, this); } if (Const.getOS().equals("Windows 95")) { base = new String[] {"command.com", "/C"}; if (insertScript) { tempFile = KettleVFS.createTempFile( "kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this); fileObject = createTemporaryShellFile(tempFile, realScript); } } else if (Const.getOS().startsWith("Windows")) { base = new String[] {"cmd.exe", "/C"}; if (insertScript) { tempFile = KettleVFS.createTempFile( "kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this); fileObject = createTemporaryShellFile(tempFile, realScript); } } else { if (insertScript) { tempFile = KettleVFS.createTempFile( "kettle", "shell", System.getProperty("java.io.tmpdir"), this); fileObject = createTemporaryShellFile(tempFile, realScript); } base = new String[] {KettleVFS.getFilename(fileObject)}; } // Construct the arguments... if (argFromPrevious && cmdRows != null) { // Add the base command... for (int i = 0; i < base.length; i++) { cmds.add(base[i]); } if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) { // for windows all arguments including the command itself // need to be // included in 1 argument to cmd/command. StringBuffer cmdline = new StringBuffer(300); cmdline.append('"'); cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject))); // Add the arguments from previous results... for (int i = 0; i < cmdRows.size(); i++) { // Normally just one row, but once in a while to remain compatible we have multiple. RowMetaAndData r = cmdRows.get(i); for (int j = 0; j < r.size(); j++) { cmdline.append(' '); cmdline.append(Const.optionallyQuoteStringByOS(r.getString(j, null))); } } cmdline.append('"'); cmds.add(cmdline.toString()); } else { // Add the arguments from previous results... for (int i = 0; i < cmdRows.size(); i++) { // Normally just one row, but once in a while to remain compatible we have multiple. RowMetaAndData r = cmdRows.get(i); for (int j = 0; j < r.size(); j++) { cmds.add(Const.optionallyQuoteStringByOS(r.getString(j, null))); } } } } else if (args != null) { // Add the base command... for (int i = 0; i < base.length; i++) { cmds.add(base[i]); } if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) { // for windows all arguments including the command itself // need to be // included in 1 argument to cmd/command. StringBuffer cmdline = new StringBuffer(300); cmdline.append('"'); cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject))); for (int i = 0; i < args.length; i++) { cmdline.append(' '); cmdline.append(Const.optionallyQuoteStringByOS(args[i])); } cmdline.append('"'); cmds.add(cmdline.toString()); } else { for (int i = 0; i < args.length; i++) { cmds.add(args[i]); } } } StringBuffer command = new StringBuffer(); Iterator<String> it = cmds.iterator(); boolean first = true; while (it.hasNext()) { if (!first) { command.append(' '); } else { first = false; } command.append(it.next()); } if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "JobShell.ExecCommand", command.toString())); } // Build the environment variable list... ProcessBuilder procBuilder = new ProcessBuilder(cmds); Map<String, String> env = procBuilder.environment(); String[] variables = listVariables(); for (int i = 0; i < variables.length; i++) { env.put(variables[i], getVariable(variables[i])); } if (getWorkDirectory() != null && !Const.isEmpty(Const.rtrim(getWorkDirectory()))) { String vfsFilename = environmentSubstitute(getWorkDirectory()); File file = new File(KettleVFS.getFilename(KettleVFS.getFileObject(vfsFilename, this))); procBuilder.directory(file); } Process proc = procBuilder.start(); // any error message? StreamLogger errorLogger = new StreamLogger(log, proc.getErrorStream(), "(stderr)", true); // any output? StreamLogger outputLogger = new StreamLogger(log, proc.getInputStream(), "(stdout)"); // kick them off Thread errorLoggerThread = new Thread(errorLogger); errorLoggerThread.start(); Thread outputLoggerThread = new Thread(outputLogger); outputLoggerThread.start(); proc.waitFor(); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobShell.CommandFinished", command.toString())); } // What's the exit status? result.setExitStatus(proc.exitValue()); if (result.getExitStatus() != 0) { if (log.isDetailed()) { logDetailed( BaseMessages.getString( PKG, "JobShell.ExitStatus", environmentSubstitute(getFilename()), "" + result.getExitStatus())); } result.setNrErrors(1); } // wait until loggers read all data from stdout and stderr errorLoggerThread.join(); outputLoggerThread.join(); // close the streams // otherwise you get "Too many open files, java.io.IOException" after a lot of iterations proc.getErrorStream().close(); proc.getOutputStream().close(); } catch (IOException ioe) { logError( BaseMessages.getString( PKG, "JobShell.ErrorRunningShell", environmentSubstitute(getFilename()), ioe.toString()), ioe); result.setNrErrors(1); } catch (InterruptedException ie) { logError( BaseMessages.getString( PKG, "JobShell.Shellinterupted", environmentSubstitute(getFilename()), ie.toString()), ie); result.setNrErrors(1); } catch (Exception e) { logError( BaseMessages.getString( PKG, "JobShell.UnexpectedError", environmentSubstitute(getFilename()), e.toString()), e); result.setNrErrors(1); } finally { // If we created a temporary file, remove it... // if (tempFile != null) { try { tempFile.delete(); } catch (Exception e) { BaseMessages.getString( PKG, "JobShell.UnexpectedError", tempFile.toString(), e.toString()); } } } if (result.getNrErrors() > 0) { result.setResult(false); } else { result.setResult(true); } }