public boolean init(StepMetaInterface smi, StepDataInterface sdi) { meta = (RowGeneratorMeta) smi; data = (RowGeneratorData) sdi; if (super.init(smi, sdi)) { // Determine the number of rows to generate... data.rowLimit = Const.toLong(environmentSubstitute(meta.getRowLimit()), -1L); data.rowsWritten = 0L; if (data.rowLimit < 0L) // Unable to parse { logError(BaseMessages.getString(PKG, "RowGenerator.Wrong.RowLimit.Number")); return false; // fail } // Create a row (constants) with all the values in it... List<CheckResultInterface> remarks = new ArrayList<CheckResultInterface>(); // stores the errors... RowMetaAndData outputRow = buildRow(meta, remarks, getStepname()); if (!remarks.isEmpty()) { for (int i = 0; i < remarks.size(); i++) { CheckResult cr = (CheckResult) remarks.get(i); logError(cr.getText()); } return false; } data.outputRowData = outputRow.getData(); data.outputRowMeta = outputRow.getRowMeta(); return true; } return false; }
private void readData(Node stepnode) throws KettleXMLException { try { urlInField = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "url_in_field")); urlFieldname = XMLHandler.getTagValue(stepnode, "url_field_name"); includeRowNumber = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "rownum")); rowNumberField = XMLHandler.getTagValue(stepnode, "rownum_field"); includeUrl = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "include_url")); urlField = XMLHandler.getTagValue(stepnode, "url_Field"); readfrom = XMLHandler.getTagValue(stepnode, "read_from"); Node fields = XMLHandler.getSubNode(stepnode, "fields"); int nrFields = XMLHandler.countNodes(fields, "field"); Node urlnode = XMLHandler.getSubNode(stepnode, "urls"); int nrUrls = XMLHandler.countNodes(urlnode, "url"); allocate(nrUrls, nrFields); for (int i = 0; i < nrUrls; i++) { Node urlnamenode = XMLHandler.getSubNodeByNr(urlnode, "url", i); url[i] = XMLHandler.getNodeValue(urlnamenode); } for (int i = 0; i < nrFields; i++) { Node fnode = XMLHandler.getSubNodeByNr(fields, "field", i); RssInputField field = new RssInputField(fnode); inputFields[i] = field; } // Is there a limit on the number of rows we process? rowLimit = Const.toLong(XMLHandler.getTagValue(stepnode, "limit"), 0L); } catch (Exception e) { throw new KettleXMLException("Unable to load step info from XML", e); } }
public void check( List<CheckResultInterface> remarks, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String input[], String output[], RowMetaInterface info) { CheckResult cr; if (prev != null && prev.size() > 0) { cr = new CheckResult( CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "RowGeneratorMeta.CheckResult.NoInputStreamsError"), stepMeta); remarks.add(cr); } else { cr = new CheckResult( CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "RowGeneratorMeta.CheckResult.NoInputStreamOk"), stepMeta); remarks.add(cr); String strLimit = transMeta.environmentSubstitute(rowLimit); if (Const.toLong(strLimit, -1L) <= 0) { cr = new CheckResult( CheckResultInterface.TYPE_RESULT_WARNING, BaseMessages.getString(PKG, "RowGeneratorMeta.CheckResult.WarnNoRows"), stepMeta); remarks.add(cr); } else { cr = new CheckResult( CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString( PKG, "RowGeneratorMeta.CheckResult.WillReturnRows", strLimit), stepMeta); remarks.add(cr); } } // See if we have input streams leading to this step! if (input.length > 0) { cr = new CheckResult( CheckResultInterface.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "RowGeneratorMeta.CheckResult.NoInputError"), stepMeta); remarks.add(cr); } else { cr = new CheckResult( CheckResultInterface.TYPE_RESULT_OK, BaseMessages.getString(PKG, "RowGeneratorMeta.CheckResult.NoInputOk"), stepMeta); remarks.add(cr); } }
@Override public AbstractMeta decode(String graphXml) throws Exception { mxGraph graph = new mxGraph(); mxCodec codec = new mxCodec(); Document doc = mxUtils.parseXml(graphXml); codec.decode(doc.getDocumentElement(), graph.getModel()); mxCell root = (mxCell) graph.getDefaultParent(); TransMeta transMeta = new TransMeta(); decodeCommRootAttr(root, transMeta); transMeta.setTransstatus(Const.toInt(root.getAttribute("trans_status"), -1)); transMeta.setTransversion(root.getAttribute("trans_version")); if (transMeta.getRepository() != null) transMeta.setSharedObjects(transMeta.getRepository().readTransSharedObjects(transMeta)); else transMeta.setSharedObjects(transMeta.readSharedObjects()); transMeta.importFromMetaStore(); decodeDatabases(root, transMeta); decodeNote(graph, transMeta); int count = graph.getModel().getChildCount(root); for (int i = 0; i < count; i++) { mxCell cell = (mxCell) graph.getModel().getChildAt(root, i); if (cell.isVertex()) { Element e = (Element) cell.getValue(); if (PropsUI.TRANS_STEP_NAME.equals(e.getTagName())) { StepDecoder stepDecoder = (StepDecoder) PluginFactory.getBean(cell.getAttribute("ctype")); StepMeta stepMeta = stepDecoder.decodeStep(cell, transMeta.getDatabases(), transMeta.getMetaStore()); stepMeta.setParentTransMeta(transMeta); if (stepMeta.isMissing()) { transMeta.addMissingTrans((MissingTrans) stepMeta.getStepMetaInterface()); } StepMeta check = transMeta.findStep(stepMeta.getName()); if (check != null) { if (!check.isShared()) { // Don't overwrite shared objects transMeta.addOrReplaceStep(stepMeta); } else { check.setDraw(stepMeta.isDrawn()); // Just keep the drawn flag and location check.setLocation(stepMeta.getLocation()); } } else { transMeta.addStep(stepMeta); // simply add it. } } } } // Have all StreamValueLookups, etc. reference the correct source steps... // for (int i = 0; i < transMeta.nrSteps(); i++) { StepMeta stepMeta = transMeta.getStep(i); StepMetaInterface sii = stepMeta.getStepMetaInterface(); if (sii != null) { sii.searchInfoAndTargetSteps(transMeta.getSteps()); } } count = graph.getModel().getChildCount(root); for (int i = 0; i < count; i++) { mxCell cell = (mxCell) graph.getModel().getChildAt(root, i); if (cell.isEdge()) { mxCell source = (mxCell) cell.getSource(); mxCell target = (mxCell) cell.getTarget(); TransHopMeta hopinf = new TransHopMeta(null, null, true); String[] stepNames = transMeta.getStepNames(); for (int j = 0; j < stepNames.length; j++) { if (stepNames[j].equalsIgnoreCase(source.getAttribute("label"))) hopinf.setFromStep(transMeta.getStep(j)); if (stepNames[j].equalsIgnoreCase(target.getAttribute("label"))) hopinf.setToStep(transMeta.getStep(j)); } transMeta.addTransHop(hopinf); } } JSONObject jsonObject = JSONObject.fromObject(root.getAttribute("transLogTable")); TransLogTable transLogTable = transMeta.getTransLogTable(); transLogTable.setConnectionName(jsonObject.optString("connection")); transLogTable.setSchemaName(jsonObject.optString("schema")); transLogTable.setTableName(jsonObject.optString("table")); transLogTable.setLogSizeLimit(jsonObject.optString("size_limit_lines")); transLogTable.setLogInterval(jsonObject.optString("interval")); transLogTable.setTimeoutInDays(jsonObject.optString("timeout_days")); JSONArray jsonArray = jsonObject.optJSONArray("fields"); if (jsonArray != null) { for (int i = 0; i < jsonArray.size(); i++) { JSONObject fieldJson = jsonArray.getJSONObject(i); String id = fieldJson.optString("id"); LogTableField field = transLogTable.findField(id); if (field == null) { field = transLogTable.getFields().get(i); } if (field != null) { field.setFieldName(fieldJson.optString("name")); field.setEnabled(fieldJson.optBoolean("enabled")); field.setSubject(StepMeta.findStep(transMeta.getSteps(), fieldJson.optString("subject"))); } } } jsonObject = JSONObject.fromObject(root.getAttribute("stepLogTable")); StepLogTable stepLogTable = transMeta.getStepLogTable(); stepLogTable.setConnectionName(jsonObject.optString("connection")); stepLogTable.setSchemaName(jsonObject.optString("schema")); stepLogTable.setTableName(jsonObject.optString("table")); stepLogTable.setTimeoutInDays(jsonObject.optString("timeout_days")); jsonArray = jsonObject.optJSONArray("fields"); if (jsonArray != null) { for (int i = 0; i < jsonArray.size(); i++) { JSONObject fieldJson = jsonArray.getJSONObject(i); String id = fieldJson.optString("id"); LogTableField field = stepLogTable.findField(id); if (field == null && i < stepLogTable.getFields().size()) { field = stepLogTable.getFields().get(i); } if (field != null) { field.setFieldName(fieldJson.optString("name")); field.setEnabled(fieldJson.optBoolean("enabled")); } } } jsonObject = JSONObject.fromObject(root.getAttribute("performanceLogTable")); PerformanceLogTable performanceLogTable = transMeta.getPerformanceLogTable(); performanceLogTable.setConnectionName(jsonObject.optString("connection")); performanceLogTable.setSchemaName(jsonObject.optString("schema")); performanceLogTable.setTableName(jsonObject.optString("table")); performanceLogTable.setLogInterval(jsonObject.optString("interval")); performanceLogTable.setTimeoutInDays(jsonObject.optString("timeout_days")); jsonArray = jsonObject.optJSONArray("fields"); if (jsonArray != null) { for (int i = 0; i < jsonArray.size(); i++) { JSONObject fieldJson = jsonArray.getJSONObject(i); String id = fieldJson.optString("id"); LogTableField field = performanceLogTable.findField(id); if (field == null && i < performanceLogTable.getFields().size()) { field = performanceLogTable.getFields().get(i); } if (field != null) { field.setFieldName(fieldJson.optString("name")); field.setEnabled(fieldJson.optBoolean("enabled")); } } } jsonObject = JSONObject.fromObject(root.getAttribute("metricsLogTable")); MetricsLogTable metricsLogTable = transMeta.getMetricsLogTable(); metricsLogTable.setConnectionName(jsonObject.optString("connection")); metricsLogTable.setSchemaName(jsonObject.optString("schema")); metricsLogTable.setTableName(jsonObject.optString("table")); metricsLogTable.setTimeoutInDays(jsonObject.optString("timeout_days")); jsonArray = jsonObject.optJSONArray("fields"); if (jsonArray != null) { for (int i = 0; i < jsonArray.size(); i++) { JSONObject fieldJson = jsonArray.getJSONObject(i); String id = fieldJson.optString("id"); LogTableField field = metricsLogTable.findField(id); if (field == null && i < metricsLogTable.getFields().size()) { field = metricsLogTable.getFields().get(i); } if (field != null) { field.setFieldName(fieldJson.optString("name")); field.setEnabled(fieldJson.optBoolean("enabled")); } } } jsonArray = JSONArray.fromObject(root.getAttribute("partitionschemas")); for (int i = 0; i < jsonArray.size(); i++) { jsonObject = jsonArray.getJSONObject(i); PartitionSchema partitionSchema = decodePartitionSchema(jsonObject); PartitionSchema check = transMeta.findPartitionSchema(partitionSchema.getName()); if (check != null) { if (!check.isShared()) { transMeta.addOrReplacePartitionSchema(partitionSchema); } } else { transMeta.getPartitionSchemas().add(partitionSchema); } } decodeSlaveServers(root, transMeta); jsonArray = JSONArray.fromObject(root.getAttribute("clusterSchemas")); for (int i = 0; i < jsonArray.size(); i++) { jsonObject = jsonArray.getJSONObject(i); ClusterSchema clusterSchema = decodeClusterSchema(jsonObject, transMeta.getSlaveServers()); clusterSchema.shareVariablesWith(transMeta); ClusterSchema check = transMeta.findClusterSchema(clusterSchema.getName()); if (check != null) { if (!check.isShared()) { transMeta.addOrReplaceClusterSchema(clusterSchema); } } else { transMeta.getClusterSchemas().add(clusterSchema); } } for (int i = 0; i < transMeta.nrSteps(); i++) { transMeta.getStep(i).setClusterSchemaAfterLoading(transMeta.getClusterSchemas()); } transMeta.setSizeRowset(Const.toInt(root.getAttribute("size_rowset"), Const.ROWS_IN_ROWSET)); transMeta.setSleepTimeEmpty( Const.toInt(root.getAttribute("sleep_time_empty"), Const.TIMEOUT_GET_MILLIS)); transMeta.setSleepTimeFull( Const.toInt(root.getAttribute("sleep_time_full"), Const.TIMEOUT_PUT_MILLIS)); transMeta.setUsingUniqueConnections( "Y".equalsIgnoreCase(root.getAttribute("unique_connections"))); transMeta.setFeedbackShown(!"N".equalsIgnoreCase(root.getAttribute("feedback_shown"))); transMeta.setFeedbackSize(Const.toInt(root.getAttribute("feedback_size"), Const.ROWS_UPDATE)); transMeta.setUsingThreadPriorityManagment( !"N".equalsIgnoreCase(root.getAttribute("using_thread_priorities"))); transMeta.setCapturingStepPerformanceSnapShots( "Y".equalsIgnoreCase(root.getAttribute("capture_step_performance"))); transMeta.setStepPerformanceCapturingDelay( Const.toLong(root.getAttribute("step_performance_capturing_delay"), 1000)); transMeta.setStepPerformanceCapturingSizeLimit( root.getAttribute("step_performance_capturing_size_limit")); transMeta.setKey(XMLHandler.stringToBinary(root.getAttribute("key_for_session_key"))); transMeta.setPrivateKey("Y".equals(root.getAttribute("is_key_private"))); return transMeta; }
private void readData(Node stepnode) throws KettleXMLException { try { fileType = XMLHandler.getTagValue(stepnode, "file_type"); encoding = XMLHandler.getTagValue(stepnode, "encoding"); includeFilename = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "include")); filenameField = XMLHandler.getTagValue(stepnode, "include_field"); includeRowNumber = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "rownum")); String addresult = XMLHandler.getTagValue(stepnode, "isaddresult"); if (Utils.isEmpty(addresult)) { isaddresult = true; } else { isaddresult = "Y".equalsIgnoreCase(addresult); } section = XMLHandler.getTagValue(stepnode, "section"); iniSectionField = XMLHandler.getTagValue(stepnode, "ini_section_field"); includeIniSection = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "ini_section")); filefield = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "filefield")); rowNumberField = XMLHandler.getTagValue(stepnode, "rownum_field"); dynamicFilenameField = XMLHandler.getTagValue(stepnode, "filename_Field"); resetRowNumber = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "resetrownumber")); resolvevaluevariable = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "resolvevaluevariable")); Node filenode = XMLHandler.getSubNode(stepnode, "file"); Node fields = XMLHandler.getSubNode(stepnode, "fields"); int nrFiles = XMLHandler.countNodes(filenode, "name"); int nrFields = XMLHandler.countNodes(fields, "field"); allocate(nrFiles, nrFields); for (int i = 0; i < nrFiles; i++) { Node filenamenode = XMLHandler.getSubNodeByNr(filenode, "name", i); Node filemasknode = XMLHandler.getSubNodeByNr(filenode, "filemask", i); Node excludefilemasknode = XMLHandler.getSubNodeByNr(filenode, "exclude_filemask", i); Node fileRequirednode = XMLHandler.getSubNodeByNr(filenode, "file_required", i); Node includeSubFoldersnode = XMLHandler.getSubNodeByNr(filenode, "include_subfolders", i); fileName[i] = XMLHandler.getNodeValue(filenamenode); fileMask[i] = XMLHandler.getNodeValue(filemasknode); excludeFileMask[i] = XMLHandler.getNodeValue(excludefilemasknode); fileRequired[i] = XMLHandler.getNodeValue(fileRequirednode); includeSubFolders[i] = XMLHandler.getNodeValue(includeSubFoldersnode); } for (int i = 0; i < nrFields; i++) { Node fnode = XMLHandler.getSubNodeByNr(fields, "field", i); inputFields[i] = new PropertyInputField(); inputFields[i].setName(XMLHandler.getTagValue(fnode, "name")); inputFields[i].setColumn(getColumnByCode(XMLHandler.getTagValue(fnode, "column"))); inputFields[i].setType( ValueMetaFactory.getIdForValueMeta(XMLHandler.getTagValue(fnode, "type"))); inputFields[i].setLength(Const.toInt(XMLHandler.getTagValue(fnode, "length"), -1)); inputFields[i].setPrecision(Const.toInt(XMLHandler.getTagValue(fnode, "precision"), -1)); String srepeat = XMLHandler.getTagValue(fnode, "repeat"); inputFields[i].setTrimType(getTrimTypeByCode(XMLHandler.getTagValue(fnode, "trim_type"))); if (srepeat != null) { inputFields[i].setRepeated(YES.equalsIgnoreCase(srepeat)); } else { inputFields[i].setRepeated(false); } inputFields[i].setFormat(XMLHandler.getTagValue(fnode, "format")); inputFields[i].setCurrencySymbol(XMLHandler.getTagValue(fnode, "currency")); inputFields[i].setDecimalSymbol(XMLHandler.getTagValue(fnode, "decimal")); inputFields[i].setGroupSymbol(XMLHandler.getTagValue(fnode, "group")); } // Is there a limit on the number of rows we process? rowLimit = Const.toLong(XMLHandler.getTagValue(stepnode, "limit"), 0L); shortFileFieldName = XMLHandler.getTagValue(stepnode, "shortFileFieldName"); pathFieldName = XMLHandler.getTagValue(stepnode, "pathFieldName"); hiddenFieldName = XMLHandler.getTagValue(stepnode, "hiddenFieldName"); lastModificationTimeFieldName = XMLHandler.getTagValue(stepnode, "lastModificationTimeFieldName"); uriNameFieldName = XMLHandler.getTagValue(stepnode, "uriNameFieldName"); rootUriNameFieldName = XMLHandler.getTagValue(stepnode, "rootUriNameFieldName"); extensionFieldName = XMLHandler.getTagValue(stepnode, "extensionFieldName"); sizeFieldName = XMLHandler.getTagValue(stepnode, "sizeFieldName"); } catch (Exception e) { throw new KettleXMLException("Unable to load step info from XML", e); } }