public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
meta = (RowGeneratorMeta) smi;
data = (RowGeneratorData) sdi;
if (super.init(smi, sdi)) {
// Determine the number of rows to generate...
data.rowLimit = Const.toLong(environmentSubstitute(meta.getRowLimit()), -1L);
data.rowsWritten = 0L;
if (data.rowLimit < 0L) // Unable to parse
{
logError(BaseMessages.getString(PKG, "RowGenerator.Wrong.RowLimit.Number"));
return false; // fail
}
// Create a row (constants) with all the values in it...
List<CheckResultInterface> remarks =
new ArrayList<CheckResultInterface>(); // stores the errors...
RowMetaAndData outputRow = buildRow(meta, remarks, getStepname());
if (!remarks.isEmpty()) {
for (int i = 0; i < remarks.size(); i++) {
CheckResult cr = (CheckResult) remarks.get(i);
logError(cr.getText());
}
return false;
}
data.outputRowData = outputRow.getData();
data.outputRowMeta = outputRow.getRowMeta();
return true;
}
return false;
}
private void readData(Node stepnode) throws KettleXMLException {
try {
urlInField = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "url_in_field"));
urlFieldname = XMLHandler.getTagValue(stepnode, "url_field_name");
includeRowNumber = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "rownum"));
rowNumberField = XMLHandler.getTagValue(stepnode, "rownum_field");
includeUrl = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "include_url"));
urlField = XMLHandler.getTagValue(stepnode, "url_Field");
readfrom = XMLHandler.getTagValue(stepnode, "read_from");
Node fields = XMLHandler.getSubNode(stepnode, "fields");
int nrFields = XMLHandler.countNodes(fields, "field");
Node urlnode = XMLHandler.getSubNode(stepnode, "urls");
int nrUrls = XMLHandler.countNodes(urlnode, "url");
allocate(nrUrls, nrFields);
for (int i = 0; i < nrUrls; i++) {
Node urlnamenode = XMLHandler.getSubNodeByNr(urlnode, "url", i);
url[i] = XMLHandler.getNodeValue(urlnamenode);
}
for (int i = 0; i < nrFields; i++) {
Node fnode = XMLHandler.getSubNodeByNr(fields, "field", i);
RssInputField field = new RssInputField(fnode);
inputFields[i] = field;
}
// Is there a limit on the number of rows we process?
rowLimit = Const.toLong(XMLHandler.getTagValue(stepnode, "limit"), 0L);
} catch (Exception e) {
throw new KettleXMLException("Unable to load step info from XML", e);
}
}
public void check(
List<CheckResultInterface> remarks,
TransMeta transMeta,
StepMeta stepMeta,
RowMetaInterface prev,
String input[],
String output[],
RowMetaInterface info) {
CheckResult cr;
if (prev != null && prev.size() > 0) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_ERROR,
BaseMessages.getString(PKG, "RowGeneratorMeta.CheckResult.NoInputStreamsError"),
stepMeta);
remarks.add(cr);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(PKG, "RowGeneratorMeta.CheckResult.NoInputStreamOk"),
stepMeta);
remarks.add(cr);
String strLimit = transMeta.environmentSubstitute(rowLimit);
if (Const.toLong(strLimit, -1L) <= 0) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_WARNING,
BaseMessages.getString(PKG, "RowGeneratorMeta.CheckResult.WarnNoRows"),
stepMeta);
remarks.add(cr);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(
PKG, "RowGeneratorMeta.CheckResult.WillReturnRows", strLimit),
stepMeta);
remarks.add(cr);
}
}
// See if we have input streams leading to this step!
if (input.length > 0) {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_ERROR,
BaseMessages.getString(PKG, "RowGeneratorMeta.CheckResult.NoInputError"),
stepMeta);
remarks.add(cr);
} else {
cr =
new CheckResult(
CheckResultInterface.TYPE_RESULT_OK,
BaseMessages.getString(PKG, "RowGeneratorMeta.CheckResult.NoInputOk"),
stepMeta);
remarks.add(cr);
}
}
@Override
public AbstractMeta decode(String graphXml) throws Exception {
mxGraph graph = new mxGraph();
mxCodec codec = new mxCodec();
Document doc = mxUtils.parseXml(graphXml);
codec.decode(doc.getDocumentElement(), graph.getModel());
mxCell root = (mxCell) graph.getDefaultParent();
TransMeta transMeta = new TransMeta();
decodeCommRootAttr(root, transMeta);
transMeta.setTransstatus(Const.toInt(root.getAttribute("trans_status"), -1));
transMeta.setTransversion(root.getAttribute("trans_version"));
if (transMeta.getRepository() != null)
transMeta.setSharedObjects(transMeta.getRepository().readTransSharedObjects(transMeta));
else transMeta.setSharedObjects(transMeta.readSharedObjects());
transMeta.importFromMetaStore();
decodeDatabases(root, transMeta);
decodeNote(graph, transMeta);
int count = graph.getModel().getChildCount(root);
for (int i = 0; i < count; i++) {
mxCell cell = (mxCell) graph.getModel().getChildAt(root, i);
if (cell.isVertex()) {
Element e = (Element) cell.getValue();
if (PropsUI.TRANS_STEP_NAME.equals(e.getTagName())) {
StepDecoder stepDecoder = (StepDecoder) PluginFactory.getBean(cell.getAttribute("ctype"));
StepMeta stepMeta =
stepDecoder.decodeStep(cell, transMeta.getDatabases(), transMeta.getMetaStore());
stepMeta.setParentTransMeta(transMeta);
if (stepMeta.isMissing()) {
transMeta.addMissingTrans((MissingTrans) stepMeta.getStepMetaInterface());
}
StepMeta check = transMeta.findStep(stepMeta.getName());
if (check != null) {
if (!check.isShared()) {
// Don't overwrite shared objects
transMeta.addOrReplaceStep(stepMeta);
} else {
check.setDraw(stepMeta.isDrawn()); // Just keep the drawn flag and location
check.setLocation(stepMeta.getLocation());
}
} else {
transMeta.addStep(stepMeta); // simply add it.
}
}
}
}
// Have all StreamValueLookups, etc. reference the correct source steps...
//
for (int i = 0; i < transMeta.nrSteps(); i++) {
StepMeta stepMeta = transMeta.getStep(i);
StepMetaInterface sii = stepMeta.getStepMetaInterface();
if (sii != null) {
sii.searchInfoAndTargetSteps(transMeta.getSteps());
}
}
count = graph.getModel().getChildCount(root);
for (int i = 0; i < count; i++) {
mxCell cell = (mxCell) graph.getModel().getChildAt(root, i);
if (cell.isEdge()) {
mxCell source = (mxCell) cell.getSource();
mxCell target = (mxCell) cell.getTarget();
TransHopMeta hopinf = new TransHopMeta(null, null, true);
String[] stepNames = transMeta.getStepNames();
for (int j = 0; j < stepNames.length; j++) {
if (stepNames[j].equalsIgnoreCase(source.getAttribute("label")))
hopinf.setFromStep(transMeta.getStep(j));
if (stepNames[j].equalsIgnoreCase(target.getAttribute("label")))
hopinf.setToStep(transMeta.getStep(j));
}
transMeta.addTransHop(hopinf);
}
}
JSONObject jsonObject = JSONObject.fromObject(root.getAttribute("transLogTable"));
TransLogTable transLogTable = transMeta.getTransLogTable();
transLogTable.setConnectionName(jsonObject.optString("connection"));
transLogTable.setSchemaName(jsonObject.optString("schema"));
transLogTable.setTableName(jsonObject.optString("table"));
transLogTable.setLogSizeLimit(jsonObject.optString("size_limit_lines"));
transLogTable.setLogInterval(jsonObject.optString("interval"));
transLogTable.setTimeoutInDays(jsonObject.optString("timeout_days"));
JSONArray jsonArray = jsonObject.optJSONArray("fields");
if (jsonArray != null) {
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject fieldJson = jsonArray.getJSONObject(i);
String id = fieldJson.optString("id");
LogTableField field = transLogTable.findField(id);
if (field == null) {
field = transLogTable.getFields().get(i);
}
if (field != null) {
field.setFieldName(fieldJson.optString("name"));
field.setEnabled(fieldJson.optBoolean("enabled"));
field.setSubject(StepMeta.findStep(transMeta.getSteps(), fieldJson.optString("subject")));
}
}
}
jsonObject = JSONObject.fromObject(root.getAttribute("stepLogTable"));
StepLogTable stepLogTable = transMeta.getStepLogTable();
stepLogTable.setConnectionName(jsonObject.optString("connection"));
stepLogTable.setSchemaName(jsonObject.optString("schema"));
stepLogTable.setTableName(jsonObject.optString("table"));
stepLogTable.setTimeoutInDays(jsonObject.optString("timeout_days"));
jsonArray = jsonObject.optJSONArray("fields");
if (jsonArray != null) {
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject fieldJson = jsonArray.getJSONObject(i);
String id = fieldJson.optString("id");
LogTableField field = stepLogTable.findField(id);
if (field == null && i < stepLogTable.getFields().size()) {
field = stepLogTable.getFields().get(i);
}
if (field != null) {
field.setFieldName(fieldJson.optString("name"));
field.setEnabled(fieldJson.optBoolean("enabled"));
}
}
}
jsonObject = JSONObject.fromObject(root.getAttribute("performanceLogTable"));
PerformanceLogTable performanceLogTable = transMeta.getPerformanceLogTable();
performanceLogTable.setConnectionName(jsonObject.optString("connection"));
performanceLogTable.setSchemaName(jsonObject.optString("schema"));
performanceLogTable.setTableName(jsonObject.optString("table"));
performanceLogTable.setLogInterval(jsonObject.optString("interval"));
performanceLogTable.setTimeoutInDays(jsonObject.optString("timeout_days"));
jsonArray = jsonObject.optJSONArray("fields");
if (jsonArray != null) {
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject fieldJson = jsonArray.getJSONObject(i);
String id = fieldJson.optString("id");
LogTableField field = performanceLogTable.findField(id);
if (field == null && i < performanceLogTable.getFields().size()) {
field = performanceLogTable.getFields().get(i);
}
if (field != null) {
field.setFieldName(fieldJson.optString("name"));
field.setEnabled(fieldJson.optBoolean("enabled"));
}
}
}
jsonObject = JSONObject.fromObject(root.getAttribute("metricsLogTable"));
MetricsLogTable metricsLogTable = transMeta.getMetricsLogTable();
metricsLogTable.setConnectionName(jsonObject.optString("connection"));
metricsLogTable.setSchemaName(jsonObject.optString("schema"));
metricsLogTable.setTableName(jsonObject.optString("table"));
metricsLogTable.setTimeoutInDays(jsonObject.optString("timeout_days"));
jsonArray = jsonObject.optJSONArray("fields");
if (jsonArray != null) {
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject fieldJson = jsonArray.getJSONObject(i);
String id = fieldJson.optString("id");
LogTableField field = metricsLogTable.findField(id);
if (field == null && i < metricsLogTable.getFields().size()) {
field = metricsLogTable.getFields().get(i);
}
if (field != null) {
field.setFieldName(fieldJson.optString("name"));
field.setEnabled(fieldJson.optBoolean("enabled"));
}
}
}
jsonArray = JSONArray.fromObject(root.getAttribute("partitionschemas"));
for (int i = 0; i < jsonArray.size(); i++) {
jsonObject = jsonArray.getJSONObject(i);
PartitionSchema partitionSchema = decodePartitionSchema(jsonObject);
PartitionSchema check = transMeta.findPartitionSchema(partitionSchema.getName());
if (check != null) {
if (!check.isShared()) {
transMeta.addOrReplacePartitionSchema(partitionSchema);
}
} else {
transMeta.getPartitionSchemas().add(partitionSchema);
}
}
decodeSlaveServers(root, transMeta);
jsonArray = JSONArray.fromObject(root.getAttribute("clusterSchemas"));
for (int i = 0; i < jsonArray.size(); i++) {
jsonObject = jsonArray.getJSONObject(i);
ClusterSchema clusterSchema = decodeClusterSchema(jsonObject, transMeta.getSlaveServers());
clusterSchema.shareVariablesWith(transMeta);
ClusterSchema check = transMeta.findClusterSchema(clusterSchema.getName());
if (check != null) {
if (!check.isShared()) {
transMeta.addOrReplaceClusterSchema(clusterSchema);
}
} else {
transMeta.getClusterSchemas().add(clusterSchema);
}
}
for (int i = 0; i < transMeta.nrSteps(); i++) {
transMeta.getStep(i).setClusterSchemaAfterLoading(transMeta.getClusterSchemas());
}
transMeta.setSizeRowset(Const.toInt(root.getAttribute("size_rowset"), Const.ROWS_IN_ROWSET));
transMeta.setSleepTimeEmpty(
Const.toInt(root.getAttribute("sleep_time_empty"), Const.TIMEOUT_GET_MILLIS));
transMeta.setSleepTimeFull(
Const.toInt(root.getAttribute("sleep_time_full"), Const.TIMEOUT_PUT_MILLIS));
transMeta.setUsingUniqueConnections(
"Y".equalsIgnoreCase(root.getAttribute("unique_connections")));
transMeta.setFeedbackShown(!"N".equalsIgnoreCase(root.getAttribute("feedback_shown")));
transMeta.setFeedbackSize(Const.toInt(root.getAttribute("feedback_size"), Const.ROWS_UPDATE));
transMeta.setUsingThreadPriorityManagment(
!"N".equalsIgnoreCase(root.getAttribute("using_thread_priorities")));
transMeta.setCapturingStepPerformanceSnapShots(
"Y".equalsIgnoreCase(root.getAttribute("capture_step_performance")));
transMeta.setStepPerformanceCapturingDelay(
Const.toLong(root.getAttribute("step_performance_capturing_delay"), 1000));
transMeta.setStepPerformanceCapturingSizeLimit(
root.getAttribute("step_performance_capturing_size_limit"));
transMeta.setKey(XMLHandler.stringToBinary(root.getAttribute("key_for_session_key")));
transMeta.setPrivateKey("Y".equals(root.getAttribute("is_key_private")));
return transMeta;
}
private void readData(Node stepnode) throws KettleXMLException {
try {
fileType = XMLHandler.getTagValue(stepnode, "file_type");
encoding = XMLHandler.getTagValue(stepnode, "encoding");
includeFilename = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "include"));
filenameField = XMLHandler.getTagValue(stepnode, "include_field");
includeRowNumber = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "rownum"));
String addresult = XMLHandler.getTagValue(stepnode, "isaddresult");
if (Utils.isEmpty(addresult)) {
isaddresult = true;
} else {
isaddresult = "Y".equalsIgnoreCase(addresult);
}
section = XMLHandler.getTagValue(stepnode, "section");
iniSectionField = XMLHandler.getTagValue(stepnode, "ini_section_field");
includeIniSection = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "ini_section"));
filefield = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "filefield"));
rowNumberField = XMLHandler.getTagValue(stepnode, "rownum_field");
dynamicFilenameField = XMLHandler.getTagValue(stepnode, "filename_Field");
resetRowNumber = "Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "resetrownumber"));
resolvevaluevariable =
"Y".equalsIgnoreCase(XMLHandler.getTagValue(stepnode, "resolvevaluevariable"));
Node filenode = XMLHandler.getSubNode(stepnode, "file");
Node fields = XMLHandler.getSubNode(stepnode, "fields");
int nrFiles = XMLHandler.countNodes(filenode, "name");
int nrFields = XMLHandler.countNodes(fields, "field");
allocate(nrFiles, nrFields);
for (int i = 0; i < nrFiles; i++) {
Node filenamenode = XMLHandler.getSubNodeByNr(filenode, "name", i);
Node filemasknode = XMLHandler.getSubNodeByNr(filenode, "filemask", i);
Node excludefilemasknode = XMLHandler.getSubNodeByNr(filenode, "exclude_filemask", i);
Node fileRequirednode = XMLHandler.getSubNodeByNr(filenode, "file_required", i);
Node includeSubFoldersnode = XMLHandler.getSubNodeByNr(filenode, "include_subfolders", i);
fileName[i] = XMLHandler.getNodeValue(filenamenode);
fileMask[i] = XMLHandler.getNodeValue(filemasknode);
excludeFileMask[i] = XMLHandler.getNodeValue(excludefilemasknode);
fileRequired[i] = XMLHandler.getNodeValue(fileRequirednode);
includeSubFolders[i] = XMLHandler.getNodeValue(includeSubFoldersnode);
}
for (int i = 0; i < nrFields; i++) {
Node fnode = XMLHandler.getSubNodeByNr(fields, "field", i);
inputFields[i] = new PropertyInputField();
inputFields[i].setName(XMLHandler.getTagValue(fnode, "name"));
inputFields[i].setColumn(getColumnByCode(XMLHandler.getTagValue(fnode, "column")));
inputFields[i].setType(
ValueMetaFactory.getIdForValueMeta(XMLHandler.getTagValue(fnode, "type")));
inputFields[i].setLength(Const.toInt(XMLHandler.getTagValue(fnode, "length"), -1));
inputFields[i].setPrecision(Const.toInt(XMLHandler.getTagValue(fnode, "precision"), -1));
String srepeat = XMLHandler.getTagValue(fnode, "repeat");
inputFields[i].setTrimType(getTrimTypeByCode(XMLHandler.getTagValue(fnode, "trim_type")));
if (srepeat != null) {
inputFields[i].setRepeated(YES.equalsIgnoreCase(srepeat));
} else {
inputFields[i].setRepeated(false);
}
inputFields[i].setFormat(XMLHandler.getTagValue(fnode, "format"));
inputFields[i].setCurrencySymbol(XMLHandler.getTagValue(fnode, "currency"));
inputFields[i].setDecimalSymbol(XMLHandler.getTagValue(fnode, "decimal"));
inputFields[i].setGroupSymbol(XMLHandler.getTagValue(fnode, "group"));
}
// Is there a limit on the number of rows we process?
rowLimit = Const.toLong(XMLHandler.getTagValue(stepnode, "limit"), 0L);
shortFileFieldName = XMLHandler.getTagValue(stepnode, "shortFileFieldName");
pathFieldName = XMLHandler.getTagValue(stepnode, "pathFieldName");
hiddenFieldName = XMLHandler.getTagValue(stepnode, "hiddenFieldName");
lastModificationTimeFieldName =
XMLHandler.getTagValue(stepnode, "lastModificationTimeFieldName");
uriNameFieldName = XMLHandler.getTagValue(stepnode, "uriNameFieldName");
rootUriNameFieldName = XMLHandler.getTagValue(stepnode, "rootUriNameFieldName");
extensionFieldName = XMLHandler.getTagValue(stepnode, "extensionFieldName");
sizeFieldName = XMLHandler.getTagValue(stepnode, "sizeFieldName");
} catch (Exception e) {
throw new KettleXMLException("Unable to load step info from XML", e);
}
}
