@Test public void testErrorRedirect() throws Exception { JsonInputField field = new JsonInputField("value"); field.setPath("$.value"); field.setType(ValueMetaInterface.TYPE_STRING); String input1 = "{{"; String input2 = "{ \"value\": \"ok\" }"; JsonInputMeta meta = createSimpleMeta("json", field); meta.setRemoveSourceField(true); when(helper.stepMeta.isDoingErrorHandling()).thenReturn(true); JsonInput jsonInput = createJsonInput("json", meta, new Object[] {input1}, new Object[] {input2}); StepErrorMeta errMeta = new StepErrorMeta(jsonInput, helper.stepMeta); errMeta.setEnabled(true); errMeta.setErrorFieldsValuename("err field"); when(helper.stepMeta.getStepErrorMeta()).thenReturn(errMeta); final List<Object[]> errorLines = new ArrayList<>(); jsonInput.addRowListener( new RowComparatorListener(new Object[] {"ok"}) { @Override public void errorRowWrittenEvent(RowMetaInterface rowMeta, Object[] row) throws KettleStepException { errorLines.add(row); } }); processRows(jsonInput, 3); Assert.assertEquals("fwd error", 1, errorLines.size()); Assert.assertEquals("input in err line", input1, errorLines.get(0)[0]); Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten()); }
public StepErrorMeta loadStepErrorMeta( VariableSpace variables, StepMeta stepMeta, List<StepMeta> steps) throws KettleException { StepErrorMeta meta = new StepErrorMeta(variables, stepMeta); meta.setTargetStep( StepMeta.findStep( steps, repository.getStepAttributeString( stepMeta.getObjectId(), "step_error_handling_target_step"))); meta.setEnabled( repository.getStepAttributeBoolean( stepMeta.getObjectId(), "step_error_handling_is_enabled")); meta.setNrErrorsValuename( repository.getStepAttributeString( stepMeta.getObjectId(), "step_error_handling_nr_valuename")); meta.setErrorDescriptionsValuename( repository.getStepAttributeString( stepMeta.getObjectId(), "step_error_handling_descriptions_valuename")); meta.setErrorFieldsValuename( repository.getStepAttributeString( stepMeta.getObjectId(), "step_error_handling_fields_valuename")); meta.setErrorCodesValuename( repository.getStepAttributeString( stepMeta.getObjectId(), "step_error_handling_codes_valuename")); meta.setMaxErrors( repository.getStepAttributeString( stepMeta.getObjectId(), "step_error_handling_max_errors")); meta.setMaxPercentErrors( repository.getStepAttributeString( stepMeta.getObjectId(), "step_error_handling_max_pct_errors")); meta.setMinPercentRows( repository.getStepAttributeString( stepMeta.getObjectId(), "step_error_handling_min_pct_rows")); return meta; }
public void saveStepErrorMeta(StepErrorMeta meta, ObjectId id_transformation, ObjectId id_step) throws KettleException { repository.saveStepAttribute( id_transformation, id_step, "step_error_handling_source_step", meta.getSourceStep() != null ? meta.getSourceStep().getName() : ""); repository.saveStepAttribute( id_transformation, id_step, "step_error_handling_target_step", meta.getTargetStep() != null ? meta.getTargetStep().getName() : ""); repository.saveStepAttribute( id_transformation, id_step, "step_error_handling_is_enabled", meta.isEnabled()); repository.saveStepAttribute( id_transformation, id_step, "step_error_handling_nr_valuename", meta.getNrErrorsValuename()); repository.saveStepAttribute( id_transformation, id_step, "step_error_handling_descriptions_valuename", meta.getErrorDescriptionsValuename()); repository.saveStepAttribute( id_transformation, id_step, "step_error_handling_fields_valuename", meta.getErrorFieldsValuename()); repository.saveStepAttribute( id_transformation, id_step, "step_error_handling_codes_valuename", meta.getErrorCodesValuename()); repository.saveStepAttribute( id_transformation, id_step, "step_error_handling_max_errors", meta.getMaxErrors()); repository.saveStepAttribute( id_transformation, id_step, "step_error_handling_max_pct_errors", meta.getMaxPercentErrors()); repository.saveStepAttribute( id_transformation, id_step, "step_error_handling_min_pct_rows", meta.getMinPercentRows()); }
public DataNode elementToDataNode(final RepositoryElementInterface element) throws KettleException { TransMeta transMeta = (TransMeta) element; DataNode rootNode = new DataNode(NODE_TRANS); DataNode stepsNode = rootNode.addNode(NODE_STEPS); if (transMeta.getPrivateTransformationDatabases() != null) { // save all private transformations database name http://jira.pentaho.com/browse/PPP-3405 DataNode privateDatabaseNode = rootNode.addNode(NODE_TRANS_PRIVATE_DATABASES); for (String privateDatabase : transMeta.getPrivateTransformationDatabases()) { privateDatabaseNode.addNode(privateDatabase); } } // Also save all the steps in the transformation! // int stepNr = 0; for (StepMeta step : transMeta.getSteps()) { stepNr++; DataNode stepNode = stepsNode.addNode( sanitizeNodeName(step.getName()) + "_" + stepNr + EXT_STEP); // $NON-NLS-1$ // Store the main data // stepNode.setProperty(PROP_NAME, step.getName()); stepNode.setProperty(PROP_DESCRIPTION, step.getDescription()); stepNode.setProperty(PROP_STEP_TYPE, step.getStepID()); stepNode.setProperty(PROP_STEP_DISTRIBUTE, step.isDistributes()); stepNode.setProperty( PROP_STEP_ROW_DISTRIBUTION, step.getRowDistribution() == null ? null : step.getRowDistribution().getCode()); stepNode.setProperty(PROP_STEP_COPIES, step.getCopies()); stepNode.setProperty(PROP_STEP_COPIES_STRING, step.getCopiesString()); stepNode.setProperty(PROP_STEP_GUI_LOCATION_X, step.getLocation().x); stepNode.setProperty(PROP_STEP_GUI_LOCATION_Y, step.getLocation().y); stepNode.setProperty(PROP_STEP_GUI_DRAW, step.isDrawn()); // Also save the step group attributes map // AttributesMapUtil.saveAttributesMap(stepNode, step); // Save the step metadata using the repository save method, NOT XML // That is because we want to keep the links to databases, conditions, etc by ID, not name. // StepMetaInterface stepMetaInterface = step.getStepMetaInterface(); DataNode stepCustomNode = new DataNode(NODE_STEP_CUSTOM); Repository proxy = new RepositoryProxy(stepCustomNode); compatibleSaveRep(stepMetaInterface, proxy, null, null); stepMetaInterface.saveRep(proxy, proxy.getMetaStore(), null, null); stepNode.addNode(stepCustomNode); // Save the partitioning information by reference as well... // StepPartitioningMeta partitioningMeta = step.getStepPartitioningMeta(); if (partitioningMeta != null && partitioningMeta.getPartitionSchema() != null && partitioningMeta.isPartitioned()) { DataNodeRef ref = new DataNodeRef(partitioningMeta.getPartitionSchema().getObjectId().getId()); stepNode.setProperty(PROP_PARTITIONING_SCHEMA, ref); stepNode.setProperty( PROP_PARTITIONING_METHOD, partitioningMeta.getMethodCode()); // method of partitioning if (partitioningMeta.getPartitioner() != null) { DataNode partitionerCustomNode = new DataNode(NODE_PARTITIONER_CUSTOM); proxy = new RepositoryProxy(partitionerCustomNode); partitioningMeta.getPartitioner().saveRep(proxy, null, null); stepNode.addNode(partitionerCustomNode); } } // Save the clustering information as well... // stepNode.setProperty( PROP_CLUSTER_SCHEMA, step.getClusterSchema() == null ? "" : step.getClusterSchema() //$NON-NLS-1$ .getName()); // Save the error hop metadata // StepErrorMeta stepErrorMeta = step.getStepErrorMeta(); if (stepErrorMeta != null) { stepNode.setProperty( PROP_STEP_ERROR_HANDLING_SOURCE_STEP, stepErrorMeta.getSourceStep() != null ? stepErrorMeta.getSourceStep().getName() : ""); //$NON-NLS-1$ stepNode.setProperty( PROP_STEP_ERROR_HANDLING_TARGET_STEP, stepErrorMeta.getTargetStep() != null ? stepErrorMeta.getTargetStep().getName() : ""); //$NON-NLS-1$ stepNode.setProperty(PROP_STEP_ERROR_HANDLING_IS_ENABLED, stepErrorMeta.isEnabled()); stepNode.setProperty( PROP_STEP_ERROR_HANDLING_NR_VALUENAME, stepErrorMeta.getNrErrorsValuename()); stepNode.setProperty( PROP_STEP_ERROR_HANDLING_DESCRIPTIONS_VALUENAME, stepErrorMeta.getErrorDescriptionsValuename()); stepNode.setProperty( PROP_STEP_ERROR_HANDLING_FIELDS_VALUENAME, stepErrorMeta.getErrorFieldsValuename()); stepNode.setProperty( PROP_STEP_ERROR_HANDLING_CODES_VALUENAME, stepErrorMeta.getErrorCodesValuename()); stepNode.setProperty(PROP_STEP_ERROR_HANDLING_MAX_ERRORS, stepErrorMeta.getMaxErrors()); stepNode.setProperty( PROP_STEP_ERROR_HANDLING_MAX_PCT_ERRORS, stepErrorMeta.getMaxPercentErrors()); stepNode.setProperty( PROP_STEP_ERROR_HANDLING_MIN_PCT_ROWS, stepErrorMeta.getMinPercentRows()); } } // Save the notes // DataNode notesNode = rootNode.addNode(NODE_NOTES); notesNode.setProperty(PROP_NR_NOTES, transMeta.nrNotes()); for (int i = 0; i < transMeta.nrNotes(); i++) { NotePadMeta note = transMeta.getNote(i); DataNode noteNode = notesNode.addNode(NOTE_PREFIX + i); noteNode.setProperty(PROP_XML, note.getXML()); } // Finally, save the hops // DataNode hopsNode = rootNode.addNode(NODE_HOPS); hopsNode.setProperty(PROP_NR_HOPS, transMeta.nrTransHops()); for (int i = 0; i < transMeta.nrTransHops(); i++) { TransHopMeta hop = transMeta.getTransHop(i); DataNode hopNode = hopsNode.addNode(TRANS_HOP_PREFIX + i); hopNode.setProperty(TRANS_HOP_FROM, hop.getFromStep().getName()); hopNode.setProperty(TRANS_HOP_TO, hop.getToStep().getName()); hopNode.setProperty(TRANS_HOP_ENABLED, hop.isEnabled()); } // Parameters // String[] paramKeys = transMeta.listParameters(); DataNode paramsNode = rootNode.addNode(NODE_PARAMETERS); paramsNode.setProperty(PROP_NR_PARAMETERS, paramKeys == null ? 0 : paramKeys.length); for (int idx = 0; idx < paramKeys.length; idx++) { DataNode paramNode = paramsNode.addNode(TRANS_PARAM_PREFIX + idx); String key = paramKeys[idx]; String description = transMeta.getParameterDescription(paramKeys[idx]); String defaultValue = transMeta.getParameterDefault(paramKeys[idx]); paramNode.setProperty(PARAM_KEY, key != null ? key : ""); // $NON-NLS-1$ paramNode.setProperty(PARAM_DEFAULT, defaultValue != null ? defaultValue : ""); // $NON-NLS-1$ paramNode.setProperty(PARAM_DESC, description != null ? description : ""); // $NON-NLS-1$ } // Let's not forget to save the details of the transformation itself. // This includes logging information, parameters, etc. // saveTransformationDetails(rootNode, transMeta); return rootNode; }
public void dataNodeToElement(final DataNode rootNode, final RepositoryElementInterface element) throws KettleException { TransMeta transMeta = (TransMeta) element; List<String> privateTransformationDatabases = null; // read the private databases DataNode privateDatabases = rootNode.getNode(NODE_TRANS_PRIVATE_DATABASES); // if we have node than we use new format we could remove unexpected node if (privateDatabases != null) { privateTransformationDatabases = new ArrayList<String>(); for (DataNode privateDatabase : privateDatabases.getNodes()) { privateTransformationDatabases.add(privateDatabase.getName()); } } transMeta.setPrivateTransformationDatabases(privateTransformationDatabases); // read the steps... // DataNode stepsNode = rootNode.getNode(NODE_STEPS); for (DataNode stepNode : stepsNode.getNodes()) { StepMeta stepMeta = new StepMeta(new StringObjectId(stepNode.getId().toString())); stepMeta.setParentTransMeta(transMeta); // for tracing, retain hierarchy // Read the basics // stepMeta.setName(getString(stepNode, PROP_NAME)); if (stepNode.hasProperty(PROP_DESCRIPTION)) { stepMeta.setDescription(getString(stepNode, PROP_DESCRIPTION)); } stepMeta.setDistributes(stepNode.getProperty(PROP_STEP_DISTRIBUTE).getBoolean()); DataProperty rowDistributionProperty = stepNode.getProperty(PROP_STEP_ROW_DISTRIBUTION); String rowDistributionCode = rowDistributionProperty == null ? null : rowDistributionProperty.getString(); RowDistributionInterface rowDistribution = PluginRegistry.getInstance() .loadClass( RowDistributionPluginType.class, rowDistributionCode, RowDistributionInterface.class); stepMeta.setRowDistribution(rowDistribution); stepMeta.setDraw(stepNode.getProperty(PROP_STEP_GUI_DRAW).getBoolean()); int copies = (int) stepNode.getProperty(PROP_STEP_COPIES).getLong(); String copiesString = stepNode.getProperty(PROP_STEP_COPIES_STRING) != null ? stepNode.getProperty(PROP_STEP_COPIES_STRING).getString() : StringUtils.EMPTY; if (!Const.isEmpty(copiesString)) { stepMeta.setCopiesString(copiesString); } else { stepMeta.setCopies(copies); // for backward compatibility } int x = (int) stepNode.getProperty(PROP_STEP_GUI_LOCATION_X).getLong(); int y = (int) stepNode.getProperty(PROP_STEP_GUI_LOCATION_Y).getLong(); stepMeta.setLocation(x, y); // Load the group attributes map // AttributesMapUtil.loadAttributesMap(stepNode, stepMeta); String stepType = getString(stepNode, PROP_STEP_TYPE); // Create a new StepMetaInterface object... // PluginRegistry registry = PluginRegistry.getInstance(); PluginInterface stepPlugin = registry.findPluginWithId(StepPluginType.class, stepType); StepMetaInterface stepMetaInterface = null; if (stepPlugin != null) { stepMetaInterface = (StepMetaInterface) registry.loadClass(stepPlugin); stepType = stepPlugin.getIds()[0]; // revert to the default in case we loaded an alternate version } else { stepMeta.setStepMetaInterface( (StepMetaInterface) new MissingTrans(stepMeta.getName(), stepType)); transMeta.addMissingTrans((MissingTrans) stepMeta.getStepMetaInterface()); } stepMeta.setStepID(stepType); // Read the metadata from the repository too... // RepositoryProxy proxy = new RepositoryProxy(stepNode.getNode(NODE_STEP_CUSTOM)); if (!stepMeta.isMissing()) { readRepCompatibleStepMeta(stepMetaInterface, proxy, null, transMeta.getDatabases()); stepMetaInterface.readRep(proxy, transMeta.getMetaStore(), null, transMeta.getDatabases()); stepMeta.setStepMetaInterface(stepMetaInterface); } // Get the partitioning as well... StepPartitioningMeta stepPartitioningMeta = new StepPartitioningMeta(); if (stepNode.hasProperty(PROP_PARTITIONING_SCHEMA)) { String partSchemaId = stepNode.getProperty(PROP_PARTITIONING_SCHEMA).getRef().getId().toString(); String schemaName = repo.loadPartitionSchema(new StringObjectId(partSchemaId), null).getName(); stepPartitioningMeta.setPartitionSchemaName(schemaName); String methodCode = getString(stepNode, PROP_PARTITIONING_METHOD); stepPartitioningMeta.setMethod(StepPartitioningMeta.getMethod(methodCode)); if (stepPartitioningMeta.getPartitioner() != null) { proxy = new RepositoryProxy(stepNode.getNode(NODE_PARTITIONER_CUSTOM)); stepPartitioningMeta.getPartitioner().loadRep(proxy, null); } stepPartitioningMeta.hasChanged(true); } stepMeta.setStepPartitioningMeta(stepPartitioningMeta); stepMeta .getStepPartitioningMeta() .setPartitionSchemaAfterLoading(transMeta.getPartitionSchemas()); // Get the cluster schema name String clusterSchemaName = getString(stepNode, PROP_CLUSTER_SCHEMA); stepMeta.setClusterSchemaName(clusterSchemaName); if (clusterSchemaName != null && transMeta.getClusterSchemas() != null) { // Get the cluster schema from the given name for (ClusterSchema clusterSchema : transMeta.getClusterSchemas()) { if (clusterSchema.getName().equals(clusterSchemaName)) { stepMeta.setClusterSchema(clusterSchema); break; } } } transMeta.addStep(stepMeta); } for (DataNode stepNode : stepsNode.getNodes()) { ObjectId stepObjectId = new StringObjectId(stepNode.getId().toString()); StepMeta stepMeta = StepMeta.findStep(transMeta.getSteps(), stepObjectId); // Also load the step error handling metadata // if (stepNode.hasProperty(PROP_STEP_ERROR_HANDLING_SOURCE_STEP)) { StepErrorMeta meta = new StepErrorMeta(transMeta, stepMeta); meta.setTargetStep( StepMeta.findStep( transMeta.getSteps(), stepNode.getProperty(PROP_STEP_ERROR_HANDLING_TARGET_STEP).getString())); meta.setEnabled(stepNode.getProperty(PROP_STEP_ERROR_HANDLING_IS_ENABLED).getBoolean()); meta.setNrErrorsValuename(getString(stepNode, PROP_STEP_ERROR_HANDLING_NR_VALUENAME)); meta.setErrorDescriptionsValuename( getString(stepNode, PROP_STEP_ERROR_HANDLING_DESCRIPTIONS_VALUENAME)); meta.setErrorFieldsValuename( getString(stepNode, PROP_STEP_ERROR_HANDLING_FIELDS_VALUENAME)); meta.setErrorCodesValuename(getString(stepNode, PROP_STEP_ERROR_HANDLING_CODES_VALUENAME)); meta.setMaxErrors(getString(stepNode, PROP_STEP_ERROR_HANDLING_MAX_ERRORS)); meta.setMaxPercentErrors(getString(stepNode, PROP_STEP_ERROR_HANDLING_MAX_PCT_ERRORS)); meta.setMinPercentRows(getString(stepNode, PROP_STEP_ERROR_HANDLING_MIN_PCT_ROWS)); meta.getSourceStep().setStepErrorMeta(meta); // a bit of a trick, I know. } } // Have all StreamValueLookups, etc. reference the correct source steps... // for (int i = 0; i < transMeta.nrSteps(); i++) { StepMeta stepMeta = transMeta.getStep(i); StepMetaInterface sii = stepMeta.getStepMetaInterface(); if (sii != null) { sii.searchInfoAndTargetSteps(transMeta.getSteps()); } } // Read the notes... // DataNode notesNode = rootNode.getNode(NODE_NOTES); int nrNotes = (int) notesNode.getProperty(PROP_NR_NOTES).getLong(); for (DataNode noteNode : notesNode.getNodes()) { String xml = getString(noteNode, PROP_XML); transMeta.addNote( new NotePadMeta( XMLHandler.getSubNode(XMLHandler.loadXMLString(xml), NotePadMeta.XML_TAG))); } if (transMeta.nrNotes() != nrNotes) { throw new KettleException( "The number of notes read [" + transMeta.nrNotes() + "] was not the number we expected [" + nrNotes + "]"); } // Read the hops... // DataNode hopsNode = rootNode.getNode(NODE_HOPS); int nrHops = (int) hopsNode.getProperty(PROP_NR_HOPS).getLong(); for (DataNode hopNode : hopsNode.getNodes()) { String stepFromName = getString(hopNode, TRANS_HOP_FROM); String stepToName = getString(hopNode, TRANS_HOP_TO); boolean enabled = true; if (hopNode.hasProperty(TRANS_HOP_ENABLED)) { enabled = hopNode.getProperty(TRANS_HOP_ENABLED).getBoolean(); } StepMeta stepFrom = StepMeta.findStep(transMeta.getSteps(), stepFromName); StepMeta stepTo = StepMeta.findStep(transMeta.getSteps(), stepToName); // Make sure to only accept valid hops PDI-5519 // if (stepFrom != null && stepTo != null) { transMeta.addTransHop(new TransHopMeta(stepFrom, stepTo, enabled)); } } if (transMeta.nrTransHops() != nrHops) { throw new KettleException( "The number of hops read [" + transMeta.nrTransHops() + "] was not the number we expected [" + nrHops + "]"); } // Load the details at the end, to make sure we reference the databases correctly, etc. // loadTransformationDetails(rootNode, transMeta); transMeta.eraseParameters(); DataNode paramsNode = rootNode.getNode(NODE_PARAMETERS); int count = (int) paramsNode.getProperty(PROP_NR_PARAMETERS).getLong(); for (int idx = 0; idx < count; idx++) { DataNode paramNode = paramsNode.getNode(TRANS_PARAM_PREFIX + idx); String key = getString(paramNode, PARAM_KEY); String def = getString(paramNode, PARAM_DEFAULT); String desc = getString(paramNode, PARAM_DESC); transMeta.addParameterDefinition(key, def, desc); } transMeta.activateParameters(); }