@Test
  public void testErrorRedirect() throws Exception {
    JsonInputField field = new JsonInputField("value");
    field.setPath("$.value");
    field.setType(ValueMetaInterface.TYPE_STRING);

    String input1 = "{{";
    String input2 = "{ \"value\": \"ok\" }";

    JsonInputMeta meta = createSimpleMeta("json", field);
    meta.setRemoveSourceField(true);
    when(helper.stepMeta.isDoingErrorHandling()).thenReturn(true);
    JsonInput jsonInput =
        createJsonInput("json", meta, new Object[] {input1}, new Object[] {input2});
    StepErrorMeta errMeta = new StepErrorMeta(jsonInput, helper.stepMeta);
    errMeta.setEnabled(true);
    errMeta.setErrorFieldsValuename("err field");
    when(helper.stepMeta.getStepErrorMeta()).thenReturn(errMeta);
    final List<Object[]> errorLines = new ArrayList<>();
    jsonInput.addRowListener(
        new RowComparatorListener(new Object[] {"ok"}) {
          @Override
          public void errorRowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
              throws KettleStepException {
            errorLines.add(row);
          }
        });
    processRows(jsonInput, 3);
    Assert.assertEquals("fwd error", 1, errorLines.size());
    Assert.assertEquals("input in err line", input1, errorLines.get(0)[0]);
    Assert.assertEquals("rows written", 1, jsonInput.getLinesWritten());
  }
  public StepErrorMeta loadStepErrorMeta(
      VariableSpace variables, StepMeta stepMeta, List<StepMeta> steps) throws KettleException {
    StepErrorMeta meta = new StepErrorMeta(variables, stepMeta);

    meta.setTargetStep(
        StepMeta.findStep(
            steps,
            repository.getStepAttributeString(
                stepMeta.getObjectId(), "step_error_handling_target_step")));
    meta.setEnabled(
        repository.getStepAttributeBoolean(
            stepMeta.getObjectId(), "step_error_handling_is_enabled"));
    meta.setNrErrorsValuename(
        repository.getStepAttributeString(
            stepMeta.getObjectId(), "step_error_handling_nr_valuename"));
    meta.setErrorDescriptionsValuename(
        repository.getStepAttributeString(
            stepMeta.getObjectId(), "step_error_handling_descriptions_valuename"));
    meta.setErrorFieldsValuename(
        repository.getStepAttributeString(
            stepMeta.getObjectId(), "step_error_handling_fields_valuename"));
    meta.setErrorCodesValuename(
        repository.getStepAttributeString(
            stepMeta.getObjectId(), "step_error_handling_codes_valuename"));
    meta.setMaxErrors(
        repository.getStepAttributeString(
            stepMeta.getObjectId(), "step_error_handling_max_errors"));
    meta.setMaxPercentErrors(
        repository.getStepAttributeString(
            stepMeta.getObjectId(), "step_error_handling_max_pct_errors"));
    meta.setMinPercentRows(
        repository.getStepAttributeString(
            stepMeta.getObjectId(), "step_error_handling_min_pct_rows"));

    return meta;
  }
 public void saveStepErrorMeta(StepErrorMeta meta, ObjectId id_transformation, ObjectId id_step)
     throws KettleException {
   repository.saveStepAttribute(
       id_transformation,
       id_step,
       "step_error_handling_source_step",
       meta.getSourceStep() != null ? meta.getSourceStep().getName() : "");
   repository.saveStepAttribute(
       id_transformation,
       id_step,
       "step_error_handling_target_step",
       meta.getTargetStep() != null ? meta.getTargetStep().getName() : "");
   repository.saveStepAttribute(
       id_transformation, id_step, "step_error_handling_is_enabled", meta.isEnabled());
   repository.saveStepAttribute(
       id_transformation,
       id_step,
       "step_error_handling_nr_valuename",
       meta.getNrErrorsValuename());
   repository.saveStepAttribute(
       id_transformation,
       id_step,
       "step_error_handling_descriptions_valuename",
       meta.getErrorDescriptionsValuename());
   repository.saveStepAttribute(
       id_transformation,
       id_step,
       "step_error_handling_fields_valuename",
       meta.getErrorFieldsValuename());
   repository.saveStepAttribute(
       id_transformation,
       id_step,
       "step_error_handling_codes_valuename",
       meta.getErrorCodesValuename());
   repository.saveStepAttribute(
       id_transformation, id_step, "step_error_handling_max_errors", meta.getMaxErrors());
   repository.saveStepAttribute(
       id_transformation,
       id_step,
       "step_error_handling_max_pct_errors",
       meta.getMaxPercentErrors());
   repository.saveStepAttribute(
       id_transformation, id_step, "step_error_handling_min_pct_rows", meta.getMinPercentRows());
 }
예제 #4
0
  public DataNode elementToDataNode(final RepositoryElementInterface element)
      throws KettleException {
    TransMeta transMeta = (TransMeta) element;

    DataNode rootNode = new DataNode(NODE_TRANS);

    DataNode stepsNode = rootNode.addNode(NODE_STEPS);

    if (transMeta.getPrivateTransformationDatabases() != null) {
      // save all private transformations database name http://jira.pentaho.com/browse/PPP-3405
      DataNode privateDatabaseNode = rootNode.addNode(NODE_TRANS_PRIVATE_DATABASES);
      for (String privateDatabase : transMeta.getPrivateTransformationDatabases()) {
        privateDatabaseNode.addNode(privateDatabase);
      }
    }
    // Also save all the steps in the transformation!
    //
    int stepNr = 0;
    for (StepMeta step : transMeta.getSteps()) {
      stepNr++;
      DataNode stepNode =
          stepsNode.addNode(
              sanitizeNodeName(step.getName()) + "_" + stepNr + EXT_STEP); // $NON-NLS-1$

      // Store the main data
      //
      stepNode.setProperty(PROP_NAME, step.getName());
      stepNode.setProperty(PROP_DESCRIPTION, step.getDescription());
      stepNode.setProperty(PROP_STEP_TYPE, step.getStepID());
      stepNode.setProperty(PROP_STEP_DISTRIBUTE, step.isDistributes());
      stepNode.setProperty(
          PROP_STEP_ROW_DISTRIBUTION,
          step.getRowDistribution() == null ? null : step.getRowDistribution().getCode());
      stepNode.setProperty(PROP_STEP_COPIES, step.getCopies());
      stepNode.setProperty(PROP_STEP_COPIES_STRING, step.getCopiesString());
      stepNode.setProperty(PROP_STEP_GUI_LOCATION_X, step.getLocation().x);
      stepNode.setProperty(PROP_STEP_GUI_LOCATION_Y, step.getLocation().y);
      stepNode.setProperty(PROP_STEP_GUI_DRAW, step.isDrawn());

      // Also save the step group attributes map
      //
      AttributesMapUtil.saveAttributesMap(stepNode, step);

      // Save the step metadata using the repository save method, NOT XML
      // That is because we want to keep the links to databases, conditions, etc by ID, not name.
      //
      StepMetaInterface stepMetaInterface = step.getStepMetaInterface();
      DataNode stepCustomNode = new DataNode(NODE_STEP_CUSTOM);
      Repository proxy = new RepositoryProxy(stepCustomNode);
      compatibleSaveRep(stepMetaInterface, proxy, null, null);
      stepMetaInterface.saveRep(proxy, proxy.getMetaStore(), null, null);
      stepNode.addNode(stepCustomNode);

      // Save the partitioning information by reference as well...
      //
      StepPartitioningMeta partitioningMeta = step.getStepPartitioningMeta();
      if (partitioningMeta != null
          && partitioningMeta.getPartitionSchema() != null
          && partitioningMeta.isPartitioned()) {
        DataNodeRef ref =
            new DataNodeRef(partitioningMeta.getPartitionSchema().getObjectId().getId());
        stepNode.setProperty(PROP_PARTITIONING_SCHEMA, ref);
        stepNode.setProperty(
            PROP_PARTITIONING_METHOD, partitioningMeta.getMethodCode()); // method of partitioning
        if (partitioningMeta.getPartitioner() != null) {
          DataNode partitionerCustomNode = new DataNode(NODE_PARTITIONER_CUSTOM);
          proxy = new RepositoryProxy(partitionerCustomNode);
          partitioningMeta.getPartitioner().saveRep(proxy, null, null);
          stepNode.addNode(partitionerCustomNode);
        }
      }

      // Save the clustering information as well...
      //
      stepNode.setProperty(
          PROP_CLUSTER_SCHEMA,
          step.getClusterSchema() == null
              ? ""
              : step.getClusterSchema() //$NON-NLS-1$
                  .getName());

      // Save the error hop metadata
      //
      StepErrorMeta stepErrorMeta = step.getStepErrorMeta();
      if (stepErrorMeta != null) {
        stepNode.setProperty(
            PROP_STEP_ERROR_HANDLING_SOURCE_STEP,
            stepErrorMeta.getSourceStep() != null
                ? stepErrorMeta.getSourceStep().getName()
                : ""); //$NON-NLS-1$
        stepNode.setProperty(
            PROP_STEP_ERROR_HANDLING_TARGET_STEP,
            stepErrorMeta.getTargetStep() != null
                ? stepErrorMeta.getTargetStep().getName()
                : ""); //$NON-NLS-1$
        stepNode.setProperty(PROP_STEP_ERROR_HANDLING_IS_ENABLED, stepErrorMeta.isEnabled());
        stepNode.setProperty(
            PROP_STEP_ERROR_HANDLING_NR_VALUENAME, stepErrorMeta.getNrErrorsValuename());
        stepNode.setProperty(
            PROP_STEP_ERROR_HANDLING_DESCRIPTIONS_VALUENAME,
            stepErrorMeta.getErrorDescriptionsValuename());
        stepNode.setProperty(
            PROP_STEP_ERROR_HANDLING_FIELDS_VALUENAME, stepErrorMeta.getErrorFieldsValuename());
        stepNode.setProperty(
            PROP_STEP_ERROR_HANDLING_CODES_VALUENAME, stepErrorMeta.getErrorCodesValuename());
        stepNode.setProperty(PROP_STEP_ERROR_HANDLING_MAX_ERRORS, stepErrorMeta.getMaxErrors());
        stepNode.setProperty(
            PROP_STEP_ERROR_HANDLING_MAX_PCT_ERRORS, stepErrorMeta.getMaxPercentErrors());
        stepNode.setProperty(
            PROP_STEP_ERROR_HANDLING_MIN_PCT_ROWS, stepErrorMeta.getMinPercentRows());
      }
    }

    // Save the notes
    //
    DataNode notesNode = rootNode.addNode(NODE_NOTES);
    notesNode.setProperty(PROP_NR_NOTES, transMeta.nrNotes());
    for (int i = 0; i < transMeta.nrNotes(); i++) {
      NotePadMeta note = transMeta.getNote(i);
      DataNode noteNode = notesNode.addNode(NOTE_PREFIX + i);

      noteNode.setProperty(PROP_XML, note.getXML());
    }

    // Finally, save the hops
    //
    DataNode hopsNode = rootNode.addNode(NODE_HOPS);
    hopsNode.setProperty(PROP_NR_HOPS, transMeta.nrTransHops());
    for (int i = 0; i < transMeta.nrTransHops(); i++) {
      TransHopMeta hop = transMeta.getTransHop(i);
      DataNode hopNode = hopsNode.addNode(TRANS_HOP_PREFIX + i);
      hopNode.setProperty(TRANS_HOP_FROM, hop.getFromStep().getName());
      hopNode.setProperty(TRANS_HOP_TO, hop.getToStep().getName());
      hopNode.setProperty(TRANS_HOP_ENABLED, hop.isEnabled());
    }

    // Parameters
    //
    String[] paramKeys = transMeta.listParameters();
    DataNode paramsNode = rootNode.addNode(NODE_PARAMETERS);
    paramsNode.setProperty(PROP_NR_PARAMETERS, paramKeys == null ? 0 : paramKeys.length);

    for (int idx = 0; idx < paramKeys.length; idx++) {
      DataNode paramNode = paramsNode.addNode(TRANS_PARAM_PREFIX + idx);
      String key = paramKeys[idx];
      String description = transMeta.getParameterDescription(paramKeys[idx]);
      String defaultValue = transMeta.getParameterDefault(paramKeys[idx]);

      paramNode.setProperty(PARAM_KEY, key != null ? key : ""); // $NON-NLS-1$
      paramNode.setProperty(PARAM_DEFAULT, defaultValue != null ? defaultValue : ""); // $NON-NLS-1$
      paramNode.setProperty(PARAM_DESC, description != null ? description : ""); // $NON-NLS-1$
    }

    // Let's not forget to save the details of the transformation itself.
    // This includes logging information, parameters, etc.
    //
    saveTransformationDetails(rootNode, transMeta);

    return rootNode;
  }
예제 #5
0
  public void dataNodeToElement(final DataNode rootNode, final RepositoryElementInterface element)
      throws KettleException {
    TransMeta transMeta = (TransMeta) element;

    List<String> privateTransformationDatabases = null;
    // read the private databases
    DataNode privateDatabases = rootNode.getNode(NODE_TRANS_PRIVATE_DATABASES);
    // if we have node than we use new format we could remove unexpected node
    if (privateDatabases != null) {
      privateTransformationDatabases = new ArrayList<String>();
      for (DataNode privateDatabase : privateDatabases.getNodes()) {
        privateTransformationDatabases.add(privateDatabase.getName());
      }
    }
    transMeta.setPrivateTransformationDatabases(privateTransformationDatabases);

    // read the steps...
    //
    DataNode stepsNode = rootNode.getNode(NODE_STEPS);
    for (DataNode stepNode : stepsNode.getNodes()) {

      StepMeta stepMeta = new StepMeta(new StringObjectId(stepNode.getId().toString()));
      stepMeta.setParentTransMeta(transMeta); // for tracing, retain hierarchy

      // Read the basics
      //
      stepMeta.setName(getString(stepNode, PROP_NAME));
      if (stepNode.hasProperty(PROP_DESCRIPTION)) {
        stepMeta.setDescription(getString(stepNode, PROP_DESCRIPTION));
      }
      stepMeta.setDistributes(stepNode.getProperty(PROP_STEP_DISTRIBUTE).getBoolean());
      DataProperty rowDistributionProperty = stepNode.getProperty(PROP_STEP_ROW_DISTRIBUTION);
      String rowDistributionCode =
          rowDistributionProperty == null ? null : rowDistributionProperty.getString();
      RowDistributionInterface rowDistribution =
          PluginRegistry.getInstance()
              .loadClass(
                  RowDistributionPluginType.class,
                  rowDistributionCode,
                  RowDistributionInterface.class);
      stepMeta.setRowDistribution(rowDistribution);
      stepMeta.setDraw(stepNode.getProperty(PROP_STEP_GUI_DRAW).getBoolean());
      int copies = (int) stepNode.getProperty(PROP_STEP_COPIES).getLong();
      String copiesString =
          stepNode.getProperty(PROP_STEP_COPIES_STRING) != null
              ? stepNode.getProperty(PROP_STEP_COPIES_STRING).getString()
              : StringUtils.EMPTY;
      if (!Const.isEmpty(copiesString)) {
        stepMeta.setCopiesString(copiesString);
      } else {
        stepMeta.setCopies(copies); // for backward compatibility
      }

      int x = (int) stepNode.getProperty(PROP_STEP_GUI_LOCATION_X).getLong();
      int y = (int) stepNode.getProperty(PROP_STEP_GUI_LOCATION_Y).getLong();
      stepMeta.setLocation(x, y);

      // Load the group attributes map
      //
      AttributesMapUtil.loadAttributesMap(stepNode, stepMeta);

      String stepType = getString(stepNode, PROP_STEP_TYPE);

      // Create a new StepMetaInterface object...
      //
      PluginRegistry registry = PluginRegistry.getInstance();
      PluginInterface stepPlugin = registry.findPluginWithId(StepPluginType.class, stepType);

      StepMetaInterface stepMetaInterface = null;
      if (stepPlugin != null) {
        stepMetaInterface = (StepMetaInterface) registry.loadClass(stepPlugin);
        stepType =
            stepPlugin.getIds()[0]; // revert to the default in case we loaded an alternate version
      } else {
        stepMeta.setStepMetaInterface(
            (StepMetaInterface) new MissingTrans(stepMeta.getName(), stepType));
        transMeta.addMissingTrans((MissingTrans) stepMeta.getStepMetaInterface());
      }

      stepMeta.setStepID(stepType);

      // Read the metadata from the repository too...
      //
      RepositoryProxy proxy = new RepositoryProxy(stepNode.getNode(NODE_STEP_CUSTOM));
      if (!stepMeta.isMissing()) {
        readRepCompatibleStepMeta(stepMetaInterface, proxy, null, transMeta.getDatabases());
        stepMetaInterface.readRep(proxy, transMeta.getMetaStore(), null, transMeta.getDatabases());
        stepMeta.setStepMetaInterface(stepMetaInterface);
      }

      // Get the partitioning as well...
      StepPartitioningMeta stepPartitioningMeta = new StepPartitioningMeta();
      if (stepNode.hasProperty(PROP_PARTITIONING_SCHEMA)) {
        String partSchemaId =
            stepNode.getProperty(PROP_PARTITIONING_SCHEMA).getRef().getId().toString();
        String schemaName =
            repo.loadPartitionSchema(new StringObjectId(partSchemaId), null).getName();

        stepPartitioningMeta.setPartitionSchemaName(schemaName);
        String methodCode = getString(stepNode, PROP_PARTITIONING_METHOD);
        stepPartitioningMeta.setMethod(StepPartitioningMeta.getMethod(methodCode));
        if (stepPartitioningMeta.getPartitioner() != null) {
          proxy = new RepositoryProxy(stepNode.getNode(NODE_PARTITIONER_CUSTOM));
          stepPartitioningMeta.getPartitioner().loadRep(proxy, null);
        }
        stepPartitioningMeta.hasChanged(true);
      }
      stepMeta.setStepPartitioningMeta(stepPartitioningMeta);

      stepMeta
          .getStepPartitioningMeta()
          .setPartitionSchemaAfterLoading(transMeta.getPartitionSchemas());
      // Get the cluster schema name
      String clusterSchemaName = getString(stepNode, PROP_CLUSTER_SCHEMA);
      stepMeta.setClusterSchemaName(clusterSchemaName);
      if (clusterSchemaName != null && transMeta.getClusterSchemas() != null) {
        // Get the cluster schema from the given name
        for (ClusterSchema clusterSchema : transMeta.getClusterSchemas()) {
          if (clusterSchema.getName().equals(clusterSchemaName)) {
            stepMeta.setClusterSchema(clusterSchema);
            break;
          }
        }
      }

      transMeta.addStep(stepMeta);
    }

    for (DataNode stepNode : stepsNode.getNodes()) {

      ObjectId stepObjectId = new StringObjectId(stepNode.getId().toString());
      StepMeta stepMeta = StepMeta.findStep(transMeta.getSteps(), stepObjectId);

      // Also load the step error handling metadata
      //
      if (stepNode.hasProperty(PROP_STEP_ERROR_HANDLING_SOURCE_STEP)) {
        StepErrorMeta meta = new StepErrorMeta(transMeta, stepMeta);
        meta.setTargetStep(
            StepMeta.findStep(
                transMeta.getSteps(),
                stepNode.getProperty(PROP_STEP_ERROR_HANDLING_TARGET_STEP).getString()));
        meta.setEnabled(stepNode.getProperty(PROP_STEP_ERROR_HANDLING_IS_ENABLED).getBoolean());
        meta.setNrErrorsValuename(getString(stepNode, PROP_STEP_ERROR_HANDLING_NR_VALUENAME));
        meta.setErrorDescriptionsValuename(
            getString(stepNode, PROP_STEP_ERROR_HANDLING_DESCRIPTIONS_VALUENAME));
        meta.setErrorFieldsValuename(
            getString(stepNode, PROP_STEP_ERROR_HANDLING_FIELDS_VALUENAME));
        meta.setErrorCodesValuename(getString(stepNode, PROP_STEP_ERROR_HANDLING_CODES_VALUENAME));
        meta.setMaxErrors(getString(stepNode, PROP_STEP_ERROR_HANDLING_MAX_ERRORS));
        meta.setMaxPercentErrors(getString(stepNode, PROP_STEP_ERROR_HANDLING_MAX_PCT_ERRORS));
        meta.setMinPercentRows(getString(stepNode, PROP_STEP_ERROR_HANDLING_MIN_PCT_ROWS));
        meta.getSourceStep().setStepErrorMeta(meta); // a bit of a trick, I know.
      }
    }

    // Have all StreamValueLookups, etc. reference the correct source steps...
    //
    for (int i = 0; i < transMeta.nrSteps(); i++) {
      StepMeta stepMeta = transMeta.getStep(i);
      StepMetaInterface sii = stepMeta.getStepMetaInterface();
      if (sii != null) {
        sii.searchInfoAndTargetSteps(transMeta.getSteps());
      }
    }

    // Read the notes...
    //
    DataNode notesNode = rootNode.getNode(NODE_NOTES);
    int nrNotes = (int) notesNode.getProperty(PROP_NR_NOTES).getLong();
    for (DataNode noteNode : notesNode.getNodes()) {
      String xml = getString(noteNode, PROP_XML);
      transMeta.addNote(
          new NotePadMeta(
              XMLHandler.getSubNode(XMLHandler.loadXMLString(xml), NotePadMeta.XML_TAG)));
    }
    if (transMeta.nrNotes() != nrNotes) {
      throw new KettleException(
          "The number of notes read ["
              + transMeta.nrNotes()
              + "] was not the number we expected ["
              + nrNotes
              + "]");
    }

    // Read the hops...
    //
    DataNode hopsNode = rootNode.getNode(NODE_HOPS);
    int nrHops = (int) hopsNode.getProperty(PROP_NR_HOPS).getLong();
    for (DataNode hopNode : hopsNode.getNodes()) {
      String stepFromName = getString(hopNode, TRANS_HOP_FROM);
      String stepToName = getString(hopNode, TRANS_HOP_TO);
      boolean enabled = true;
      if (hopNode.hasProperty(TRANS_HOP_ENABLED)) {
        enabled = hopNode.getProperty(TRANS_HOP_ENABLED).getBoolean();
      }

      StepMeta stepFrom = StepMeta.findStep(transMeta.getSteps(), stepFromName);
      StepMeta stepTo = StepMeta.findStep(transMeta.getSteps(), stepToName);

      // Make sure to only accept valid hops PDI-5519
      //
      if (stepFrom != null && stepTo != null) {
        transMeta.addTransHop(new TransHopMeta(stepFrom, stepTo, enabled));
      }
    }
    if (transMeta.nrTransHops() != nrHops) {
      throw new KettleException(
          "The number of hops read ["
              + transMeta.nrTransHops()
              + "] was not the number we expected ["
              + nrHops
              + "]");
    }

    // Load the details at the end, to make sure we reference the databases correctly, etc.
    //
    loadTransformationDetails(rootNode, transMeta);

    transMeta.eraseParameters();

    DataNode paramsNode = rootNode.getNode(NODE_PARAMETERS);

    int count = (int) paramsNode.getProperty(PROP_NR_PARAMETERS).getLong();
    for (int idx = 0; idx < count; idx++) {
      DataNode paramNode = paramsNode.getNode(TRANS_PARAM_PREFIX + idx);
      String key = getString(paramNode, PARAM_KEY);
      String def = getString(paramNode, PARAM_DEFAULT);
      String desc = getString(paramNode, PARAM_DESC);
      transMeta.addParameterDefinition(key, def, desc);
    }

    transMeta.activateParameters();
  }