public void getFields(
      RowMetaInterface r,
      String name,
      RowMetaInterface info[],
      StepMeta nextStep,
      VariableSpace space)
      throws KettleStepException {
    // It's best that this method doesn't change anything by itself.
    // Eventually it's the Mapping step that's going to tell this step how to behave meta-data wise.
    // It is the mapping step that tells the mapping output step what fields to rename.
    //
    if (inputValueRenames != null) {
      for (MappingValueRename valueRename : inputValueRenames) {
        ValueMetaInterface valueMeta = r.searchValueMeta(valueRename.getTargetValueName());
        if (valueMeta != null) {
          valueMeta.setName(valueRename.getSourceValueName());
        }
      }
    }

    // This is the optionally entered stuff in the output tab of the mapping dialog.
    //
    if (outputValueRenames != null) {
      for (MappingValueRename valueRename : outputValueRenames) {
        ValueMetaInterface valueMeta = r.searchValueMeta(valueRename.getSourceValueName());
        if (valueMeta != null) {
          valueMeta.setName(valueRename.getTargetValueName());
        }
      }
    }
  }
  public SQLStatement getSQLStatements(
      TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev) throws KettleStepException {
    SQLStatement retval =
        new SQLStatement(stepMeta.getName(), databaseMeta, null); // default: nothing to do!

    if (databaseMeta != null) {
      if (prev != null && prev.size() > 0) {
        // Copy the row
        RowMetaInterface tableFields = new RowMeta();

        // Now change the field names
        for (int i = 0; i < fieldTable.length; i++) {
          ValueMetaInterface v = prev.searchValueMeta(fieldStream[i]);
          if (v != null) {
            ValueMetaInterface tableField = v.clone();
            tableField.setName(fieldTable[i]);
            tableFields.addValueMeta(tableField);
          } else {
            throw new KettleStepException(
                "Unable to find field [" + fieldStream[i] + "] in the input rows");
          }
        }

        if (!Const.isEmpty(tableName)) {
          Database db = new Database(loggingObject, databaseMeta);
          db.shareVariablesWith(transMeta);
          try {
            db.connect();

            String schemaTable =
                databaseMeta.getQuotedSchemaTableCombination(
                    transMeta.environmentSubstitute(schemaName),
                    transMeta.environmentSubstitute(tableName));
            String sql = db.getDDL(schemaTable, tableFields, null, false, null, true);

            if (sql.length() == 0) retval.setSQL(null);
            else retval.setSQL(sql);
          } catch (KettleException e) {
            retval.setError(
                BaseMessages.getString(PKG, "GPBulkLoaderMeta.GetSQL.ErrorOccurred")
                    + e.getMessage()); // $NON-NLS-1$
          }
        } else {
          retval.setError(
              BaseMessages.getString(
                  PKG, "GPBulkLoaderMeta.GetSQL.NoTableDefinedOnConnection")); // $NON-NLS-1$
        }
      } else {
        retval.setError(
            BaseMessages.getString(
                PKG, "GPBulkLoaderMeta.GetSQL.NotReceivingAnyFields")); // $NON-NLS-1$
      }
    } else {
      retval.setError(
          BaseMessages.getString(
              PKG, "GPBulkLoaderMeta.GetSQL.NoConnectionDefined")); // $NON-NLS-1$
    }

    return retval;
  }
 public void getFields(
     RowMetaInterface row,
     String name,
     RowMetaInterface[] info,
     StepMeta nextStep,
     VariableSpace space)
     throws KettleStepException {
   if (Const.isEmpty(info) || info[0] == null) // null or length 0 : no info from database
   {
     for (int i = 0; i < returnValueNewName.length; i++) {
       ValueMetaInterface v = new ValueMeta(returnValueNewName[i], returnValueDefaultType[i]);
       v.setOrigin(name);
       row.addValueMeta(v);
     }
   } else {
     for (int i = 0; i < returnValueNewName.length; i++) {
       ValueMetaInterface v = info[0].searchValueMeta(returnValueField[i]);
       if (v != null) {
         v.setName(returnValueNewName[i]);
         v.setOrigin(name);
         row.addValueMeta(v);
       }
     }
   }
 }
  private void sql() {
    try {
      IngresVectorwiseLoaderMeta info = new IngresVectorwiseLoaderMeta();
      getInfo(info);
      RowMetaInterface prev = transMeta.getPrevStepFields(stepname);
      StepMeta stepMeta = transMeta.findStep(stepname);

      // Only use the fields that were specified.
      //
      RowMetaInterface prevNew = new RowMeta();

      for (int i = 0; i < info.getFieldDatabase().length; i++) {
        ValueMetaInterface insValue = prev.searchValueMeta(info.getFieldStream()[i]);
        if (insValue != null) {
          ValueMetaInterface insertValue = insValue.clone();
          insertValue.setName(info.getFieldDatabase()[i]);
          prevNew.addValueMeta(insertValue);
        } else {
          throw new KettleStepException(
              BaseMessages.getString(
                  PKG,
                  "IngresVectorWiseLoaderDialog.FailedToFindField.Message",
                  info.getFieldStream()[i]));
        }
      }
      prev = prevNew;

      SQLStatement sql = info.getSQLStatements(transMeta, stepMeta, prev, repository, metaStore);
      if (!sql.hasError()) {
        if (sql.hasSQL()) {
          SQLEditor sqledit =
              new SQLEditor(
                  transMeta,
                  shell,
                  SWT.NONE,
                  info.getDatabaseMeta(),
                  transMeta.getDbCache(),
                  sql.getSQL());
          sqledit.open();
        } else {
          MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION);
          mb.setMessage(
              BaseMessages.getString(PKG, "IngresVectorWiseLoaderDialog.NoSQL.DialogMessage"));
          mb.setText(BaseMessages.getString(PKG, "IngresVectorWiseLoaderDialog.NoSQL.DialogTitle"));
          mb.open();
        }
      } else {
        MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR);
        mb.setMessage(sql.getError());
        mb.setText(BaseMessages.getString(PKG, "System.Dialog.Error.Title"));
        mb.open();
      }
    } catch (KettleException ke) {
      new ErrorDialog(
          shell,
          BaseMessages.getString(PKG, "IngresVectorWiseLoaderDialog.BuildSQLError.DialogTitle"),
          BaseMessages.getString(PKG, "IngresVectorWiseLoaderDialog.BuildSQLError.DialogMessage"),
          ke);
    }
  }
 public static ValueMetaInterface createValueMeta(String name, int type, int length, int precision)
     throws KettlePluginException {
   PluginInterface stringPlugin =
       pluginRegistry.getPlugin(ValueMetaPluginType.class, String.valueOf(type));
   if (stringPlugin == null) {
     throw new KettlePluginException("Unable to locate value meta plugin of type (id) " + type);
   }
   ValueMetaInterface valueMeta = pluginRegistry.loadClass(stringPlugin, ValueMetaInterface.class);
   valueMeta.setName(name);
   valueMeta.setLength(length, precision);
   return valueMeta;
 }
  @Test
  public void testGetFieldDefinitionInteger() {
    String integerName = "integerName";
    ValueMetaInterface valueMetaInterface = new ValueMetaInteger();
    valueMetaInterface.setName(integerName);
    valueMetaInterface.setPrecision(0);
    valueMetaInterface.setLength(9);
    assertGetFieldDefinition(valueMetaInterface, "INT");

    valueMetaInterface.setLength(18);
    assertGetFieldDefinition(valueMetaInterface, "BIGINT");

    valueMetaInterface.setLength(19);
    assertGetFieldDefinition(valueMetaInterface, "FLOAT");
  }
  @Test
  public void testGetFieldDefinitionBigNumber() {
    String bigNumberName = "bigNumberName";
    ValueMetaInterface valueMetaInterface = new ValueMetaBigNumber();
    valueMetaInterface.setName(bigNumberName);
    valueMetaInterface.setPrecision(0);
    valueMetaInterface.setLength(9);
    assertGetFieldDefinition(valueMetaInterface, "INT");

    valueMetaInterface.setLength(18);
    assertGetFieldDefinition(valueMetaInterface, "BIGINT");

    valueMetaInterface.setLength(19);
    assertGetFieldDefinition(valueMetaInterface, "FLOAT");

    valueMetaInterface.setPrecision(10);
    valueMetaInterface.setLength(16);
    assertGetFieldDefinition(valueMetaInterface, "FLOAT");

    valueMetaInterface.setLength(15);
    assertGetFieldDefinition(valueMetaInterface, "DOUBLE");
  }
Пример #8
0
  public void getFields(
      RowMetaInterface row,
      String name,
      RowMetaInterface[] info,
      StepMeta nextStep,
      VariableSpace space)
      throws KettleStepException {

    // Remember the types of the row.
    int fieldnrs[] = new int[fieldName.length];
    ValueMetaInterface[] values = new ValueMetaInterface[fieldName.length];

    for (int i = 0; i < fieldName.length; i++) {
      fieldnrs[i] = row.indexOfValue(fieldName[i]);
      ValueMetaInterface v = row.getValueMeta(fieldnrs[i]);
      values[i] = v.clone(); // copy value : default settings!
      switch (aggregateType[i]) {
        case TYPE_AGGREGATE_AVERAGE:
        case TYPE_AGGREGATE_COUNT:
        case TYPE_AGGREGATE_SUM:
          values[i].setType(Value.VALUE_TYPE_NUMBER);
          values[i].setLength(-1, -1);
          break;
      }
    }

    // Only the aggregate is returned!
    row.clear();

    for (int i = 0; i < fieldName.length; i++) {
      ValueMetaInterface v = values[i];
      v.setName(fieldNewName[i]);
      v.setOrigin(name);
      row.addValueMeta(v);
    }
  }
Пример #9
0
  public void getFields(
      RowMetaInterface row,
      String origin,
      RowMetaInterface info[],
      StepMeta nextStep,
      VariableSpace space)
      throws KettleStepException {
    // First load some interesting data...

    // Then see which fields get added to the row.
    //
    TransMeta mappingTransMeta = null;
    try {
      mappingTransMeta = loadMappingMeta(this, repository, space);
    } catch (KettleException e) {
      throw new KettleStepException(
          BaseMessages.getString(PKG, "MappingMeta.Exception.UnableToLoadMappingTransformation"),
          e);
    }

    // The field structure may depend on the input parameters as well (think of parameter
    // replacements in MDX queries for instance)
    if (mappingParameters != null) {

      // See if we need to pass all variables from the parent or not...
      //
      if (mappingParameters.isInheritingAllVariables()) {
        mappingTransMeta.copyVariablesFrom(space);
      }

      // Just set the variables in the transformation statically.
      // This just means: set a number of variables or parameter values:
      //
      List<String> subParams = Arrays.asList(mappingTransMeta.listParameters());

      for (int i = 0; i < mappingParameters.getVariable().length; i++) {
        String name = mappingParameters.getVariable()[i];
        String value = space.environmentSubstitute(mappingParameters.getInputField()[i]);
        if (!Const.isEmpty(name) && !Const.isEmpty(value)) {
          if (subParams.contains(name)) {
            try {
              mappingTransMeta.setParameterValue(name, value);
            } catch (UnknownParamException e) {
              // this is explicitly checked for up front
            }
          }
          mappingTransMeta.setVariable(name, value);
        }
      }
    }

    // Keep track of all the fields that need renaming...
    //
    List<MappingValueRename> inputRenameList = new ArrayList<MappingValueRename>();

    /*
     * Before we ask the mapping outputs anything, we should teach the mapping
     * input steps in the sub-transformation about the data coming in...
     */
    for (MappingIODefinition definition : inputMappings) {

      RowMetaInterface inputRowMeta;

      if (definition.isMainDataPath() || Const.isEmpty(definition.getInputStepname())) {
        // The row metadata, what we pass to the mapping input step
        // definition.getOutputStep(), is "row"
        // However, we do need to re-map some fields...
        //
        inputRowMeta = row.clone();
        if (!inputRowMeta.isEmpty()) {
          for (MappingValueRename valueRename : definition.getValueRenames()) {
            ValueMetaInterface valueMeta =
                inputRowMeta.searchValueMeta(valueRename.getSourceValueName());
            if (valueMeta == null) {
              throw new KettleStepException(
                  BaseMessages.getString(
                      PKG,
                      "MappingMeta.Exception.UnableToFindField",
                      valueRename.getSourceValueName()));
            }
            valueMeta.setName(valueRename.getTargetValueName());
          }
        }
      } else {
        // The row metadata that goes to the info mapping input comes from the
        // specified step
        // In fact, it's one of the info steps that is going to contain this
        // information...
        //
        String[] infoSteps = getInfoSteps();
        int infoStepIndex = Const.indexOfString(definition.getInputStepname(), infoSteps);
        if (infoStepIndex < 0) {
          throw new KettleStepException(
              BaseMessages.getString(
                  PKG,
                  "MappingMeta.Exception.UnableToFindMetadataInfo",
                  definition.getInputStepname()));
        }
        if (info[infoStepIndex] != null) {
          inputRowMeta = info[infoStepIndex].clone();
        } else {
          inputRowMeta = null;
        }
      }

      // What is this mapping input step?
      //
      StepMeta mappingInputStep =
          mappingTransMeta.findMappingInputStep(definition.getOutputStepname());

      // We're certain it's a MappingInput step...
      //
      MappingInputMeta mappingInputMeta =
          (MappingInputMeta) mappingInputStep.getStepMetaInterface();

      // Inform the mapping input step about what it's going to receive...
      //
      mappingInputMeta.setInputRowMeta(inputRowMeta);

      // What values are we changing names for?
      //
      mappingInputMeta.setValueRenames(definition.getValueRenames());

      // Keep a list of the input rename values that need to be changed back at
      // the output
      //
      if (definition.isRenamingOnOutput())
        Mapping.addInputRenames(inputRenameList, definition.getValueRenames());
    }

    // All the mapping steps now know what they will be receiving.
    // That also means that the sub-transformation / mapping has everything it
    // needs.
    // So that means that the MappingOutput steps know exactly what the output
    // is going to be.
    // That could basically be anything.
    // It also could have absolutely no resemblance to what came in on the
    // input.
    // The relative old approach is therefore no longer suited.
    //
    // OK, but what we *can* do is have the MappingOutput step rename the
    // appropriate fields.
    // The mapping step will tell this step how it's done.
    //
    // Let's look for the mapping output step that is relevant for this actual
    // call...
    //
    MappingIODefinition mappingOutputDefinition = null;
    if (nextStep == null) {
      // This is the main step we read from...
      // Look up the main step to write to.
      // This is the output mapping definition with "main path" enabled.
      //
      for (MappingIODefinition definition : outputMappings) {
        if (definition.isMainDataPath() || Const.isEmpty(definition.getOutputStepname())) {
          // This is the definition to use...
          //
          mappingOutputDefinition = definition;
        }
      }
    } else {
      // Is there an output mapping definition for this step?
      // If so, we can look up the Mapping output step to see what has changed.
      //

      for (MappingIODefinition definition : outputMappings) {
        if (nextStep.getName().equals(definition.getOutputStepname())
            || definition.isMainDataPath()
            || Const.isEmpty(definition.getOutputStepname())) {
          mappingOutputDefinition = definition;
        }
      }
    }

    if (mappingOutputDefinition == null) {
      throw new KettleStepException(
          BaseMessages.getString(PKG, "MappingMeta.Exception.UnableToFindMappingDefinition"));
    }

    // OK, now find the mapping output step in the mapping...
    // This method in TransMeta takes into account a number of things, such as
    // the step not specified, etc.
    // The method never returns null but throws an exception.
    //
    StepMeta mappingOutputStep =
        mappingTransMeta.findMappingOutputStep(mappingOutputDefinition.getInputStepname());

    // We know it's a mapping output step...
    MappingOutputMeta mappingOutputMeta =
        (MappingOutputMeta) mappingOutputStep.getStepMetaInterface();

    // Change a few columns.
    mappingOutputMeta.setOutputValueRenames(mappingOutputDefinition.getValueRenames());

    // Perhaps we need to change a few input columns back to the original?
    //
    mappingOutputMeta.setInputValueRenames(inputRenameList);

    // Now we know wat's going to come out of there...
    // This is going to be the full row, including all the remapping, etc.
    //
    RowMetaInterface mappingOutputRowMeta = mappingTransMeta.getStepFields(mappingOutputStep);

    row.clear();
    row.addRowMeta(mappingOutputRowMeta);
  }
Пример #10
0
  public SQLStatement getSQLStatements(
      TransMeta transMeta,
      StepMeta stepMeta,
      RowMetaInterface prev,
      Repository repository,
      IMetaStore metaStore)
      throws KettleStepException {
    SQLStatement retval =
        new SQLStatement(stepMeta.getName(), databaseMeta, null); // default: nothing to do!

    if (databaseMeta != null) {
      if (prev != null && prev.size() > 0) {
        // Copy the row
        RowMetaInterface tableFields = new RowMeta();

        // Now change the field names
        // the key fields
        if (keyLookup != null) {
          for (int i = 0; i < keyLookup.length; i++) {
            ValueMetaInterface v = prev.searchValueMeta(keyStream[i]);
            if (v != null) {
              ValueMetaInterface tableField = v.clone();
              tableField.setName(keyLookup[i]);
              tableFields.addValueMeta(tableField);
            } else {
              throw new KettleStepException(
                  "Unable to find field [" + keyStream[i] + "] in the input rows");
            }
          }
        }
        // the lookup fields
        for (int i = 0; i < updateLookup.length; i++) {
          ValueMetaInterface v = prev.searchValueMeta(updateStream[i]);
          if (v != null) {
            ValueMetaInterface vk = tableFields.searchValueMeta(updateStream[i]);
            if (vk == null) { // do not add again when already added as key fields
              ValueMetaInterface tableField = v.clone();
              tableField.setName(updateLookup[i]);
              tableFields.addValueMeta(tableField);
            }
          } else {
            throw new KettleStepException(
                "Unable to find field [" + updateStream[i] + "] in the input rows");
          }
        }

        if (!Const.isEmpty(tableName)) {
          Database db = new Database(loggingObject, databaseMeta);
          db.shareVariablesWith(transMeta);
          try {
            db.connect();

            String schemaTable =
                databaseMeta.getQuotedSchemaTableCombination(schemaName, tableName);
            String cr_table = db.getDDL(schemaTable, tableFields, null, false, null, true);

            String cr_index = "";
            String[] idx_fields = null;

            if (keyLookup != null && keyLookup.length > 0) {
              idx_fields = new String[keyLookup.length];
              for (int i = 0; i < keyLookup.length; i++) {
                idx_fields[i] = keyLookup[i];
              }
            } else {
              retval.setError(
                  BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.MissingKeyFields"));
            }

            // Key lookup dimensions...
            if (idx_fields != null
                && idx_fields.length > 0
                && !db.checkIndexExists(schemaName, tableName, idx_fields)) {
              String indexname = "idx_" + tableName + "_lookup";
              cr_index =
                  db.getCreateIndexStatement(
                      schemaTable, indexname, idx_fields, false, false, false, true);
            }

            String sql = cr_table + cr_index;
            if (sql.length() == 0) {
              retval.setSQL(null);
            } else {
              retval.setSQL(sql);
            }
          } catch (KettleException e) {
            retval.setError(
                BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.ErrorOccurred")
                    + e.getMessage());
          }
        } else {
          retval.setError(
              BaseMessages.getString(
                  PKG, "InsertUpdateMeta.ReturnValue.NoTableDefinedOnConnection"));
        }
      } else {
        retval.setError(
            BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.NotReceivingAnyFields"));
      }
    } else {
      retval.setError(
          BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.NoConnectionDefined"));
    }

    return retval;
  }
  public void getFields(
      RowMetaInterface row,
      String origin,
      RowMetaInterface[] info,
      StepMeta nextStep,
      VariableSpace space)
      throws KettleStepException {
    // Row should normally be empty when we get here.
    // That is because there is no previous step to this mapping input step from the viewpoint of
    // this single sub-transformation.
    // From the viewpoint of the transformation that executes the mapping, it's important to know
    // what comes out at the exit points.
    // For that reason we need to re-order etc, based on the input specification...
    //
    if (inputRowMeta != null && !inputRowMeta.isEmpty()) {
      // this gets set only in the parent transformation...
      // It includes all the renames that needed to be done
      //
      if (selectingAndSortingUnspecifiedFields) {

        // First rename any fields...
        if (valueRenames != null) {
          for (MappingValueRename valueRename : valueRenames) {
            ValueMetaInterface valueMeta =
                inputRowMeta.searchValueMeta(valueRename.getSourceValueName());
            if (valueMeta == null) {
              throw new KettleStepException(
                  BaseMessages.getString(
                      PKG,
                      "MappingInput.Exception.UnableToFindMappedValue",
                      valueRename.getSourceValueName()));
            }
            valueMeta.setName(valueRename.getTargetValueName());
          }
        }

        // Select the specified fields from the input, re-order everything and put the other fields
        // at the back, sorted...
        //
        RowMetaInterface newRow = new RowMeta();

        for (int i = 0; i < fieldName.length; i++) {
          int index = inputRowMeta.indexOfValue(fieldName[i]);
          if (index < 0) {
            throw new KettleStepException(
                BaseMessages.getString(
                    PKG, "MappingInputMeta.Exception.UnknownField", fieldName[i]));
          }

          newRow.addValueMeta(inputRowMeta.getValueMeta(index));
        }

        // Now get the unspecified fields.
        // Sort the fields
        // Add them after the specified fields...
        //
        List<String> extra = new ArrayList<String>();
        for (int i = 0; i < inputRowMeta.size(); i++) {
          String fieldName = inputRowMeta.getValueMeta(i).getName();
          if (newRow.indexOfValue(fieldName) < 0) {
            extra.add(fieldName);
          }
        }
        Collections.sort(extra);
        for (String fieldName : extra) {
          ValueMetaInterface extraValue = inputRowMeta.searchValueMeta(fieldName);
          newRow.addValueMeta(extraValue);
        }

        // now merge the new row...
        // This is basically the input row meta data with the fields re-ordered.
        //
        row.mergeRowMeta(newRow);
      } else {
        row.mergeRowMeta(inputRowMeta);

        // Validate the existence of all the specified fields...
        //
        if (!row.isEmpty()) {
          for (int i = 0; i < fieldName.length; i++) {
            if (row.indexOfValue(fieldName[i]) < 0) {
              throw new KettleStepException(
                  BaseMessages.getString(
                      PKG, "MappingInputMeta.Exception.UnknownField", fieldName[i]));
            }
          }
        }
      }
    } else {
      // We'll have to work with the statically provided information
      for (int i = 0; i < fieldName.length; i++) {
        if (!Const.isEmpty(fieldName[i])) {
          ValueMetaInterface v = new ValueMeta(fieldName[i], fieldType[i]);
          if (v.getType() == ValueMetaInterface.TYPE_NONE)
            v.setType(ValueMetaInterface.TYPE_STRING);
          v.setLength(fieldLength[i]);
          v.setPrecision(fieldPrecision[i]);
          v.setOrigin(origin);
          row.addValueMeta(v);
        }
      }
    }
  }
  public SQLStatement getSQLStatements(
      TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev) {
    SQLStatement retval =
        new SQLStatement(stepMeta.getName(), databaseWriteMeta, null); // default: nothing to do!

    int i;

    if (databaseWriteMeta != null) {
      if (prev != null && prev.size() > 0) {
        if (!Const.isEmpty(tablename)) {
          String schemaTable =
              databaseWriteMeta.getQuotedSchemaTableCombination(schemaName, tablename);
          Database db = new Database(databaseWriteMeta);
          try {
            boolean doHash = false;
            String cr_table = null;

            db.connect();

            // OK, what do we put in the new table??
            RowMetaInterface fields = new RowMeta();

            ValueMetaInterface vkeyfield = null;
            if (!Const.isEmpty(technicalKeyField)) {
              // First, the new technical key...
              vkeyfield = new ValueMeta(technicalKeyField, ValueMetaInterface.TYPE_INTEGER);
              vkeyfield.setLength(10);
              vkeyfield.setPrecision(0);
            }

            // Then the hashcode (optional)
            ValueMetaInterface vhashfield = null;
            if (useHash && !Const.isEmpty(hashField)) {
              vhashfield = new ValueMeta(hashField, ValueMetaInterface.TYPE_INTEGER);
              vhashfield.setLength(15);
              vhashfield.setPrecision(0);
              doHash = true;
            }

            // Then the last update field (optional)
            ValueMetaInterface vLastUpdateField = null;
            if (!Const.isEmpty(lastUpdateField)) {
              vLastUpdateField = new ValueMeta(lastUpdateField, ValueMetaInterface.TYPE_DATE);
            }

            if (!db.checkTableExists(schemaTable)) {
              if (vkeyfield != null) {
                // Add technical key field.
                fields.addValueMeta(vkeyfield);
              }

              // Add the keys only to the table
              if (keyField != null && keyLookup != null) {
                int cnt = keyField.length;
                for (i = 0; i < cnt; i++) {
                  String error_field = ""; // $NON-NLS-1$

                  // Find the value in the stream
                  ValueMetaInterface v = prev.searchValueMeta(keyField[i]);
                  if (v != null) {
                    String name = keyLookup[i];
                    ValueMetaInterface newValue = v.clone();
                    newValue.setName(name);

                    if (vkeyfield != null) {
                      if (name.equals(vkeyfield.getName())
                          || (doHash == true && name.equals(vhashfield.getName()))) {
                        error_field += name;
                      }
                    }
                    if (error_field.length() > 0) {
                      retval.setError(
                          Messages.getString(
                              "ConcurrentCombinationLookupMeta.ReturnValue.NameCollision",
                              error_field)); //$NON-NLS-1$
                    } else {
                      fields.addValueMeta(newValue);
                    }
                  }
                }
              }

              if (doHash == true) {
                fields.addValueMeta(vhashfield);
              }

              if (vLastUpdateField != null) {
                fields.addValueMeta(vLastUpdateField);
              }
            } else {
              // Table already exists

              // Get the fields that are in the table now:
              RowMetaInterface tabFields = db.getTableFields(schemaTable);

              // Don't forget to quote these as well...
              databaseWriteMeta.quoteReservedWords(tabFields);

              if (vkeyfield != null && tabFields.searchValueMeta(vkeyfield.getName()) == null) {
                // Add technical key field if it didn't exist yet
                fields.addValueMeta(vkeyfield);
              }

              // Add the already existing fields
              int cnt = tabFields.size();
              for (i = 0; i < cnt; i++) {
                ValueMetaInterface v = tabFields.getValueMeta(i);

                fields.addValueMeta(v);
              }

              // Find the missing fields in the real table
              String keyLookup[] = getKeyLookup();
              String keyField[] = getKeyField();
              if (keyField != null && keyLookup != null) {
                cnt = keyField.length;
                for (i = 0; i < cnt; i++) {
                  // Find the value in the stream
                  ValueMetaInterface v = prev.searchValueMeta(keyField[i]);
                  if (v != null) {
                    ValueMetaInterface newValue = v.clone();
                    newValue.setName(keyLookup[i]);

                    // Does the corresponding name exist in the table
                    if (tabFields.searchValueMeta(newValue.getName()) == null) {
                      fields.addValueMeta(newValue); // nope --> add
                    }
                  }
                }
              }

              if (doHash == true && tabFields.searchValueMeta(vhashfield.getName()) == null) {
                // Add hash field
                fields.addValueMeta(vhashfield);
              }

              if (vLastUpdateField != null
                  && tabFields.searchValueMeta(vLastUpdateField.getName()) == null) {
                fields.addValueMeta(vLastUpdateField);
              }
            }

            cr_table =
                db.getDDL(
                    schemaTable,
                    fields,
                    (CREATION_METHOD_SEQUENCE.equals(getTechKeyCreation())
                            && sequenceFrom != null
                            && sequenceFrom.length() != 0)
                        ? null
                        : technicalKeyField,
                    CREATION_METHOD_AUTOINC.equals(getTechKeyCreation()),
                    null,
                    true);

            //
            // OK, now let's build the index
            //

            // What fields do we put int the index?
            // Only the hashcode or all fields?
            String cr_index = ""; // $NON-NLS-1$
            String cr_uniq_index = ""; // $NON-NLS-1$
            String idx_fields[] = null;
            if (useHash) {
              if (hashField != null && hashField.length() > 0) {
                idx_fields = new String[] {hashField};
              } else {
                retval.setError(
                    Messages.getString(
                        "ConcurrentCombinationLookupMeta.ReturnValue.NotHashFieldSpecified")); //$NON-NLS-1$
              }
            } else // index on all key fields...
            {
              if (!Const.isEmpty(keyLookup)) {
                int nrfields = keyLookup.length;
                if (nrfields > 32
                    && databaseWriteMeta.getDatabaseType() == DatabaseMeta.TYPE_DATABASE_ORACLE) {
                  nrfields = 32; // Oracle indexes are limited to 32 fields...
                }
                idx_fields = new String[nrfields];
                for (i = 0; i < nrfields; i++) idx_fields[i] = keyLookup[i];
              } else {
                retval.setError(
                    Messages.getString(
                        "ConcurrentCombinationLookupMeta.ReturnValue.NotFieldsSpecified")); //$NON-NLS-1$
              }
            }

            // OK, now get the create index statement...

            if (!Const.isEmpty(technicalKeyField)) {
              String techKeyArr[] = new String[] {technicalKeyField};
              if (!db.checkIndexExists(schemaName, tablename, techKeyArr)) {
                String indexname = "idx_" + tablename + "_pk"; // $NON-NLS-1$ //$NON-NLS-2$
                cr_uniq_index =
                    db.getCreateIndexStatement(
                        schemaName, tablename, indexname, techKeyArr, true, true, false, true);
                cr_uniq_index += Const.CR;
              }
            }

            // OK, now get the create lookup index statement...
            if (!Const.isEmpty(idx_fields)
                && !db.checkIndexExists(schemaName, tablename, idx_fields)) {
              String indexname = "idx_" + tablename + "_lookup"; // $NON-NLS-1$ //$NON-NLS-2$
              cr_index =
                  db.getCreateIndexStatement(
                      schemaName, tablename, indexname, idx_fields, false, false, false, true);
              cr_index += Const.CR;
            }

            //
            // Don't forget the sequence (optional)
            //
            String cr_seq = ""; // $NON-NLS-1$
            if (databaseWriteMeta.supportsSequences() && !Const.isEmpty(sequenceFrom)) {
              if (!db.checkSequenceExists(schemaName, sequenceFrom)) {
                cr_seq +=
                    db.getCreateSequenceStatement(schemaName, sequenceFrom, 1L, 1L, -1L, true);
                cr_seq += Const.CR;
              }
            }
            retval.setSQL(cr_table + cr_uniq_index + cr_index + cr_seq);
          } catch (KettleException e) {
            retval.setError(
                Messages.getString("ConcurrentCombinationLookupMeta.ReturnValue.ErrorOccurred")
                    + Const.CR
                    + e.getMessage()); // $NON-NLS-1$
          }
        } else {
          retval.setError(
              Messages.getString(
                  "ConcurrentCombinationLookupMeta.ReturnValue.NotTableDefined")); //$NON-NLS-1$
        }
      } else {
        retval.setError(
            Messages.getString(
                "ConcurrentCombinationLookupMeta.ReturnValue.NotReceivingField")); //$NON-NLS-1$
      }
    } else {
      retval.setError(
          Messages.getString(
              "ConcurrentCombinationLookupMeta.ReturnValue.NotConnectionDefined")); //$NON-NLS-1$
    }

    return retval;
  }
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (MySQLTableOutputMeta) smi;
    data = (MySQLTableOutputData) sdi;

    ArrayList<Object[]> rows = new ArrayList<Object[]>();
    for (int counter = 0; counter < data.batchSize; counter++) {
      Object[] r = getRow();
      ; // this also waits for a previous step to be finished.
      if (r != null) {
        rows.add(r);
      }
    }
    if (rows.isEmpty()) {
      return false;
    }

    if (first) {
      first = false;
      data.outputRowMeta = getInputRowMeta().clone();
      meta.getFields(data.outputRowMeta, getStepname(), null, null, this);

      data.insertRowMeta = new RowMeta();

      //
      // Cache the position of the compare fields in Row row
      //
      data.valuenrs = new int[meta.getFieldDatabase().length];
      for (int i = 0; i < meta.getFieldDatabase().length; i++) {
        data.valuenrs[i] = getInputRowMeta().indexOfValue(meta.getFieldStream()[i]);
        if (data.valuenrs[i] < 0) {
          throw new KettleStepException(
              Messages.getString(
                  "TableOutput.Exception.FieldRequired", meta.getFieldStream()[i])); // $NON-NLS-1$
        }
      }

      for (int i = 0; i < meta.getFieldDatabase().length; i++) {
        ValueMetaInterface insValue = getInputRowMeta().searchValueMeta(meta.getFieldStream()[i]);
        if (insValue != null) {
          ValueMetaInterface insertValue = insValue.clone();
          insertValue.setName(meta.getFieldDatabase()[i]);
          data.insertRowMeta.addValueMeta(insertValue);
        } else {
          throw new KettleStepException(
              Messages.getString(
                  "TableOutput.Exception.FailedToFindField",
                  meta.getFieldStream()[i])); // $NON-NLS-1$
        }
      }
    }

    try {
      ArrayList<Object[]> outputRowData = writeToTable(getInputRowMeta(), rows);
      if (outputRowData != null) {
        for (Object[] row : outputRowData) {
          putRow(data.outputRowMeta, row); // in case we want it go
          // further...
          incrementLinesOutput();
        }
      }

      if (checkFeedback(getLinesRead())) {
        if (log.isBasic()) logBasic("linenr " + getLinesRead()); // $NON-NLS-1$
      }
    } catch (KettleException e) {
      logError("Because of an error, this step can't continue: ", e);
      setErrors(1);
      stopAll();
      setOutputDone(); // signal end to receiver(s)
      return false;
    }

    return true;
  }
 private void assertGetFieldDefinition(
     ValueMetaInterface valueMetaInterface, String name, String expectedType) {
   valueMetaInterface = valueMetaInterface.clone();
   valueMetaInterface.setName(name);
   assertGetFieldDefinition(valueMetaInterface, expectedType);
 }
  @Override
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (VerticaBulkLoaderMeta) smi;
    data = (VerticaBulkLoaderData) sdi;

    Object[] r = getRow(); // this also waits for a previous step to be
    // finished.
    if (r == null) // no more input to be expected...
    {

      try {
        data.close();
      } catch (IOException ioe) {
        throw new KettleStepException("Error releasing resources", ioe);
      }
      return false;
    }

    if (first) {

      first = false;

      data.outputRowMeta = getInputRowMeta().clone();
      meta.getFields(data.outputRowMeta, getStepname(), null, null, this);

      RowMetaInterface tableMeta = meta.getTableRowMetaInterface();

      if (!meta.specifyFields()) {

        // Just take the whole input row
        data.insertRowMeta = getInputRowMeta().clone();
        data.selectedRowFieldIndices = new int[data.insertRowMeta.size()];

        data.colSpecs = new ArrayList<ColumnSpec>(data.insertRowMeta.size());

        for (int insertFieldIdx = 0; insertFieldIdx < data.insertRowMeta.size(); insertFieldIdx++) {
          data.selectedRowFieldIndices[insertFieldIdx] = insertFieldIdx;
          ValueMetaInterface inputValueMeta = data.insertRowMeta.getValueMeta(insertFieldIdx);
          ValueMetaInterface insertValueMeta = inputValueMeta.clone();
          ValueMetaInterface targetValueMeta = tableMeta.getValueMeta(insertFieldIdx);
          ColumnSpec cs = getColumnSpecFromField(inputValueMeta, insertValueMeta, targetValueMeta);
          data.colSpecs.add(insertFieldIdx, cs);
        }

      } else {

        int numberOfInsertFields = meta.getFieldDatabase().length;
        data.insertRowMeta = new RowMeta();
        data.colSpecs = new ArrayList<ColumnSpec>(numberOfInsertFields);

        // Cache the position of the selected fields in the row array
        data.selectedRowFieldIndices = new int[numberOfInsertFields];
        for (int insertFieldIdx = 0; insertFieldIdx < numberOfInsertFields; insertFieldIdx++) {

          String inputFieldName = meta.getFieldStream()[insertFieldIdx];
          int inputFieldIdx = getInputRowMeta().indexOfValue(inputFieldName);
          if (inputFieldIdx < 0) {
            throw new KettleStepException(
                BaseMessages.getString(
                    PKG,
                    "VerticaBulkLoader.Exception.FieldRequired",
                    inputFieldName)); //$NON-NLS-1$
          }
          data.selectedRowFieldIndices[insertFieldIdx] = inputFieldIdx;

          String insertFieldName = meta.getFieldDatabase()[insertFieldIdx];
          ValueMetaInterface inputValueMeta = getInputRowMeta().getValueMeta(inputFieldIdx);
          if (inputValueMeta == null) {
            throw new KettleStepException(
                BaseMessages.getString(
                    PKG,
                    "VerticaBulkLoader.Exception.FailedToFindField",
                    meta.getFieldStream()[insertFieldIdx])); // $NON-NLS-1$
          }
          ValueMetaInterface insertValueMeta = inputValueMeta.clone();
          insertValueMeta.setName(insertFieldName);
          data.insertRowMeta.addValueMeta(insertValueMeta);

          ValueMetaInterface targetValueMeta = tableMeta.searchValueMeta(insertFieldName);
          ColumnSpec cs = getColumnSpecFromField(inputValueMeta, insertValueMeta, targetValueMeta);
          data.colSpecs.add(insertFieldIdx, cs);
        }
      }

      try {
        data.pipedInputStream = new PipedInputStream();
        data.encoder = new StreamEncoder(data.colSpecs, data.pipedInputStream);

        initializeWorker();
        data.encoder.writeHeader();

      } catch (IOException ioe) {
        throw new KettleStepException("Error creating stream encoder", ioe);
      }
    }

    try {
      Object[] outputRowData = writeToOutputStream(r);
      if (outputRowData != null) {
        putRow(data.outputRowMeta, outputRowData); // in case we want it
        // go further...
        incrementLinesOutput();
      }

      if (checkFeedback(getLinesRead())) {
        if (log.isBasic()) logBasic("linenr " + getLinesRead()); // $NON-NLS-1$
      }
    } catch (KettleException e) {
      logError("Because of an error, this step can't continue: ", e);
      setErrors(1);
      stopAll();
      setOutputDone(); // signal end to receiver(s)
      return false;
    } catch (IOException e) {
      e.printStackTrace();
    }

    return true;
  }
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (TableOutputMeta) smi;
    data = (TableOutputData) sdi;

    Object[] r = getRow(); // this also waits for a previous step to be finished.
    if (r == null) { // no more input to be expected...

      return false;
    }

    if (first) {
      first = false;
      data.outputRowMeta = getInputRowMeta().clone();
      meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);

      if (!meta.specifyFields()) {
        // Just take the input row
        data.insertRowMeta = getInputRowMeta().clone();
      } else {

        data.insertRowMeta = new RowMeta();

        //
        // Cache the position of the compare fields in Row row
        //
        data.valuenrs = new int[meta.getFieldDatabase().length];
        for (int i = 0; i < meta.getFieldDatabase().length; i++) {
          data.valuenrs[i] = getInputRowMeta().indexOfValue(meta.getFieldStream()[i]);
          if (data.valuenrs[i] < 0) {
            throw new KettleStepException(
                BaseMessages.getString(
                    PKG, "TableOutput.Exception.FieldRequired", meta.getFieldStream()[i]));
          }
        }

        for (int i = 0; i < meta.getFieldDatabase().length; i++) {
          ValueMetaInterface insValue = getInputRowMeta().searchValueMeta(meta.getFieldStream()[i]);
          if (insValue != null) {
            ValueMetaInterface insertValue = insValue.clone();
            insertValue.setName(meta.getFieldDatabase()[i]);
            data.insertRowMeta.addValueMeta(insertValue);
          } else {
            throw new KettleStepException(
                BaseMessages.getString(
                    PKG, "TableOutput.Exception.FailedToFindField", meta.getFieldStream()[i]));
          }
        }
      }
    }

    try {
      Object[] outputRowData = writeToTable(getInputRowMeta(), r);
      if (outputRowData != null) {
        putRow(data.outputRowMeta, outputRowData); // in case we want it go further...
        incrementLinesOutput();
      }

      if (checkFeedback(getLinesRead())) {
        if (log.isBasic()) {
          logBasic("linenr " + getLinesRead());
        }
      }
    } catch (KettleException e) {
      logError("Because of an error, this step can't continue: ", e);
      setErrors(1);
      stopAll();
      setOutputDone(); // signal end to receiver(s)
      return false;
    }

    return true;
  }