Пример #1
0
  public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
    meta = (JsonInputMeta) smi;
    data = (JsonInputData) sdi;

    if (super.init(smi, sdi)) {
      data.rownr = 1L;
      data.nrInputFields = meta.getInputFields().length;
      // Take care of variable substitution
      for (int i = 0; i < data.nrInputFields; i++) {
        JsonInputField field = meta.getInputFields()[i];
        field.setPath(environmentSubstitute(field.getPath()));
      }

      try {
        // Init a new JSON reader
        data.jsonReader = new JsonReader();
        data.jsonReader.SetIgnoreMissingPath(meta.isIgnoreMissingPath());

      } catch (KettleException e) {
        logError(e.getMessage());
        return false;
      }
      return true;
    }
    return false;
  }
  @Override
  public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
    meta = (VerticaBulkLoaderMeta) smi;
    data = (VerticaBulkLoaderData) sdi;

    if (super.init(smi, sdi)) {
      try {
        data.databaseMeta = meta.getDatabaseMeta();

        data.db = new Database(this, meta.getDatabaseMeta());
        data.db.shareVariablesWith(this);

        if (getTransMeta().isUsingUniqueConnections()) {
          synchronized (getTrans()) {
            data.db.connect(getTrans().getThreadName(), getPartitionID());
          }
        } else {
          data.db.connect(getPartitionID());
        }

        if (log.isBasic()) logBasic("Connected to database [" + meta.getDatabaseMeta() + "]");

        data.db.setAutoCommit(false);

        return true;
      } catch (KettleException e) {
        logError("An error occurred intialising this step: " + e.getMessage());
        stopAll();
        setErrors(1);
      }
    }
    return false;
  }
  public SQLStatement getSQLStatements(
      TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev) throws KettleStepException {
    SQLStatement retval =
        new SQLStatement(stepMeta.getName(), databaseMeta, null); // default: nothing to do!

    if (databaseMeta != null) {
      if (prev != null && prev.size() > 0) {
        // Copy the row
        RowMetaInterface tableFields = new RowMeta();

        // Now change the field names
        for (int i = 0; i < fieldTable.length; i++) {
          ValueMetaInterface v = prev.searchValueMeta(fieldStream[i]);
          if (v != null) {
            ValueMetaInterface tableField = v.clone();
            tableField.setName(fieldTable[i]);
            tableFields.addValueMeta(tableField);
          } else {
            throw new KettleStepException(
                "Unable to find field [" + fieldStream[i] + "] in the input rows");
          }
        }

        if (!Const.isEmpty(tableName)) {
          Database db = new Database(loggingObject, databaseMeta);
          db.shareVariablesWith(transMeta);
          try {
            db.connect();

            String schemaTable =
                databaseMeta.getQuotedSchemaTableCombination(
                    transMeta.environmentSubstitute(schemaName),
                    transMeta.environmentSubstitute(tableName));
            String sql = db.getDDL(schemaTable, tableFields, null, false, null, true);

            if (sql.length() == 0) retval.setSQL(null);
            else retval.setSQL(sql);
          } catch (KettleException e) {
            retval.setError(
                BaseMessages.getString(PKG, "GPBulkLoaderMeta.GetSQL.ErrorOccurred")
                    + e.getMessage()); // $NON-NLS-1$
          }
        } else {
          retval.setError(
              BaseMessages.getString(
                  PKG, "GPBulkLoaderMeta.GetSQL.NoTableDefinedOnConnection")); // $NON-NLS-1$
        }
      } else {
        retval.setError(
            BaseMessages.getString(
                PKG, "GPBulkLoaderMeta.GetSQL.NotReceivingAnyFields")); // $NON-NLS-1$
      }
    } else {
      retval.setError(
          BaseMessages.getString(
              PKG, "GPBulkLoaderMeta.GetSQL.NoConnectionDefined")); // $NON-NLS-1$
    }

    return retval;
  }
Пример #4
0
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    try {
      // Grab a row
      Object[] outputRowData = getOneRow();
      if (outputRowData == null) {
        setOutputDone(); // signal end to receiver(s)
        return false; // end of data or error.
      }

      if (isRowLevel())
        logRowlevel(
            BaseMessages.getString(
                PKG, "LoadFileInput.Log.ReadRow", data.outputRowMeta.getString(outputRowData)));

      putRow(data.outputRowMeta, outputRowData);

      if (meta.getRowLimit() > 0
          && data.rownr > meta.getRowLimit()) // limit has been reached: stop now.
      {
        setOutputDone();
        return false;
      }
    } catch (KettleException e) {
      logError(
          BaseMessages.getString(
              PKG, "LoadFileInput.ErrorInStepRunning", e.getMessage())); // $NON-NLS-1$
      logError(Const.getStackTracker(e));
      setErrors(1);
      stopAll();
      setOutputDone(); // signal end to receiver(s)
      return false;
    }
    return true;
  }
  @Test
  public void testReturnDigitsOnly() throws KettleException {
    RowMeta inputRowMeta = new RowMeta();
    ValueMetaString nameMeta = new ValueMetaString("Name");
    inputRowMeta.addValueMeta(nameMeta);
    ValueMetaString valueMeta = new ValueMetaString("Value");
    inputRowMeta.addValueMeta(valueMeta);

    RowSet inputRowSet =
        smh.getMockInputRowSet(new Object[][] {{"name1", "qwe123asd456zxc"}, {"name2", null}});
    inputRowSet.setRowMeta(inputRowMeta);

    Calculator calculator =
        new Calculator(smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans);
    calculator.getInputRowSets().add(inputRowSet);
    calculator.setInputRowMeta(inputRowMeta);
    calculator.init(smh.initStepMetaInterface, smh.initStepDataInterface);

    CalculatorMeta meta = new CalculatorMeta();
    meta.setCalculation(
        new CalculatorMetaFunction[] {
          new CalculatorMetaFunction(
              "digits",
              CalculatorMetaFunction.CALC_GET_ONLY_DIGITS,
              "Value",
              null,
              null,
              ValueMetaInterface.TYPE_STRING,
              0,
              0,
              false,
              "",
              "",
              "",
              "")
        });

    // Verify output
    try {
      calculator.addRowListener(
          new RowAdapter() {
            @Override
            public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
                throws KettleStepException {
              assertEquals("123456", row[2]);
            }
          });
      calculator.processRow(meta, new CalculatorData());
    } catch (KettleException ke) {
      ke.printStackTrace();
      fail();
    }
  }
  public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
    meta = (SalesforceDeleteMeta) smi;
    data = (SalesforceDeleteData) sdi;

    if (super.init(smi, sdi)) {

      try {
        data.realModule = environmentSubstitute(meta.getModule());
        // Check if module is specified
        if (Const.isEmpty(data.realModule)) {
          log.logError(
              BaseMessages.getString(PKG, "SalesforceDeleteDialog.ModuleMissing.DialogMessage"));
          return false;
        }

        String realUser = environmentSubstitute(meta.getUserName());
        // Check if username is specified
        if (Const.isEmpty(realUser)) {
          log.logError(
              BaseMessages.getString(PKG, "SalesforceDeleteDialog.UsernameMissing.DialogMessage"));
          return false;
        }

        // initialize variables
        data.realURL = environmentSubstitute(meta.getTargetURL());
        // create a Salesforce connection
        data.connection =
            new SalesforceConnection(
                log, data.realURL, realUser, environmentSubstitute(meta.getPassword()));
        // set timeout
        data.connection.setTimeOut(Const.toInt(environmentSubstitute(meta.getTimeOut()), 0));
        // Do we use compression?
        data.connection.setUsingCompression(meta.isUsingCompression());
        // Do we rollback all changes on error
        data.connection.rollbackAllChangesOnError(meta.isRollbackAllChangesOnError());

        // Now connect ...
        data.connection.connect();

        return true;
      } catch (KettleException ke) {
        logError(
            BaseMessages.getString(PKG, "SalesforceDelete.Log.ErrorOccurredDuringStepInitialize")
                + ke.getMessage()); // $NON-NLS-1$
      }
      return true;
    }
    return false;
  }
Пример #7
0
  public void loadRep(
      Repository rep,
      IMetaStore metaStore,
      ObjectId id_jobentry,
      List<DatabaseMeta> databases,
      List<SlaveServer> slaveServers)
      throws KettleException {
    try {
      serverName = rep.getJobEntryAttributeString(id_jobentry, "servername");
      userName = rep.getJobEntryAttributeString(id_jobentry, "username");
      password =
          Encr.decryptPasswordOptionallyEncrypted(
              rep.getJobEntryAttributeString(id_jobentry, "password"));
      serverPort = rep.getJobEntryAttributeString(id_jobentry, "serverport");
      ftpDirectory = rep.getJobEntryAttributeString(id_jobentry, "ftpdirectory");
      localDirectory = rep.getJobEntryAttributeString(id_jobentry, "localdirectory");
      wildcard = rep.getJobEntryAttributeString(id_jobentry, "wildcard");
      onlyGettingNewFiles = rep.getJobEntryAttributeBoolean(id_jobentry, "only_new");

      usehttpproxy = rep.getJobEntryAttributeBoolean(id_jobentry, "usehttpproxy");
      httpProxyHost = rep.getJobEntryAttributeString(id_jobentry, "httpproxyhost");
      httpproxyusername = rep.getJobEntryAttributeString(id_jobentry, "httpproxyusername");
      httpProxyPassword = rep.getJobEntryAttributeString(id_jobentry, "httpproxypassword");

      publicpublickey = rep.getJobEntryAttributeBoolean(id_jobentry, "publicpublickey");
      keyFilename = rep.getJobEntryAttributeString(id_jobentry, "keyfilename");
      keyFilePass = rep.getJobEntryAttributeString(id_jobentry, "keyfilepass");

      useBasicAuthentication =
          rep.getJobEntryAttributeBoolean(id_jobentry, "usebasicauthentication");
      afterFtpPut = rep.getJobEntryAttributeString(id_jobentry, "afterftpput");
      destinationfolder = rep.getJobEntryAttributeString(id_jobentry, "destinationfolder");

      createdestinationfolder =
          rep.getJobEntryAttributeBoolean(id_jobentry, "createdestinationfolder");
      cachehostkey = rep.getJobEntryAttributeBoolean(id_jobentry, "cachehostkey");
      timeout = (int) rep.getJobEntryAttributeInteger(id_jobentry, "timeout");

      createtargetfolder = rep.getJobEntryAttributeBoolean(id_jobentry, "createtargetfolder");
      includeSubFolders = rep.getJobEntryAttributeBoolean(id_jobentry, "includeSubFolders");

    } catch (KettleException dbe) {
      throw new KettleException(
          BaseMessages.getString(
              PKG, "JobSSH2GET.Log.UnableLoadRep", "" + id_jobentry, dbe.getMessage()));
    }
  }
Пример #8
0
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (AccessOutputMeta) smi;
    data = (AccessOutputData) sdi;

    Object[] r = getRow(); // this also waits for a previous step to be finished.
    if (r == null) // no more input to be expected...
    {
      setOutputDone();
      return false;
    }

    if (first && meta.isDoNotOpenNewFileInit()) {
      try {
        if (!OpenFile()) {
          return false;
        }

      } catch (Exception e) {
        logError("An error occurred intialising this step: " + e.getMessage());
        stopAll();
        setErrors(1);
      }
    }
    try {
      writeToTable(r);
      putRow(data.outputRowMeta, r); // in case we want it go further...

      if (checkFeedback(getLinesOutput())) {
        if (log.isBasic()) {
          logBasic("linenr " + getLinesOutput());
        }
      }
    } catch (KettleException e) {
      logError("Because of an error, this step can't continue: " + e.getMessage());
      setErrors(1);
      stopAll();
      setOutputDone(); // signal end to receiver(s)
      return false;
    }

    return true;
  }
  public void getUsedArguments(
      JobMeta jobMeta, String[] commandLineArguments, IMetaStore metaStore) {

    for (JobEntryCopy jobEntryCopy : jobMeta.jobcopies) {
      if (jobEntryCopy.isTransformation()) {
        JobEntryTrans jobEntryTrans = (JobEntryTrans) jobEntryCopy.getEntry();
        try {
          TransMeta transMeta = jobEntryTrans.getTransMeta(repository, metaStore, jobMeta);
          Map<String, String> map = transMeta.getUsedArguments(commandLineArguments);
          for (String key : map.keySet()) {
            String value = map.get(key);
            if (!arguments.containsKey(key)) {
              arguments.put(key, value);
            }
          }
        } catch (KettleException ke) {
          log.logBasic(ke.getMessage(), ke);
        }
      }
    }
  }
  public static synchronized void initializePentahoSystem(String solutionPath) throws Exception {

    // this setting is useful only for running the integration tests from within IntelliJ
    // this same property is set for integration tests via the pom when running with mvn
    String folderPaths = "target/spoon/plugins";
    File f = new File(folderPaths);
    System.setProperty("KETTLE_PLUGIN_BASE_FOLDERS", f.getAbsolutePath());

    StandaloneApplicationContext appContext = new StandaloneApplicationContext(solutionPath, "");
    PentahoSystem.setSystemSettingsService(new PathBasedSystemSettings());
    if (solutionPath == null) {
      throw new MetaverseException(
          Messages.getString("ERROR.MetaverseInit.BadConfigPath", solutionPath));
    }

    try {
      ClassLoader cl = Thread.currentThread().getContextClassLoader();

      Thread.currentThread().setContextClassLoader(MetaverseUtil.class.getClassLoader());
      IPentahoObjectFactory pentahoObjectFactory = new StandaloneSpringPentahoObjectFactory();
      pentahoObjectFactory.init(solutionPath, PentahoSystem.getApplicationContext());
      PentahoSystem.registerObjectFactory(pentahoObjectFactory);

      // Restore context classloader
      Thread.currentThread().setContextClassLoader(cl);
    } catch (Exception e) {
      throw new MetaverseException(Messages.getString("ERROR.MetaverseInit.CouldNotInit"), e);
    }
    PentahoSystem.init(appContext);
    PentahoSessionHolder.setSession(new StandaloneSession());

    registerKettlePlugins();

    try {
      KettleEnvironment.init();
    } catch (KettleException e) {
      e.printStackTrace();
    }
  }
  public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
    meta = (SalesforceUpdateMeta) smi;
    data = (SalesforceUpdateData) sdi;

    if (super.init(smi, sdi)) {

      try {
        // Do we need to rollback all changes on error
        data.connection.setRollbackAllChangesOnError(meta.isRollbackAllChangesOnError());

        // Now connect ...
        data.connection.connect();

        return true;
      } catch (KettleException ke) {
        logError(
            BaseMessages.getString(PKG, "SalesforceUpdate.Log.ErrorOccurredDuringStepInitialize")
                + ke.getMessage());
      }
      return true;
    }
    return false;
  }
  public void loadRep(
      Repository rep,
      IMetaStore metaStore,
      ObjectId id_jobentry,
      List<DatabaseMeta> databases,
      List<SlaveServer> slaveServers)
      throws KettleException {
    try {
      replaceVars = rep.getJobEntryAttributeBoolean(id_jobentry, "replacevars");

      filename = rep.getJobEntryAttributeString(id_jobentry, "filename");
      fileVariableType =
          getVariableType(rep.getJobEntryAttributeString(id_jobentry, "file_variable_type"));

      // How many variableName?
      int argnr = rep.countNrJobEntryAttributes(id_jobentry, "variable_name");
      variableName = new String[argnr];
      variableValue = new String[argnr];
      variableType = new int[argnr];

      // Read them all...
      for (int a = 0; a < argnr; a++) {
        variableName[a] = rep.getJobEntryAttributeString(id_jobentry, a, "variable_name");
        variableValue[a] = rep.getJobEntryAttributeString(id_jobentry, a, "variable_value");
        variableType[a] =
            getVariableType(rep.getJobEntryAttributeString(id_jobentry, a, "variable_type"));
      }
    } catch (KettleException dbe) {
      throw new KettleException(
          BaseMessages.getString(
              PKG,
              "JobEntrySetVariables.Meta.UnableLoadRep",
              String.valueOf(id_jobentry),
              dbe.getMessage()),
          dbe);
    }
  }
Пример #13
0
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (HTTPPOSTMeta) smi;
    data = (HTTPPOSTData) sdi;

    boolean sendToErrorRow = false;
    String errorMessage = null;

    Object[] r = getRow(); // Get row from input rowset & set row busy!
    if (r == null) // no more input to be expected...
    {
      setOutputDone();
      return false;
    }
    if (first) {
      first = false;
      data.inputRowMeta = getInputRowMeta();
      data.outputRowMeta = getInputRowMeta().clone();
      meta.getFields(data.outputRowMeta, getStepname(), null, null, this);

      if (meta.isUrlInField()) {
        if (Const.isEmpty(meta.getUrlField())) {
          logError(Messages.getString("HTTPPOST.Log.NoField"));
          throw new KettleException(Messages.getString("HTTPPOST.Log.NoField"));
        }

        // cache the position of the field
        if (data.indexOfUrlField < 0) {
          String realUrlfieldName = environmentSubstitute(meta.getUrlField());
          data.indexOfUrlField = data.inputRowMeta.indexOfValue((realUrlfieldName));
          if (data.indexOfUrlField < 0) {
            // The field is unreachable !
            logError(Messages.getString("HTTPPOST.Log.ErrorFindingField", realUrlfieldName));
            throw new KettleException(
                Messages.getString("HTTPPOST.Exception.ErrorFindingField", realUrlfieldName));
          }
        }
      } else {
        data.realUrl = environmentSubstitute(meta.getUrl());
      }
      // set body parameters
      int nrargs = meta.getArgumentField().length;
      if (nrargs > 0) {
        data.useBodyParameters = true;
        data.body_parameters_nrs = new int[nrargs];
        data.bodyParameters = new NameValuePair[nrargs];
        for (int i = 0; i < nrargs; i++) {
          data.body_parameters_nrs[i] = data.inputRowMeta.indexOfValue(meta.getArgumentField()[i]);
          if (data.body_parameters_nrs[i] < 0) {
            logError(
                Messages.getString("HTTPPOST.Log.ErrorFindingField")
                    + meta.getArgumentField()[i]
                    + "]"); //$NON-NLS-1$ //$NON-NLS-2$
            throw new KettleStepException(
                Messages.getString(
                    "HTTPPOST.Exception.CouldnotFindField",
                    meta.getArgumentField()[i])); // $NON-NLS-1$ //$NON-NLS-2$
          }
          data.bodyParameters[i] =
              new NameValuePair(
                  environmentSubstitute(meta.getArgumentParameter()[i]),
                  data.outputRowMeta.getString(r, data.body_parameters_nrs[i]));
        }
      }
      // set query parameters
      int nrQuery = meta.getQueryField().length;
      if (nrQuery > 0) {
        data.useQueryParameters = true;
        data.query_parameters_nrs = new int[nrQuery];
        data.queryParameters = new NameValuePair[nrQuery];
        for (int i = 0; i < nrQuery; i++) {
          data.query_parameters_nrs[i] = data.inputRowMeta.indexOfValue(meta.getQueryField()[i]);
          if (data.query_parameters_nrs[i] < 0) {
            logError(
                Messages.getString("HTTPPOST.Log.ErrorFindingField")
                    + meta.getQueryField()[i]
                    + "]"); //$NON-NLS-1$ //$NON-NLS-2$
            throw new KettleStepException(
                Messages.getString(
                    "HTTPPOST.Exception.CouldnotFindField",
                    meta.getQueryField()[i])); // $NON-NLS-1$ //$NON-NLS-2$
          }
          data.queryParameters[i] =
              new NameValuePair(
                  environmentSubstitute(meta.getQueryParameter()[i]),
                  data.outputRowMeta.getString(r, data.query_parameters_nrs[i]));
        }
      }
      // set request entity?
      if (!Const.isEmpty(meta.getRequestEntity())) {
        data.indexOfRequestEntity =
            data.inputRowMeta.indexOfValue(environmentSubstitute(meta.getRequestEntity()));
        if (data.indexOfRequestEntity < 0) {
          throw new KettleStepException(
              Messages.getString(
                  "HTTPPOST.Exception.CouldnotFindRequestEntityField",
                  meta.getRequestEntity())); // $NON-NLS-1$ //$NON-NLS-2$
        }
      }
      data.realEncoding = environmentSubstitute(meta.getEncoding());
    } // end if first

    try {
      Object[] outputRowData = callHTTPPOST(r);
      putRow(data.outputRowMeta, outputRowData); // copy row to output rowset(s);

      if (checkFeedback(getLinesRead())) {
        if (log.isDetailed())
          logDetailed(Messages.getString("HTTPPOST.LineNumber") + getLinesRead()); // $NON-NLS-1$
      }
    } catch (KettleException e) {
      if (getStepMeta().isDoingErrorHandling()) {
        sendToErrorRow = true;
        errorMessage = e.toString();
      } else {
        logError(Messages.getString("HTTPPOST.ErrorInStepRunning") + e.getMessage()); // $NON-NLS-1$
        setErrors(1);
        logError(Const.getStackTracker(e));
        stopAll();
        setOutputDone(); // signal end to receiver(s)
        return false;
      }

      if (sendToErrorRow) {
        // Simply add this row to the error row
        putError(getInputRowMeta(), r, 1, errorMessage, null, "HTTPPOST001");
      }
    }

    return true;
  }
Пример #14
0
  public SQLStatement getSQLStatements(
      TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev) {
    SQLStatement retval =
        new SQLStatement(stepMeta.getName(), databaseMeta, null); // default: nothing to do!

    if (databaseMeta != null) {
      if (prev != null && prev.size() > 0) {
        if (!Const.isEmpty(tableName)) {
          Database db = new Database(databaseMeta);
          db.shareVariablesWith(transMeta);
          try {
            db.connect();

            String schemaTable =
                databaseMeta.getQuotedSchemaTableCombination(schemaName, tableName);
            String cr_table = db.getDDL(schemaTable, prev, null, false, null, true);

            String cr_index = ""; // $NON-NLS-1$
            String idx_fields[] = null;

            if (keyLookup != null && keyLookup.length > 0) {
              idx_fields = new String[keyLookup.length];
              for (int i = 0; i < keyLookup.length; i++) idx_fields[i] = keyLookup[i];
            } else {
              retval.setError(
                  Messages.getString("DeleteMeta.CheckResult.KeyFieldsRequired")); // $NON-NLS-1$
            }

            // Key lookup dimensions...
            if (idx_fields != null
                && idx_fields.length > 0
                && !db.checkIndexExists(schemaTable, idx_fields)) {
              String indexname = "idx_" + tableName + "_lookup"; // $NON-NLS-1$ //$NON-NLS-2$
              cr_index =
                  db.getCreateIndexStatement(
                      schemaName, tableName, indexname, idx_fields, false, false, false, true);
            }

            String sql = cr_table + cr_index;
            if (sql.length() == 0) retval.setSQL(null);
            else retval.setSQL(sql);
          } catch (KettleException e) {
            retval.setError(
                Messages.getString("DeleteMeta.Returnvalue.ErrorOccurred")
                    + e.getMessage()); // $NON-NLS-1$
          }
        } else {
          retval.setError(
              Messages.getString(
                  "DeleteMeta.Returnvalue.NoTableDefinedOnConnection")); //$NON-NLS-1$
        }
      } else {
        retval.setError(
            Messages.getString("DeleteMeta.Returnvalue.NoReceivingAnyFields")); // $NON-NLS-1$
      }
    } else {
      retval.setError(
          Messages.getString("DeleteMeta.Returnvalue.NoConnectionDefined")); // $NON-NLS-1$
    }

    return retval;
  }
Пример #15
0
  public void check(
      List<CheckResultInterface> remarks,
      TransMeta transMeta,
      StepMeta stepinfo,
      RowMetaInterface prev,
      String input[],
      String output[],
      RowMetaInterface info) {
    CheckResult cr;
    String error_message = ""; // $NON-NLS-1$

    if (databaseMeta != null) {
      Database db = new Database(databaseMeta);
      db.shareVariablesWith(transMeta);
      try {
        db.connect();

        if (!Const.isEmpty(tableName)) {
          cr =
              new CheckResult(
                  CheckResultInterface.TYPE_RESULT_OK,
                  Messages.getString("DeleteMeta.CheckResult.TablenameOK"),
                  stepinfo); //$NON-NLS-1$
          remarks.add(cr);

          boolean first = true;
          boolean error_found = false;
          error_message = ""; // $NON-NLS-1$

          // Check fields in table
          String schemaTable = databaseMeta.getQuotedSchemaTableCombination(schemaName, tableName);
          RowMetaInterface r = db.getTableFields(schemaTable);
          if (r != null) {
            cr =
                new CheckResult(
                    CheckResultInterface.TYPE_RESULT_OK,
                    Messages.getString("DeleteMeta.CheckResult.VisitTableSuccessfully"),
                    stepinfo); //$NON-NLS-1$
            remarks.add(cr);

            for (int i = 0; i < keyLookup.length; i++) {
              String lufield = keyLookup[i];

              ValueMetaInterface v = r.searchValueMeta(lufield);
              if (v == null) {
                if (first) {
                  first = false;
                  error_message +=
                      Messages.getString("DeleteMeta.CheckResult.MissingCompareFieldsInTargetTable")
                          + Const.CR; // $NON-NLS-1$
                }
                error_found = true;
                error_message += "\t\t" + lufield + Const.CR; // $NON-NLS-1$
              }
            }
            if (error_found) {
              cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepinfo);
            } else {
              cr =
                  new CheckResult(
                      CheckResultInterface.TYPE_RESULT_OK,
                      Messages.getString("DeleteMeta.CheckResult.FoundLookupFields"),
                      stepinfo); //$NON-NLS-1$
            }
            remarks.add(cr);
          } else {
            error_message =
                Messages.getString("DeleteMeta.CheckResult.CouldNotReadTableInfo"); // $NON-NLS-1$
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepinfo);
            remarks.add(cr);
          }
        }

        // Look up fields in the input stream <prev>
        if (prev != null && prev.size() > 0) {
          cr =
              new CheckResult(
                  CheckResultInterface.TYPE_RESULT_OK,
                  Messages.getString(
                      "DeleteMeta.CheckResult.ConnectedStepSuccessfully",
                      String.valueOf(prev.size())),
                  stepinfo); //$NON-NLS-1$ //$NON-NLS-2$
          remarks.add(cr);

          boolean first = true;
          error_message = ""; // $NON-NLS-1$
          boolean error_found = false;

          for (int i = 0; i < keyStream.length; i++) {
            ValueMetaInterface v = prev.searchValueMeta(keyStream[i]);
            if (v == null) {
              if (first) {
                first = false;
                error_message +=
                    Messages.getString("DeleteMeta.CheckResult.MissingFields")
                        + Const.CR; // $NON-NLS-1$
              }
              error_found = true;
              error_message += "\t\t" + keyStream[i] + Const.CR; // $NON-NLS-1$
            }
          }
          for (int i = 0; i < keyStream2.length; i++) {
            if (keyStream2[i] != null && keyStream2[i].length() > 0) {
              ValueMetaInterface v = prev.searchValueMeta(keyStream2[i]);
              if (v == null) {
                if (first) {
                  first = false;
                  error_message +=
                      Messages.getString("DeleteMeta.CheckResult.MissingFields2")
                          + Const.CR; // $NON-NLS-1$
                }
                error_found = true;
                error_message += "\t\t" + keyStream[i] + Const.CR; // $NON-NLS-1$
              }
            }
          }
          if (error_found) {
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepinfo);
          } else {
            cr =
                new CheckResult(
                    CheckResultInterface.TYPE_RESULT_OK,
                    Messages.getString("DeleteMeta.CheckResult.AllFieldsFound"),
                    stepinfo); //$NON-NLS-1$
          }
          remarks.add(cr);

          // How about the fields to insert/update the table with?
          first = true;
          error_found = false;
          error_message = ""; // $NON-NLS-1$
        } else {
          error_message =
              Messages.getString("DeleteMeta.CheckResult.MissingFields3") + Const.CR; // $NON-NLS-1$
          cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepinfo);
          remarks.add(cr);
        }
      } catch (KettleException e) {
        error_message =
            Messages.getString("DeleteMeta.CheckResult.DatabaseError")
                + e.getMessage(); // $NON-NLS-1$
        cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepinfo);
        remarks.add(cr);
      } finally {
        db.disconnect();
      }
    } else {
      error_message = Messages.getString("DeleteMeta.CheckResult.InvalidConnection"); // $NON-NLS-1$
      cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepinfo);
      remarks.add(cr);
    }

    // See if we have input streams leading to this step!
    if (input.length > 0) {
      cr =
          new CheckResult(
              CheckResultInterface.TYPE_RESULT_OK,
              Messages.getString("DeleteMeta.CheckResult.StepReceivingInfo"),
              stepinfo); //$NON-NLS-1$
      remarks.add(cr);
    } else {
      cr =
          new CheckResult(
              CheckResultInterface.TYPE_RESULT_ERROR,
              Messages.getString("DeleteMeta.CheckResult.NoInputReceived"),
              stepinfo); //$NON-NLS-1$
      remarks.add(cr);
    }
  }
Пример #16
0
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (GPLoadMeta) smi;
    data = (GPLoadData) sdi;

    try {
      Object[] r = getRow(); // Get row from input rowset & set row busy!
      if (r == null) // no more input to be expected...
      {
        setOutputDone();

        if (!preview) {
          if (output != null) {
            // Close the output
            try {
              output.close();
            } catch (IOException e) {
              throw new KettleException("Error while closing output", e);
            }

            output = null;
          }

          String loadMethod = meta.getLoadMethod();
          if (GPLoadMeta.METHOD_AUTO_END.equals(loadMethod)) {
            execute(meta, true);
          }
          //	else if ( GPLoadMeta.METHOD_AUTO_CONCURRENT.equals(meta.getLoadMethod()) )
          //	{
          //		try
          //		{
          //			if ( psqlProcess != null )
          //			{
          //				int exitVal = psqlProcess.waitFor();
          //				logBasic(BaseMessages.getString(PKG, "GPLoad.Log.ExitValueSqlldr", "" + exitVal));
          // //$NON-NLS-1$
          //			}
          //			else
          //			{
          //				throw new KettleException("Internal error: no sqlldr process running");
          //			}
          //		}
          //		catch ( Exception ex )
          //		{
          //			throw new KettleException("Error while executing sqlldr", ex);
          //		}
          //	}
        }
        return false;
      }

      if (!preview) {
        if (first) {
          first = false;
          createControlFile(environmentSubstitute(meta.getControlFile()), r, meta);
          output = new GPLoadDataOutput(meta, log.getLogLevel());

          //	if ( GPLoadMeta.METHOD_AUTO_CONCURRENT.equals(meta.getLoadMethod()) )
          //	{
          //		execute(meta, false);
          //	}
          output.open(this, gploadProcess);
        }
        output.writeLine(getInputRowMeta(), r);
      }
      putRow(getInputRowMeta(), r);
      incrementLinesOutput();

    } catch (KettleException e) {
      logError(
          BaseMessages.getString(PKG, "GPLoad.Log.ErrorInStep") + e.getMessage()); // $NON-NLS-1$
      setErrors(1);
      stopAll();
      setOutputDone(); // signal end to receiver(s)
      return false;
    }

    return true;
  }
Пример #17
0
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (OraBulkLoaderMeta) smi;
    data = (OraBulkLoaderData) sdi;

    try {
      Object[] r = getRow(); // Get row from input rowset & set row busy!
      if (r == null) {
        // no more input to be expected...

        setOutputDone();

        if (!preview) {
          if (output != null) {
            // Close the output
            try {
              output.close();
            } catch (IOException e) {
              throw new KettleException("Error while closing output", e);
            }

            output = null;
          }

          String loadMethod = meta.getLoadMethod();
          if (OraBulkLoaderMeta.METHOD_AUTO_END.equals(loadMethod)) {
            // if this is the first line, we do not need to execute loader
            // control file may not exists
            if (!first) {
              execute(meta, true);
              sqlldrProcess = null;
            }
          } else if (OraBulkLoaderMeta.METHOD_AUTO_CONCURRENT.equals(meta.getLoadMethod())) {
            try {
              if (sqlldrProcess != null) {
                int exitVal = sqlldrProcess.waitFor();
                sqlldrProcess = null;
                logBasic(
                    BaseMessages.getString(PKG, "OraBulkLoader.Log.ExitValueSqlldr", "" + exitVal));
                checkExitVal(exitVal);
              } else if (!first) {
                throw new KettleException("Internal error: no sqlldr process running");
              }
            } catch (Exception ex) {
              throw new KettleException("Error while executing sqlldr", ex);
            }
          }
        }
        return false;
      }

      if (!preview) {
        if (first) {
          first = false;

          String recTerm = Const.CR;
          if (!Utils.isEmpty(meta.getAltRecordTerm())) {
            recTerm = substituteRecordTerminator(meta.getAltRecordTerm());
          }

          createControlFile(environmentSubstitute(meta.getControlFile()), r, meta);
          output = new OraBulkDataOutput(meta, recTerm);

          if (OraBulkLoaderMeta.METHOD_AUTO_CONCURRENT.equals(meta.getLoadMethod())) {
            execute(meta, false);
          }
          output.open(this, sqlldrProcess);
        }
        output.writeLine(getInputRowMeta(), r);
      }
      putRow(getInputRowMeta(), r);
      incrementLinesOutput();

    } catch (KettleException e) {
      logError(BaseMessages.getString(PKG, "OraBulkLoader.Log.ErrorInStep") + e.getMessage());
      setErrors(1);
      stopAll();
      setOutputDone(); // signal end to receiver(s)
      return false;
    }

    return true;
  }
Пример #18
0
  public SQLStatement getSQLStatements(
      TransMeta transMeta,
      StepMeta stepMeta,
      RowMetaInterface prev,
      Repository repository,
      IMetaStore metaStore)
      throws KettleStepException {
    SQLStatement retval =
        new SQLStatement(stepMeta.getName(), databaseMeta, null); // default: nothing to do!

    if (databaseMeta != null) {
      if (prev != null && prev.size() > 0) {
        // Copy the row
        RowMetaInterface tableFields = new RowMeta();

        // Now change the field names
        // the key fields
        if (keyLookup != null) {
          for (int i = 0; i < keyLookup.length; i++) {
            ValueMetaInterface v = prev.searchValueMeta(keyStream[i]);
            if (v != null) {
              ValueMetaInterface tableField = v.clone();
              tableField.setName(keyLookup[i]);
              tableFields.addValueMeta(tableField);
            } else {
              throw new KettleStepException(
                  "Unable to find field [" + keyStream[i] + "] in the input rows");
            }
          }
        }
        // the lookup fields
        for (int i = 0; i < updateLookup.length; i++) {
          ValueMetaInterface v = prev.searchValueMeta(updateStream[i]);
          if (v != null) {
            ValueMetaInterface vk = tableFields.searchValueMeta(updateStream[i]);
            if (vk == null) { // do not add again when already added as key fields
              ValueMetaInterface tableField = v.clone();
              tableField.setName(updateLookup[i]);
              tableFields.addValueMeta(tableField);
            }
          } else {
            throw new KettleStepException(
                "Unable to find field [" + updateStream[i] + "] in the input rows");
          }
        }

        if (!Const.isEmpty(tableName)) {
          Database db = new Database(loggingObject, databaseMeta);
          db.shareVariablesWith(transMeta);
          try {
            db.connect();

            String schemaTable =
                databaseMeta.getQuotedSchemaTableCombination(schemaName, tableName);
            String cr_table = db.getDDL(schemaTable, tableFields, null, false, null, true);

            String cr_index = "";
            String[] idx_fields = null;

            if (keyLookup != null && keyLookup.length > 0) {
              idx_fields = new String[keyLookup.length];
              for (int i = 0; i < keyLookup.length; i++) {
                idx_fields[i] = keyLookup[i];
              }
            } else {
              retval.setError(
                  BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.MissingKeyFields"));
            }

            // Key lookup dimensions...
            if (idx_fields != null
                && idx_fields.length > 0
                && !db.checkIndexExists(schemaName, tableName, idx_fields)) {
              String indexname = "idx_" + tableName + "_lookup";
              cr_index =
                  db.getCreateIndexStatement(
                      schemaTable, indexname, idx_fields, false, false, false, true);
            }

            String sql = cr_table + cr_index;
            if (sql.length() == 0) {
              retval.setSQL(null);
            } else {
              retval.setSQL(sql);
            }
          } catch (KettleException e) {
            retval.setError(
                BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.ErrorOccurred")
                    + e.getMessage());
          }
        } else {
          retval.setError(
              BaseMessages.getString(
                  PKG, "InsertUpdateMeta.ReturnValue.NoTableDefinedOnConnection"));
        }
      } else {
        retval.setError(
            BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.NotReceivingAnyFields"));
      }
    } else {
      retval.setError(
          BaseMessages.getString(PKG, "InsertUpdateMeta.ReturnValue.NoConnectionDefined"));
    }

    return retval;
  }
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (ExecProcessMeta) smi;
    data = (ExecProcessData) sdi;

    Object[] r = getRow(); // Get row from input rowset & set row busy!
    if (r == null) { // no more input to be expected...

      setOutputDone();
      return false;
    }

    if (first) {
      first = false;
      // get the RowMeta
      data.previousRowMeta = getInputRowMeta().clone();
      data.NrPrevFields = data.previousRowMeta.size();
      data.outputRowMeta = data.previousRowMeta;
      meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);

      // Check is process field is provided
      if (Const.isEmpty(meta.getProcessField())) {
        logError(BaseMessages.getString(PKG, "ExecProcess.Error.ProcessFieldMissing"));
        throw new KettleException(
            BaseMessages.getString(PKG, "ExecProcess.Error.ProcessFieldMissing"));
      }

      // cache the position of the field
      if (data.indexOfProcess < 0) {
        data.indexOfProcess = data.previousRowMeta.indexOfValue(meta.getProcessField());
        if (data.indexOfProcess < 0) {
          // The field is unreachable !
          logError(
              BaseMessages.getString(PKG, "ExecProcess.Exception.CouldnotFindField")
                  + "["
                  + meta.getProcessField()
                  + "]");
          throw new KettleException(
              BaseMessages.getString(
                  PKG, "ExecProcess.Exception.CouldnotFindField", meta.getProcessField()));
        }
      }
    } // End If first

    Object[] outputRow = RowDataUtil.allocateRowData(data.outputRowMeta.size());
    for (int i = 0; i < data.NrPrevFields; i++) {
      outputRow[i] = r[i];
    }
    // get process to execute
    String processString = data.previousRowMeta.getString(r, data.indexOfProcess);

    ProcessResult processResult = new ProcessResult();

    try {
      if (Const.isEmpty(processString)) {
        throw new KettleException(BaseMessages.getString(PKG, "ExecProcess.ProcessEmpty"));
      }

      // execute and return result
      execProcess(processString, processResult);

      if (meta.isFailWhenNotSuccess()) {
        if (processResult.getExistStatus() != 0) {
          String errorString = processResult.getErrorStream();
          if (Const.isEmpty(errorString)) {
            errorString = processResult.getOutputStream();
          }
          throw new KettleException(errorString);
        }
      }

      // Add result field to input stream
      int rowIndex = data.NrPrevFields;
      outputRow[rowIndex++] = processResult.getOutputStream();

      // Add result field to input stream
      outputRow[rowIndex++] = processResult.getErrorStream();

      // Add result field to input stream
      outputRow[rowIndex++] = processResult.getExistStatus();

      // add new values to the row.
      putRow(data.outputRowMeta, outputRow); // copy row to output rowset(s);

      if (log.isRowLevel()) {
        logRowlevel(
            BaseMessages.getString(
                PKG,
                "ExecProcess.LineNumber",
                getLinesRead() + " : " + getInputRowMeta().getString(r)));
      }
    } catch (KettleException e) {

      boolean sendToErrorRow = false;
      String errorMessage = null;

      if (getStepMeta().isDoingErrorHandling()) {
        sendToErrorRow = true;
        errorMessage = e.toString();
      } else {
        logError(BaseMessages.getString(PKG, "ExecProcess.ErrorInStepRunning") + e.getMessage());
        setErrors(1);
        stopAll();
        setOutputDone(); // signal end to receiver(s)
        return false;
      }
      if (sendToErrorRow) {
        // Simply add this row to the error row
        putError(
            getInputRowMeta(), r, 1, errorMessage, meta.getResultFieldName(), "ExecProcess001");
      }
    }

    return true;
  }
Пример #20
0
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
    meta = (DatabaseLookupMeta) smi;
    data = (DatabaseLookupData) sdi;

    boolean sendToErrorRow = false;
    String errorMessage = null;

    Object[] r = getRow(); // Get row from input rowset & set row busy!
    if (r == null) { // no more input to be expected...
      setOutputDone();
      return false;
    }

    if (first) {
      first = false;

      // create the output metadata
      data.outputRowMeta = getInputRowMeta().clone();
      meta.getFields(data.outputRowMeta, getStepname(), null, null, this, repository, metaStore);

      if (meta.isCached()) {
        if (meta.getCacheSize() > 0) {
          data.look = new Hashtable<RowMetaAndData, TimedRow>((int) (meta.getCacheSize() * 1.5));
        } else {
          data.look = new Hashtable<RowMetaAndData, TimedRow>();
        }
      }

      data.db.setLookup(
          environmentSubstitute(meta.getSchemaName()),
          environmentSubstitute(meta.getTablename()),
          meta.getTableKeyField(),
          meta.getKeyCondition(),
          meta.getReturnValueField(),
          meta.getReturnValueNewName(),
          meta.getOrderByClause(),
          meta.isFailingOnMultipleResults());

      // lookup the values!
      if (log.isDetailed()) {
        logDetailed(
            BaseMessages.getString(PKG, "DatabaseLookup.Log.CheckingRow")
                + getInputRowMeta().getString(r));
      }

      data.keynrs = new int[meta.getStreamKeyField1().length];
      data.keynrs2 = new int[meta.getStreamKeyField1().length];

      for (int i = 0; i < meta.getStreamKeyField1().length; i++) {
        data.keynrs[i] = getInputRowMeta().indexOfValue(meta.getStreamKeyField1()[i]);
        if (data.keynrs[i] < 0
            && // couldn't find field!
            !"IS NULL".equalsIgnoreCase(meta.getKeyCondition()[i])
            && // No field needed!
            !"IS NOT NULL".equalsIgnoreCase(meta.getKeyCondition()[i]) // No field needed!
        ) {
          throw new KettleStepException(
              BaseMessages.getString(PKG, "DatabaseLookup.ERROR0001.FieldRequired1.Exception")
                  + meta.getStreamKeyField1()[i]
                  + BaseMessages.getString(
                      PKG, "DatabaseLookup.ERROR0001.FieldRequired2.Exception"));
        }
        data.keynrs2[i] = getInputRowMeta().indexOfValue(meta.getStreamKeyField2()[i]);
        if (data.keynrs2[i] < 0
            && // couldn't find field!
            "BETWEEN".equalsIgnoreCase(meta.getKeyCondition()[i]) // 2 fields needed!
        ) {
          throw new KettleStepException(
              BaseMessages.getString(PKG, "DatabaseLookup.ERROR0001.FieldRequired3.Exception")
                  + meta.getStreamKeyField2()[i]
                  + BaseMessages.getString(
                      PKG, "DatabaseLookup.ERROR0001.FieldRequired4.Exception"));
        }
        if (log.isDebug()) {
          logDebug(
              BaseMessages.getString(PKG, "DatabaseLookup.Log.FieldHasIndex1")
                  + meta.getStreamKeyField1()[i]
                  + BaseMessages.getString(PKG, "DatabaseLookup.Log.FieldHasIndex2")
                  + data.keynrs[i]);
        }
      }

      data.nullif = new Object[meta.getReturnValueField().length];

      for (int i = 0; i < meta.getReturnValueField().length; i++) {
        ValueMetaInterface stringMeta = new ValueMeta("string", ValueMetaInterface.TYPE_STRING);
        ValueMetaInterface returnMeta =
            data.outputRowMeta.getValueMeta(i + getInputRowMeta().size());

        if (!Const.isEmpty(meta.getReturnValueDefault()[i])) {
          data.nullif[i] = returnMeta.convertData(stringMeta, meta.getReturnValueDefault()[i]);
        } else {
          data.nullif[i] = null;
        }
      }

      // Determine the types...
      data.keytypes = new int[meta.getTableKeyField().length];
      String schemaTable =
          meta.getDatabaseMeta()
              .getQuotedSchemaTableCombination(
                  environmentSubstitute(meta.getSchemaName()),
                  environmentSubstitute(meta.getTablename()));
      RowMetaInterface fields = data.db.getTableFields(schemaTable);
      if (fields != null) {
        // Fill in the types...
        for (int i = 0; i < meta.getTableKeyField().length; i++) {
          ValueMetaInterface key = fields.searchValueMeta(meta.getTableKeyField()[i]);
          if (key != null) {
            data.keytypes[i] = key.getType();
          } else {
            throw new KettleStepException(
                BaseMessages.getString(PKG, "DatabaseLookup.ERROR0001.FieldRequired5.Exception")
                    + meta.getTableKeyField()[i]
                    + BaseMessages.getString(
                        PKG, "DatabaseLookup.ERROR0001.FieldRequired6.Exception"));
          }
        }
      } else {
        throw new KettleStepException(
            BaseMessages.getString(PKG, "DatabaseLookup.ERROR0002.UnableToDetermineFieldsOfTable")
                + schemaTable
                + "]");
      }

      // Count the number of values in the lookup as well as the metadata to send along with it.
      //
      data.lookupMeta = new RowMeta();

      for (int i = 0; i < meta.getStreamKeyField1().length; i++) {
        if (data.keynrs[i] >= 0) {
          ValueMetaInterface inputValueMeta = getInputRowMeta().getValueMeta(data.keynrs[i]);

          // Try to convert type if needed in a clone, we don't want to
          // change the type in the original row
          //
          ValueMetaInterface value =
              ValueMetaFactory.cloneValueMeta(inputValueMeta, data.keytypes[i]);

          data.lookupMeta.addValueMeta(value);
        }
        if (data.keynrs2[i] >= 0) {
          ValueMetaInterface inputValueMeta = getInputRowMeta().getValueMeta(data.keynrs2[i]);

          // Try to convert type if needed in a clone, we don't want to
          // change the type in the original row
          //
          ValueMetaInterface value =
              ValueMetaFactory.cloneValueMeta(inputValueMeta, data.keytypes[i]);

          data.lookupMeta.addValueMeta(value);
        }
      }

      // We also want to know the metadata of the return values beforehand (null handling)
      data.returnMeta = new RowMeta();

      for (int i = 0; i < meta.getReturnValueField().length; i++) {
        ValueMetaInterface v =
            data.outputRowMeta.getValueMeta(getInputRowMeta().size() + i).clone();
        data.returnMeta.addValueMeta(v);
      }

      // If the user selected to load all data into the cache at startup, that's what we do now...
      //
      if (meta.isCached() && meta.isLoadingAllDataInCache()) {
        loadAllTableDataIntoTheCache();
      }
    }

    if (log.isRowLevel()) {
      logRowlevel(
          BaseMessages.getString(PKG, "DatabaseLookup.Log.GotRowFromPreviousStep")
              + getInputRowMeta().getString(r));
    }

    try {
      // add new lookup values to the row
      Object[] outputRow = lookupValues(getInputRowMeta(), r);

      if (outputRow != null) {
        // copy row to output rowset(s);
        putRow(data.outputRowMeta, outputRow);

        if (log.isRowLevel()) {
          logRowlevel(
              BaseMessages.getString(PKG, "DatabaseLookup.Log.WroteRowToNextStep")
                  + getInputRowMeta().getString(r));
        }
        if (checkFeedback(getLinesRead())) {
          logBasic("linenr " + getLinesRead());
        }
      }
    } catch (KettleException e) {
      if (getStepMeta().isDoingErrorHandling()) {
        sendToErrorRow = true;
        errorMessage = e.toString();
      } else {
        logError(
            BaseMessages.getString(PKG, "DatabaseLookup.ERROR003.UnexpectedErrorDuringProcessing")
                + e.getMessage());
        setErrors(1);
        stopAll();
        setOutputDone(); // signal end to receiver(s)
        return false;
      }
      if (sendToErrorRow) {
        // Simply add this row to the error row
        putError(getInputRowMeta(), r, 1, errorMessage, null, "DBLOOKUPD001");
      }
    }

    return true;
  }
  public void check(
      List<CheckResultInterface> remarks,
      TransMeta transMeta,
      StepMeta stepMeta,
      RowMetaInterface prev,
      String input[],
      String output[],
      RowMetaInterface info) {
    CheckResult cr;
    String error_message = ""; // $NON-NLS-1$

    if (databaseMeta != null) {
      Database db = new Database(loggingObject, databaseMeta);
      db.shareVariablesWith(transMeta);
      try {
        db.connect();

        if (!Const.isEmpty(tableName)) {
          cr =
              new CheckResult(
                  CheckResultInterface.TYPE_RESULT_OK,
                  BaseMessages.getString(PKG, "GPBulkLoaderMeta.CheckResult.TableNameOK"),
                  stepMeta); //$NON-NLS-1$
          remarks.add(cr);

          boolean first = true;
          boolean error_found = false;
          error_message = ""; // $NON-NLS-1$

          // Check fields in table
          String schemaTable =
              databaseMeta.getQuotedSchemaTableCombination(
                  transMeta.environmentSubstitute(schemaName),
                  transMeta.environmentSubstitute(tableName));
          RowMetaInterface r = db.getTableFields(schemaTable);
          if (r != null) {
            cr =
                new CheckResult(
                    CheckResultInterface.TYPE_RESULT_OK,
                    BaseMessages.getString(PKG, "GPBulkLoaderMeta.CheckResult.TableExists"),
                    stepMeta); //$NON-NLS-1$
            remarks.add(cr);

            // How about the fields to insert/dateMask in the table?
            first = true;
            error_found = false;
            error_message = ""; // $NON-NLS-1$

            for (int i = 0; i < fieldTable.length; i++) {
              String field = fieldTable[i];

              ValueMetaInterface v = r.searchValueMeta(field);
              if (v == null) {
                if (first) {
                  first = false;
                  error_message +=
                      BaseMessages.getString(
                              PKG, "GPBulkLoaderMeta.CheckResult.MissingFieldsToLoadInTargetTable")
                          + Const.CR; // $NON-NLS-1$
                }
                error_found = true;
                error_message += "\t\t" + field + Const.CR; // $NON-NLS-1$
              }
            }
            if (error_found) {
              cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
            } else {
              cr =
                  new CheckResult(
                      CheckResultInterface.TYPE_RESULT_OK,
                      BaseMessages.getString(
                          PKG, "GPBulkLoaderMeta.CheckResult.AllFieldsFoundInTargetTable"),
                      stepMeta); //$NON-NLS-1$
            }
            remarks.add(cr);
          } else {
            error_message =
                BaseMessages.getString(
                    PKG, "GPBulkLoaderMeta.CheckResult.CouldNotReadTableInfo"); // $NON-NLS-1$
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
            remarks.add(cr);
          }
        }

        // Look up fields in the input stream <prev>
        if (prev != null && prev.size() > 0) {
          cr =
              new CheckResult(
                  CheckResultInterface.TYPE_RESULT_OK,
                  BaseMessages.getString(
                      PKG, "GPBulkLoaderMeta.CheckResult.StepReceivingDatas", prev.size() + ""),
                  stepMeta); //$NON-NLS-1$ //$NON-NLS-2$
          remarks.add(cr);

          boolean first = true;
          error_message = ""; // $NON-NLS-1$
          boolean error_found = false;

          for (int i = 0; i < fieldStream.length; i++) {
            ValueMetaInterface v = prev.searchValueMeta(fieldStream[i]);
            if (v == null) {
              if (first) {
                first = false;
                error_message +=
                    BaseMessages.getString(PKG, "GPBulkLoaderMeta.CheckResult.MissingFieldsInInput")
                        + Const.CR; // $NON-NLS-1$
              }
              error_found = true;
              error_message += "\t\t" + fieldStream[i] + Const.CR; // $NON-NLS-1$
            }
          }
          if (error_found) {
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
          } else {
            cr =
                new CheckResult(
                    CheckResultInterface.TYPE_RESULT_OK,
                    BaseMessages.getString(
                        PKG, "GPBulkLoaderMeta.CheckResult.AllFieldsFoundInInput"),
                    stepMeta); //$NON-NLS-1$
          }
          remarks.add(cr);
        } else {
          error_message =
              BaseMessages.getString(PKG, "GPBulkLoaderMeta.CheckResult.MissingFieldsInInput3")
                  + Const.CR; // $NON-NLS-1$
          cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
          remarks.add(cr);
        }
      } catch (KettleException e) {
        error_message =
            BaseMessages.getString(PKG, "GPBulkLoaderMeta.CheckResult.DatabaseErrorOccurred")
                + e.getMessage(); // $NON-NLS-1$
        cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
        remarks.add(cr);
      } finally {
        db.disconnect();
      }
    } else {
      error_message =
          BaseMessages.getString(
              PKG, "GPBulkLoaderMeta.CheckResult.InvalidConnection"); // $NON-NLS-1$
      cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
      remarks.add(cr);
    }

    // See if we have input streams leading to this step!
    if (input.length > 0) {
      cr =
          new CheckResult(
              CheckResultInterface.TYPE_RESULT_OK,
              BaseMessages.getString(
                  PKG, "GPBulkLoaderMeta.CheckResult.StepReceivingInfoFromOtherSteps"),
              stepMeta); //$NON-NLS-1$
      remarks.add(cr);
    } else {
      cr =
          new CheckResult(
              CheckResultInterface.TYPE_RESULT_ERROR,
              BaseMessages.getString(PKG, "GPBulkLoaderMeta.CheckResult.NoInputError"),
              stepMeta); //$NON-NLS-1$
      remarks.add(cr);
    }
  }
  public SQLStatement getSQLStatements(
      TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev) {
    SQLStatement retval =
        new SQLStatement(stepMeta.getName(), databaseWriteMeta, null); // default: nothing to do!

    int i;

    if (databaseWriteMeta != null) {
      if (prev != null && prev.size() > 0) {
        if (!Const.isEmpty(tablename)) {
          String schemaTable =
              databaseWriteMeta.getQuotedSchemaTableCombination(schemaName, tablename);
          Database db = new Database(databaseWriteMeta);
          try {
            boolean doHash = false;
            String cr_table = null;

            db.connect();

            // OK, what do we put in the new table??
            RowMetaInterface fields = new RowMeta();

            ValueMetaInterface vkeyfield = null;
            if (!Const.isEmpty(technicalKeyField)) {
              // First, the new technical key...
              vkeyfield = new ValueMeta(technicalKeyField, ValueMetaInterface.TYPE_INTEGER);
              vkeyfield.setLength(10);
              vkeyfield.setPrecision(0);
            }

            // Then the hashcode (optional)
            ValueMetaInterface vhashfield = null;
            if (useHash && !Const.isEmpty(hashField)) {
              vhashfield = new ValueMeta(hashField, ValueMetaInterface.TYPE_INTEGER);
              vhashfield.setLength(15);
              vhashfield.setPrecision(0);
              doHash = true;
            }

            // Then the last update field (optional)
            ValueMetaInterface vLastUpdateField = null;
            if (!Const.isEmpty(lastUpdateField)) {
              vLastUpdateField = new ValueMeta(lastUpdateField, ValueMetaInterface.TYPE_DATE);
            }

            if (!db.checkTableExists(schemaTable)) {
              if (vkeyfield != null) {
                // Add technical key field.
                fields.addValueMeta(vkeyfield);
              }

              // Add the keys only to the table
              if (keyField != null && keyLookup != null) {
                int cnt = keyField.length;
                for (i = 0; i < cnt; i++) {
                  String error_field = ""; // $NON-NLS-1$

                  // Find the value in the stream
                  ValueMetaInterface v = prev.searchValueMeta(keyField[i]);
                  if (v != null) {
                    String name = keyLookup[i];
                    ValueMetaInterface newValue = v.clone();
                    newValue.setName(name);

                    if (vkeyfield != null) {
                      if (name.equals(vkeyfield.getName())
                          || (doHash == true && name.equals(vhashfield.getName()))) {
                        error_field += name;
                      }
                    }
                    if (error_field.length() > 0) {
                      retval.setError(
                          Messages.getString(
                              "ConcurrentCombinationLookupMeta.ReturnValue.NameCollision",
                              error_field)); //$NON-NLS-1$
                    } else {
                      fields.addValueMeta(newValue);
                    }
                  }
                }
              }

              if (doHash == true) {
                fields.addValueMeta(vhashfield);
              }

              if (vLastUpdateField != null) {
                fields.addValueMeta(vLastUpdateField);
              }
            } else {
              // Table already exists

              // Get the fields that are in the table now:
              RowMetaInterface tabFields = db.getTableFields(schemaTable);

              // Don't forget to quote these as well...
              databaseWriteMeta.quoteReservedWords(tabFields);

              if (vkeyfield != null && tabFields.searchValueMeta(vkeyfield.getName()) == null) {
                // Add technical key field if it didn't exist yet
                fields.addValueMeta(vkeyfield);
              }

              // Add the already existing fields
              int cnt = tabFields.size();
              for (i = 0; i < cnt; i++) {
                ValueMetaInterface v = tabFields.getValueMeta(i);

                fields.addValueMeta(v);
              }

              // Find the missing fields in the real table
              String keyLookup[] = getKeyLookup();
              String keyField[] = getKeyField();
              if (keyField != null && keyLookup != null) {
                cnt = keyField.length;
                for (i = 0; i < cnt; i++) {
                  // Find the value in the stream
                  ValueMetaInterface v = prev.searchValueMeta(keyField[i]);
                  if (v != null) {
                    ValueMetaInterface newValue = v.clone();
                    newValue.setName(keyLookup[i]);

                    // Does the corresponding name exist in the table
                    if (tabFields.searchValueMeta(newValue.getName()) == null) {
                      fields.addValueMeta(newValue); // nope --> add
                    }
                  }
                }
              }

              if (doHash == true && tabFields.searchValueMeta(vhashfield.getName()) == null) {
                // Add hash field
                fields.addValueMeta(vhashfield);
              }

              if (vLastUpdateField != null
                  && tabFields.searchValueMeta(vLastUpdateField.getName()) == null) {
                fields.addValueMeta(vLastUpdateField);
              }
            }

            cr_table =
                db.getDDL(
                    schemaTable,
                    fields,
                    (CREATION_METHOD_SEQUENCE.equals(getTechKeyCreation())
                            && sequenceFrom != null
                            && sequenceFrom.length() != 0)
                        ? null
                        : technicalKeyField,
                    CREATION_METHOD_AUTOINC.equals(getTechKeyCreation()),
                    null,
                    true);

            //
            // OK, now let's build the index
            //

            // What fields do we put int the index?
            // Only the hashcode or all fields?
            String cr_index = ""; // $NON-NLS-1$
            String cr_uniq_index = ""; // $NON-NLS-1$
            String idx_fields[] = null;
            if (useHash) {
              if (hashField != null && hashField.length() > 0) {
                idx_fields = new String[] {hashField};
              } else {
                retval.setError(
                    Messages.getString(
                        "ConcurrentCombinationLookupMeta.ReturnValue.NotHashFieldSpecified")); //$NON-NLS-1$
              }
            } else // index on all key fields...
            {
              if (!Const.isEmpty(keyLookup)) {
                int nrfields = keyLookup.length;
                if (nrfields > 32
                    && databaseWriteMeta.getDatabaseType() == DatabaseMeta.TYPE_DATABASE_ORACLE) {
                  nrfields = 32; // Oracle indexes are limited to 32 fields...
                }
                idx_fields = new String[nrfields];
                for (i = 0; i < nrfields; i++) idx_fields[i] = keyLookup[i];
              } else {
                retval.setError(
                    Messages.getString(
                        "ConcurrentCombinationLookupMeta.ReturnValue.NotFieldsSpecified")); //$NON-NLS-1$
              }
            }

            // OK, now get the create index statement...

            if (!Const.isEmpty(technicalKeyField)) {
              String techKeyArr[] = new String[] {technicalKeyField};
              if (!db.checkIndexExists(schemaName, tablename, techKeyArr)) {
                String indexname = "idx_" + tablename + "_pk"; // $NON-NLS-1$ //$NON-NLS-2$
                cr_uniq_index =
                    db.getCreateIndexStatement(
                        schemaName, tablename, indexname, techKeyArr, true, true, false, true);
                cr_uniq_index += Const.CR;
              }
            }

            // OK, now get the create lookup index statement...
            if (!Const.isEmpty(idx_fields)
                && !db.checkIndexExists(schemaName, tablename, idx_fields)) {
              String indexname = "idx_" + tablename + "_lookup"; // $NON-NLS-1$ //$NON-NLS-2$
              cr_index =
                  db.getCreateIndexStatement(
                      schemaName, tablename, indexname, idx_fields, false, false, false, true);
              cr_index += Const.CR;
            }

            //
            // Don't forget the sequence (optional)
            //
            String cr_seq = ""; // $NON-NLS-1$
            if (databaseWriteMeta.supportsSequences() && !Const.isEmpty(sequenceFrom)) {
              if (!db.checkSequenceExists(schemaName, sequenceFrom)) {
                cr_seq +=
                    db.getCreateSequenceStatement(schemaName, sequenceFrom, 1L, 1L, -1L, true);
                cr_seq += Const.CR;
              }
            }
            retval.setSQL(cr_table + cr_uniq_index + cr_index + cr_seq);
          } catch (KettleException e) {
            retval.setError(
                Messages.getString("ConcurrentCombinationLookupMeta.ReturnValue.ErrorOccurred")
                    + Const.CR
                    + e.getMessage()); // $NON-NLS-1$
          }
        } else {
          retval.setError(
              Messages.getString(
                  "ConcurrentCombinationLookupMeta.ReturnValue.NotTableDefined")); //$NON-NLS-1$
        }
      } else {
        retval.setError(
            Messages.getString(
                "ConcurrentCombinationLookupMeta.ReturnValue.NotReceivingField")); //$NON-NLS-1$
      }
    } else {
      retval.setError(
          Messages.getString(
              "ConcurrentCombinationLookupMeta.ReturnValue.NotConnectionDefined")); //$NON-NLS-1$
    }

    return retval;
  }
  private void check(
      List<CheckResultInterface> remarks,
      StepMeta stepMeta,
      RowMetaInterface prev,
      String[] input,
      DatabaseMeta daMeta) {
    CheckResult cr;
    String error_message;
    if (daMeta != null) {
      Database db = new Database(daMeta);
      try {
        db.connect();

        if (!Const.isEmpty(tablename)) {
          boolean first = true;
          boolean error_found = false;
          error_message = ""; // $NON-NLS-1$

          String schemaTable = daMeta.getQuotedSchemaTableCombination(schemaName, tablename);
          RowMetaInterface r = db.getTableFields(schemaTable);
          if (r != null) {
            for (int i = 0; i < keyLookup.length; i++) {
              String lufield = keyLookup[i];

              ValueMetaInterface v = r.searchValueMeta(lufield);
              if (v == null) {
                if (first) {
                  first = false;
                  error_message +=
                      Messages.getString(
                              "ConcurrentCombinationLookupMeta.CheckResult.MissingCompareFields")
                          + Const.CR; // $NON-NLS-1$
                }
                error_found = true;
                error_message += "\t\t" + lufield + Const.CR; // $NON-NLS-1$
              }
            }
            if (error_found) {
              cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
            } else {
              cr =
                  new CheckResult(
                      CheckResultInterface.TYPE_RESULT_OK,
                      Messages.getString(
                          "ConcurrentCombinationLookupMeta.CheckResult.AllFieldsFound"),
                      stepMeta); //$NON-NLS-1$
            }
            remarks.add(cr);

            /* Also, check the fields: tk, version, from-to, ... */
            if (!Const.isEmpty(technicalKeyField) && r.indexOfValue(technicalKeyField) < 0) {
              error_message =
                  Messages.getString(
                          "ConcurrentCombinationLookupMeta.CheckResult.TechnicalKeyNotFound",
                          technicalKeyField)
                      + Const.CR; // $NON-NLS-1$ //$NON-NLS-2$
              cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
            } else {
              error_message =
                  Messages.getString(
                          "ConcurrentCombinationLookupMeta.CheckResult.TechnicalKeyFound",
                          technicalKeyField)
                      + Const.CR; // $NON-NLS-1$ //$NON-NLS-2$
              cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, error_message, stepMeta);
            }
            remarks.add(cr);
          } else {
            error_message =
                Messages.getString(
                    "ConcurrentCombinationLookupMeta.CheckResult.CouldNotReadTableInfo"); //$NON-NLS-1$
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
            remarks.add(cr);
          }
        }

        // Look up fields in the input stream <prev>
        if (prev != null && prev.size() > 0) {
          boolean first = true;
          error_message = ""; // $NON-NLS-1$
          boolean error_found = false;

          for (int i = 0; i < keyField.length; i++) {
            ValueMetaInterface v = prev.searchValueMeta(keyField[i]);
            if (v == null) {
              if (first) {
                first = false;
                error_message +=
                    Messages.getString("ConcurrentCombinationLookupMeta.CheckResult.MissingFields")
                        + Const.CR; // $NON-NLS-1$
              }
              error_found = true;
              error_message += "\t\t" + keyField[i] + Const.CR; // $NON-NLS-1$
            }
          }
          if (error_found) {
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
          } else {
            cr =
                new CheckResult(
                    CheckResultInterface.TYPE_RESULT_OK,
                    Messages.getString(
                        "ConcurrentCombinationLookupMeta.CheckResult.AllFieldsFoundInInputStream"),
                    stepMeta); //$NON-NLS-1$
          }
          remarks.add(cr);
        } else {
          error_message =
              Messages.getString("ConcurrentCombinationLookupMeta.CheckResult.CouldNotReadFields")
                  + Const.CR; // $NON-NLS-1$
          cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
          remarks.add(cr);
        }

        // Check sequence
        if (daMeta.supportsSequences() && CREATION_METHOD_SEQUENCE.equals(getTechKeyCreation())) {
          if (Const.isEmpty(sequenceFrom)) {
            error_message +=
                Messages.getString(
                        "ConcurrentCombinationLookupMeta.CheckResult.ErrorNoSequenceName")
                    + "!"; //$NON-NLS-1$ //$NON-NLS-2$
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
            remarks.add(cr);
          } else {
            // It doesn't make sense to check the sequence name
            // if it's not filled in.
            if (db.checkSequenceExists(sequenceFrom)) {
              error_message =
                  Messages.getString(
                      "ConcurrentCombinationLookupMeta.CheckResult.ReadingSequenceOK",
                      sequenceFrom); //$NON-NLS-1$ //$NON-NLS-2$
              cr = new CheckResult(CheckResultInterface.TYPE_RESULT_OK, error_message, stepMeta);
              remarks.add(cr);
            } else {
              error_message +=
                  Messages.getString(
                          "ConcurrentCombinationLookupMeta.CheckResult.ErrorReadingSequence")
                      + sequenceFrom
                      + "!"; //$NON-NLS-1$ //$NON-NLS-2$
              cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
              remarks.add(cr);
            }
          }
        }

        if (techKeyCreation != null) {
          // post 2.2 version
          if (!(CREATION_METHOD_AUTOINC.equals(techKeyCreation)
              || CREATION_METHOD_SEQUENCE.equals(techKeyCreation)
              || CREATION_METHOD_TABLEMAX.equals(techKeyCreation))) {
            error_message +=
                Messages.getString(
                        "ConcurrentCombinationLookupMeta.CheckResult.ErrorTechKeyCreation")
                    + ": "
                    + techKeyCreation
                    + "!"; //$NON-NLS-1$ //$NON-NLS-2$
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
            remarks.add(cr);
          }
        }
      } catch (KettleException e) {
        error_message =
            Messages.getString("ConcurrentCombinationLookupMeta.CheckResult.ErrorOccurred")
                + e.getMessage(); // $NON-NLS-1$
        cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
        remarks.add(cr);
      } finally {
        db.disconnect();
      }
    } else {
      error_message =
          Messages.getString(
              "ConcurrentCombinationLookupMeta.CheckResult.InvalidConnection"); //$NON-NLS-1$
      cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
      remarks.add(cr);
    }

    // See if we have input streams leading to this step!
    if (input.length > 0) {
      cr =
          new CheckResult(
              CheckResultInterface.TYPE_RESULT_OK,
              Messages.getString(
                  "ConcurrentCombinationLookupMeta.CheckResult.ReceivingInfoFromOtherSteps"),
              stepMeta); //$NON-NLS-1$
      remarks.add(cr);
    } else {
      cr =
          new CheckResult(
              CheckResultInterface.TYPE_RESULT_ERROR,
              Messages.getString("ConcurrentCombinationLookupMeta.CheckResult.NoInputReceived"),
              stepMeta); //$NON-NLS-1$
      remarks.add(cr);
    }
  }
  public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
    meta = (MySQLTableOutputMeta) smi;
    data = (MySQLTableOutputData) sdi;

    if (super.init(smi, sdi)) {
      try {
        data.databaseMeta = meta.getDatabaseMeta();

        // Batch updates are not supported on PostgreSQL (and
        // look-a-likes) together with error handling (PDI-366)
        //
        data.specialErrorHandling =
            getStepMeta().isDoingErrorHandling()
                && (meta.getDatabaseMeta().getDatabaseType() == DatabaseMeta.TYPE_DATABASE_POSTGRES
                    || meta.getDatabaseMeta().getDatabaseType()
                        == DatabaseMeta.TYPE_DATABASE_GREENPLUM);

        // get the boolean that indicates whether or not we can release
        // savepoints
        // and set in in data.
        data.releaseSavepoint = data.specialErrorHandling;

        data.commitSize = Integer.parseInt(environmentSubstitute(meta.getCommitSize()));
        data.batchSize = Integer.parseInt(environmentSubstitute(meta.getBatchSize()));
        data.batchMode = data.commitSize > 0 && meta.useBatchUpdate();

        // Batch updates are not supported in case we are running with
        // transactions in the transformation.
        // It is also disabled when we return keys...
        //
        data.specialErrorHandling |= getTransMeta().isUsingUniqueConnections();

        if (data.batchMode && data.specialErrorHandling) {
          data.batchMode = false;
          if (log.isBasic()) logBasic(Messages.getString("TableOutput.Log.BatchModeDisabled"));
        }

        if (meta.getDatabaseMeta() == null) {
          throw new KettleException(
              Messages.getString("TableOutput.Exception.DatabaseNeedsToBeSelected"));
        }
        if (meta.getDatabaseMeta() == null) {
          logError(Messages.getString("TableOutput.Init.ConnectionMissing", getStepname()));
          return false;
        }
        data.db = new Database(meta.getDatabaseMeta());
        data.db.shareVariablesWith(this);

        if (getTransMeta().isUsingUniqueConnections()) {
          synchronized (getTrans()) {
            data.db.connect(getTrans().getThreadName(), getPartitionID());
          }
        } else {
          data.db.connect(getPartitionID());
        }

        if (log.isBasic())
          logBasic(
              "Connected to database ["
                  + meta.getDatabaseMeta()
                  + "] (commit="
                  + data.commitSize
                  + ")");

        // Postpone commit as long as possible. PDI-2091
        //
        if (data.commitSize == 0) {
          data.commitSize = Integer.MAX_VALUE;
        }
        data.db.setCommit(data.commitSize);

        if (!meta.isPartitioningEnabled() && !meta.isTableNameInField()) {
          data.tableName = environmentSubstitute(meta.getTablename());

          // Only the first one truncates in a non-partitioned step
          // copy
          //
          if (meta.truncateTable()
              && ((getCopy() == 0 && getUniqueStepNrAcrossSlaves() == 0)
                  || !Const.isEmpty(getPartitionID()))) {
            data.db.truncateTable(
                environmentSubstitute(meta.getSchemaName()),
                environmentSubstitute(meta.getTablename()));
          }
        }

        return true;
      } catch (KettleException e) {
        logError("An error occurred intialising this step: " + e.getMessage());
        stopAll();
        setErrors(1);
      }
    }
    return false;
  }
  public void assertRoundGeneral(
      final Object expectedResult,
      final int calcFunction,
      final Number value,
      final Long precision,
      final Long roundingMode,
      final int valueDataType,
      final int functionDataType)
      throws KettleException {

    final String msg =
        getKettleTypeName(valueDataType) + "->" + getKettleTypeName(functionDataType) + " ";

    final RowMeta inputRowMeta = new RowMeta();
    final List<Object> inputValues = new ArrayList<Object>(3);

    final String fieldValue = "Value";
    final ValueMetaInterface valueMeta;
    switch (valueDataType) {
      case ValueMetaInterface.TYPE_BIGNUMBER:
        valueMeta = new ValueMetaBigNumber(fieldValue);
        break;
      case ValueMetaInterface.TYPE_NUMBER:
        valueMeta = new ValueMetaNumber(fieldValue);
        break;
      case ValueMetaInterface.TYPE_INTEGER:
        valueMeta = new ValueMetaInteger(fieldValue);
        break;
      default:
        throw new IllegalArgumentException(
            msg
                + "Unexpected value dataType: "
                + value.getClass().getName()
                + ". Long, Double or BigDecimal expected.");
    }
    inputRowMeta.addValueMeta(valueMeta);
    inputValues.add(value);

    final String fieldPrecision;
    final ValueMetaInteger precisionMeta;
    if (precision == null) {
      fieldPrecision = null;
      precisionMeta = null;
    } else {
      fieldPrecision = "Precision";
      precisionMeta = new ValueMetaInteger(fieldPrecision);
      inputRowMeta.addValueMeta(precisionMeta);
      inputValues.add(precision);
    }

    final String fieldRoundingMode;
    final ValueMetaInteger roundingModeMeta;
    if (roundingMode == null) {
      fieldRoundingMode = null;
      roundingModeMeta = null;
    } else {
      fieldRoundingMode = "RoundingMode";
      roundingModeMeta = new ValueMetaInteger(fieldRoundingMode);
      inputRowMeta.addValueMeta(roundingModeMeta);
      inputValues.add(roundingMode);
    }

    RowSet inputRowSet = smh.getMockInputRowSet(inputValues.toArray());
    inputRowSet.setRowMeta(inputRowMeta);
    final String fieldA =
        inputRowMeta.size() > 0 ? inputRowMeta.getValueMetaList().get(0).getName() : null;
    final String fieldB =
        inputRowMeta.size() > 1 ? inputRowMeta.getValueMetaList().get(1).getName() : null;
    final String fieldC =
        inputRowMeta.size() > 2 ? inputRowMeta.getValueMetaList().get(2).getName() : null;

    final int resultDataType = functionDataType;

    final String fieldResult = "test";
    final int expectedResultRowSize = inputRowMeta.size() + 1;

    Calculator calculator =
        new Calculator(smh.stepMeta, smh.stepDataInterface, 0, smh.transMeta, smh.trans);
    calculator.getInputRowSets().add(inputRowSet);
    calculator.setInputRowMeta(inputRowMeta);
    calculator.init(smh.initStepMetaInterface, smh.initStepDataInterface);

    CalculatorMeta meta = new CalculatorMeta();
    meta.setCalculation(
        new CalculatorMetaFunction[] {
          new CalculatorMetaFunction(
              fieldResult,
              calcFunction,
              fieldA,
              fieldB,
              fieldC,
              resultDataType,
              2,
              0,
              false,
              "",
              "",
              "",
              "")
        });

    // Verify output
    try {
      calculator.addRowListener(
          new RowAdapter() {
            @Override
            public void rowWrittenEvent(RowMetaInterface rowMeta, Object[] row)
                throws KettleStepException {
              assertEquals(msg + " resultRowSize", expectedResultRowSize, rowMeta.size());
              final int fieldResultIndex = rowMeta.size() - 1;
              assertEquals(
                  msg + " fieldResult",
                  fieldResult,
                  rowMeta.getValueMeta(fieldResultIndex).getName());
              assertEquals(msg, expectedResult, row[fieldResultIndex]);
            }
          });
      calculator.processRow(meta, new CalculatorData());
    } catch (KettleException ke) {
      ke.printStackTrace();
      fail(msg + ke.getMessage());
    }
  }
  public void check(
      List<CheckResultInterface> remarks,
      TransMeta transMeta,
      StepMeta stepMeta,
      RowMetaInterface prev,
      String[] input,
      String[] output,
      RowMetaInterface info,
      VariableSpace space,
      Repository repository,
      IMetaStore metaStore) {
    CheckResult cr;

    if (databaseMeta != null) {
      cr =
          new CheckResult(
              CheckResult.TYPE_RESULT_OK,
              BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.ConnectionExists"),
              stepMeta);
      remarks.add(cr);

      Database db = new Database(loggingObject, databaseMeta);
      databases = new Database[] {db}; // keep track of it for cancelling purposes...

      try {
        db.connect();
        cr =
            new CheckResult(
                CheckResult.TYPE_RESULT_OK,
                BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.DBConnectionOK"),
                stepMeta);
        remarks.add(cr);

        if (sqlField != null && sqlField.length() != 0) {
          cr =
              new CheckResult(
                  CheckResult.TYPE_RESULT_OK,
                  BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.SQLFieldNameEntered"),
                  stepMeta);
        } else {
          cr =
              new CheckResult(
                  CheckResult.TYPE_RESULT_ERROR,
                  BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.SQLFieldNameMissing"),
                  stepMeta);
        }
        remarks.add(cr);
      } catch (KettleException e) {
        cr =
            new CheckResult(
                CheckResult.TYPE_RESULT_ERROR,
                BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.ErrorOccurred")
                    + e.getMessage(),
                stepMeta);
        remarks.add(cr);
      } finally {
        db.disconnect();
      }
    } else {
      cr =
          new CheckResult(
              CheckResult.TYPE_RESULT_ERROR,
              BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.ConnectionNeeded"),
              stepMeta);
      remarks.add(cr);
    }

    if (input.length > 0) {
      cr =
          new CheckResult(
              CheckResult.TYPE_RESULT_OK,
              BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.StepReceivingInfoOK"),
              stepMeta);
      remarks.add(cr);
    } else {
      cr =
          new CheckResult(
              CheckResult.TYPE_RESULT_ERROR,
              BaseMessages.getString(PKG, "ExecSQLRowMeta.CheckResult.NoInputReceivedError"),
              stepMeta);
      remarks.add(cr);
    }
  }
Пример #27
0
  public Result execute(Result result, int nr) throws KettleException {
    FileLoggingEventListener loggingEventListener = null;
    LogLevel shellLogLevel = parentJob.getLogLevel();
    if (setLogfile) {
      String realLogFilename = environmentSubstitute(getLogFilename());
      // We need to check here the log filename
      // if we do not have one, we must fail
      if (Const.isEmpty(realLogFilename)) {
        logError(BaseMessages.getString(PKG, "JobEntryShell.Exception.LogFilenameMissing"));
        result.setNrErrors(1);
        result.setResult(false);
        return result;
      }

      try {
        loggingEventListener =
            new FileLoggingEventListener(getLogChannelId(), realLogFilename, setAppendLogfile);
        KettleLogStore.getAppender().addLoggingEventListener(loggingEventListener);
      } catch (KettleException e) {
        logError(
            BaseMessages.getString(
                PKG, "JobEntryShell.Error.UnableopenAppenderFile", getLogFilename(), e.toString()));
        logError(Const.getStackTracker(e));
        result.setNrErrors(1);
        result.setResult(false);
        return result;
      }
      shellLogLevel = logFileLevel;
    }

    log.setLogLevel(shellLogLevel);

    result.setEntryNr(nr);

    // "Translate" the arguments for later
    String[] substArgs = null;
    if (arguments != null) {
      substArgs = new String[arguments.length];
      for (int idx = 0; idx < arguments.length; idx++) {
        substArgs[idx] = environmentSubstitute(arguments[idx]);
      }
    }

    int iteration = 0;
    String[] args = substArgs;
    RowMetaAndData resultRow = null;
    boolean first = true;
    List<RowMetaAndData> rows = result.getRows();

    if (log.isDetailed()) {
      logDetailed(
          BaseMessages.getString(
              PKG, "JobEntryShell.Log.FoundPreviousRows", "" + (rows != null ? rows.size() : 0)));
    }

    while ((first && !execPerRow)
        || (execPerRow && rows != null && iteration < rows.size() && result.getNrErrors() == 0)) {
      first = false;
      if (rows != null && execPerRow) {
        resultRow = rows.get(iteration);
      } else {
        resultRow = null;
      }

      List<RowMetaAndData> cmdRows = null;

      if (execPerRow) {
        // Execute for each input row

        if (argFromPrevious) {
          // Copy the input row to the (command line) arguments

          if (resultRow != null) {
            args = new String[resultRow.size()];
            for (int i = 0; i < resultRow.size(); i++) {
              args[i] = resultRow.getString(i, null);
            }
          }
        } else {
          // Just pass a single row
          List<RowMetaAndData> newList = new ArrayList<RowMetaAndData>();
          newList.add(resultRow);
          cmdRows = newList;
        }
      } else {
        if (argFromPrevious) {
          // Only put the first Row on the arguments
          args = null;
          if (resultRow != null) {
            args = new String[resultRow.size()];
            for (int i = 0; i < resultRow.size(); i++) {
              args[i] = resultRow.getString(i, null);
            }
          } else {
            cmdRows = rows;
          }
        } else {
          // Keep it as it was...
          cmdRows = rows;
        }
      }

      executeShell(result, cmdRows, args);

      iteration++;
    }

    if (setLogfile) {
      if (loggingEventListener != null) {
        KettleLogStore.getAppender().removeLoggingEventListener(loggingEventListener);
        loggingEventListener.close();

        ResultFile resultFile =
            new ResultFile(
                ResultFile.FILE_TYPE_LOG,
                loggingEventListener.getFile(),
                parentJob.getJobname(),
                getName());
        result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
      }
    }

    return result;
  }
Пример #28
0
  public void check(
      List<CheckResultInterface> remarks,
      TransMeta transMeta,
      StepMeta stepMeta,
      RowMetaInterface prev,
      String[] input,
      String[] output,
      RowMetaInterface info,
      VariableSpace space,
      Repository repository,
      IMetaStore metaStore) {
    CheckResult cr;
    String error_message = "";

    if (databaseMeta != null) {
      Database db = new Database(loggingObject, databaseMeta);
      db.shareVariablesWith(transMeta);
      try {
        db.connect();

        if (!Const.isEmpty(tableName)) {
          cr =
              new CheckResult(
                  CheckResultInterface.TYPE_RESULT_OK,
                  BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.TableNameOK"),
                  stepMeta);
          remarks.add(cr);

          boolean first = true;
          boolean error_found = false;
          error_message = "";

          // Check fields in table
          String schemaTable = databaseMeta.getQuotedSchemaTableCombination(schemaName, tableName);
          RowMetaInterface r = db.getTableFields(schemaTable);
          if (r != null) {
            cr =
                new CheckResult(
                    CheckResultInterface.TYPE_RESULT_OK,
                    BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.TableExists"),
                    stepMeta);
            remarks.add(cr);

            for (int i = 0; i < keyLookup.length; i++) {
              String lufield = keyLookup[i];

              ValueMetaInterface v = r.searchValueMeta(lufield);
              if (v == null) {
                if (first) {
                  first = false;
                  error_message +=
                      BaseMessages.getString(
                              PKG, "InsertUpdateMeta.CheckResult.MissingCompareFieldsInTargetTable")
                          + Const.CR;
                }
                error_found = true;
                error_message += "\t\t" + lufield + Const.CR;
              }
            }
            if (error_found) {
              cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
            } else {
              cr =
                  new CheckResult(
                      CheckResultInterface.TYPE_RESULT_OK,
                      BaseMessages.getString(
                          PKG, "InsertUpdateMeta.CheckResult.AllLookupFieldsFound"),
                      stepMeta);
            }
            remarks.add(cr);

            // How about the fields to insert/update in the table?
            first = true;
            error_found = false;
            error_message = "";

            for (int i = 0; i < updateLookup.length; i++) {
              String lufield = updateLookup[i];

              ValueMetaInterface v = r.searchValueMeta(lufield);
              if (v == null) {
                if (first) {
                  first = false;
                  error_message +=
                      BaseMessages.getString(
                              PKG,
                              "InsertUpdateMeta.CheckResult.MissingFieldsToUpdateInTargetTable")
                          + Const.CR;
                }
                error_found = true;
                error_message += "\t\t" + lufield + Const.CR;
              }
            }
            if (error_found) {
              cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
            } else {
              cr =
                  new CheckResult(
                      CheckResultInterface.TYPE_RESULT_OK,
                      BaseMessages.getString(
                          PKG, "InsertUpdateMeta.CheckResult.AllFieldsToUpdateFoundInTargetTable"),
                      stepMeta);
            }
            remarks.add(cr);
          } else {
            error_message =
                BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.CouldNotReadTableInfo");
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
            remarks.add(cr);
          }
        }

        // Look up fields in the input stream <prev>
        if (prev != null && prev.size() > 0) {
          cr =
              new CheckResult(
                  CheckResultInterface.TYPE_RESULT_OK,
                  BaseMessages.getString(
                      PKG, "InsertUpdateMeta.CheckResult.StepReceivingDatas", prev.size() + ""),
                  stepMeta);
          remarks.add(cr);

          boolean first = true;
          error_message = "";
          boolean error_found = false;

          for (int i = 0; i < keyStream.length; i++) {
            ValueMetaInterface v = prev.searchValueMeta(keyStream[i]);
            if (v == null) {
              if (first) {
                first = false;
                error_message +=
                    BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.MissingFieldsInInput")
                        + Const.CR;
              }
              error_found = true;
              error_message += "\t\t" + keyStream[i] + Const.CR;
            }
          }
          for (int i = 0; i < keyStream2.length; i++) {
            if (keyStream2[i] != null && keyStream2[i].length() > 0) {
              ValueMetaInterface v = prev.searchValueMeta(keyStream2[i]);
              if (v == null) {
                if (first) {
                  first = false;
                  error_message +=
                      BaseMessages.getString(
                              PKG, "InsertUpdateMeta.CheckResult.MissingFieldsInInput")
                          + Const.CR;
                }
                error_found = true;
                error_message += "\t\t" + keyStream[i] + Const.CR;
              }
            }
          }
          if (error_found) {
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
          } else {
            cr =
                new CheckResult(
                    CheckResultInterface.TYPE_RESULT_OK,
                    BaseMessages.getString(
                        PKG, "InsertUpdateMeta.CheckResult.AllFieldsFoundInInput"),
                    stepMeta);
          }
          remarks.add(cr);

          // How about the fields to insert/update the table with?
          first = true;
          error_found = false;
          error_message = "";

          for (int i = 0; i < updateStream.length; i++) {
            String lufield = updateStream[i];

            ValueMetaInterface v = prev.searchValueMeta(lufield);
            if (v == null) {
              if (first) {
                first = false;
                error_message +=
                    BaseMessages.getString(
                            PKG, "InsertUpdateMeta.CheckResult.MissingInputStreamFields")
                        + Const.CR;
              }
              error_found = true;
              error_message += "\t\t" + lufield + Const.CR;
            }
          }
          if (error_found) {
            cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
          } else {
            cr =
                new CheckResult(
                    CheckResultInterface.TYPE_RESULT_OK,
                    BaseMessages.getString(
                        PKG, "InsertUpdateMeta.CheckResult.AllFieldsFoundInInput2"),
                    stepMeta);
          }
          remarks.add(cr);
        } else {
          error_message =
              BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.MissingFieldsInInput3")
                  + Const.CR;
          cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
          remarks.add(cr);
        }
      } catch (KettleException e) {
        error_message =
            BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.DatabaseErrorOccurred")
                + e.getMessage();
        cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
        remarks.add(cr);
      } finally {
        db.disconnect();
      }
    } else {
      error_message = BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.InvalidConnection");
      cr = new CheckResult(CheckResultInterface.TYPE_RESULT_ERROR, error_message, stepMeta);
      remarks.add(cr);
    }

    // See if we have input streams leading to this step!
    if (input.length > 0) {
      cr =
          new CheckResult(
              CheckResultInterface.TYPE_RESULT_OK,
              BaseMessages.getString(
                  PKG, "InsertUpdateMeta.CheckResult.StepReceivingInfoFromOtherSteps"),
              stepMeta);
      remarks.add(cr);
    } else {
      cr =
          new CheckResult(
              CheckResultInterface.TYPE_RESULT_ERROR,
              BaseMessages.getString(PKG, "InsertUpdateMeta.CheckResult.NoInputError"),
              stepMeta);
      remarks.add(cr);
    }
  }
  public synchronized BlockingRowSet openReaderSocket(final BaseStep baseStep)
      throws IOException, KettleException {
    this.baseStep = baseStep;

    final BlockingRowSet rowSet = new BlockingRowSet(baseStep.getTransMeta().getSizeRowset());

    // Make sure we handle the case with multiple step copies running on a
    // slave...
    //
    rowSet.setThreadNameFromToCopy(sourceStep, sourceStepCopyNr, targetStep, targetStepCopyNr);
    rowSet.setRemoteSlaveServerName(targetSlaveServerName);

    final int portNumber = Integer.parseInt(baseStep.environmentSubstitute(port));
    final String realHostname = baseStep.environmentSubstitute(hostname);

    // Connect to the server socket (started during BaseStep.init())
    // Because the accept() call on the server socket can be called after we
    // reached this code
    // it is best to build in a retry loop with a time-out here.
    //
    long startTime = System.currentTimeMillis();
    boolean connected = false;
    KettleException lastException = null;

    // // timeout with retry until connected
    while (!connected
        && (TIMEOUT_IN_SECONDS > (System.currentTimeMillis() - startTime) / 1000)
        && !baseStep.isStopped()) {
      try {
        socket = new Socket();
        socket.setReuseAddress(true);

        baseStep.logDetailed(
            "Step variable MASTER_HOST : [" + baseStep.getVariable("MASTER_HOST") + "]");
        baseStep.logDetailed(
            "Opening client (reader) socket to server ["
                + Const.NVL(realHostname, "")
                + ":"
                + port
                + "]");
        socket.connect(new InetSocketAddress(realHostname, portNumber), 5000);

        connected = true;

        if (compressingStreams) {
          gzipInputStream = new GZIPInputStream(socket.getInputStream());
          bufferedInputStream = new BufferedInputStream(gzipInputStream, bufferSize);
        } else {
          bufferedInputStream = new BufferedInputStream(socket.getInputStream(), bufferSize);
        }
        inputStream = new DataInputStream(bufferedInputStream);

        lastException = null;
      } catch (Exception e) {
        lastException =
            new KettleException(
                "Unable to open socket to server " + realHostname + " port " + portNumber, e);
      }
      if (lastException != null) {
        // Sleep for a while
        try {
          Thread.sleep(250);
        } catch (InterruptedException e) {
          if (socket != null) {
            socket.shutdownInput();
            socket.shutdownOutput();
            socket.close();
            baseStep.logDetailed(
                "Closed connection to server socket to read rows from remote step on server "
                    + realHostname
                    + " port "
                    + portNumber
                    + " - Local port="
                    + socket.getLocalPort());
          }

          throw new KettleException(
              "Interrupted while trying to connect to server socket: " + e.toString());
        }
      }
    }

    // See if all was OK...
    if (lastException != null) {

      baseStep.logError("Error initialising step: " + lastException.toString());
      if (socket != null) {
        socket.shutdownInput();
        socket.shutdownOutput();
        socket.close();
        baseStep.logDetailed(
            "Closed connection to server socket to read rows from remote step on server "
                + realHostname
                + " port "
                + portNumber
                + " - Local port="
                + socket.getLocalPort());
      }
      throw lastException;
    } else {
      if (inputStream == null)
        throw new KettleException(
            "Unable to connect to the SocketWriter in the "
                + TIMEOUT_IN_SECONDS
                + "s timeout period.");
    }

    baseStep.logDetailed(
        "Opened connection to server socket to read rows from remote step on server "
            + realHostname
            + " port "
            + portNumber
            + " - Local port="
            + socket.getLocalPort());

    // Create a thread to take care of the reading from the client socket.
    // The rows read will be put in a RowSet buffer.
    // That buffer will hand over the rows to the step that has this RemoteStep
    // object defined
    // as a remote input step.
    //
    Runnable runnable =
        new Runnable() {
          public void run() {
            try {

              // First read the row meta data from the socket...
              //
              RowMetaInterface rowMeta = null;
              while (!baseStep.isStopped() && rowMeta == null) {
                try {
                  rowMeta = new RowMeta(inputStream);
                } catch (SocketTimeoutException e) {
                  rowMeta = null;
                }
              }

              if (rowMeta == null) {
                throw new KettleEOFException(); // leave now.
              }

              // And a first row of data...
              //
              Object[] rowData = getRowOfData(rowMeta);

              // Now get the data itself, row by row...
              //
              while (rowData != null && !baseStep.isStopped()) {
                baseStep.incrementLinesInput();
                baseStep.decrementLinesRead();

                if (baseStep.log.isDebug())
                  baseStep.logDebug("Received row from remote step: " + rowMeta.getString(rowData));

                baseStep.putRowTo(rowMeta, rowData, rowSet);
                baseStep.decrementLinesWritten();
                rowData = getRowOfData(rowMeta);
              }
            } catch (KettleEOFException e) {
              // Nothing, we're simply done reading...
              //
              if (baseStep.log.isDebug())
                baseStep.logDebug(
                    "Finished reading from remote step on server "
                        + hostname
                        + " port "
                        + portNumber);

            } catch (Exception e) {
              baseStep.logError("Error reading from client socket to remote step", e);
              baseStep.setErrors(1);
              baseStep.stopAll();
            } finally {
              // Close the input socket
              if (socket != null && !socket.isClosed() && !socket.isInputShutdown()) {
                try {
                  socket.shutdownInput();
                } catch (Exception e) {
                  baseStep.logError(
                      "Error shutting down input channel on client socket connection to remote step",
                      e);
                }
              }
              if (socket != null && !socket.isClosed() && !socket.isOutputShutdown()) {
                try {
                  socket.shutdownOutput();
                } catch (Exception e) {
                  baseStep.logError(
                      "Error shutting down output channel on client socket connection to remote step",
                      e);
                }
              }
              if (socket != null && !socket.isClosed()) {
                try {
                  socket.close();
                } catch (Exception e) {
                  baseStep.logError(
                      "Error shutting down client socket connection to remote step", e);
                }
              }
              if (inputStream != null) {
                try {
                  inputStream.close();
                } catch (Exception e) {
                  baseStep.logError(
                      "Error closing input stream on socket connection to remote step", e);
                }
                inputStream = null;
              }
              if (bufferedInputStream != null) {
                try {
                  bufferedInputStream.close();
                } catch (Exception e) {
                  baseStep.logError(
                      "Error closing input stream on socket connection to remote step", e);
                }
              }
              bufferedInputStream = null;
              if (gzipInputStream != null) {
                try {
                  gzipInputStream.close();
                } catch (Exception e) {
                  baseStep.logError(
                      "Error closing input stream on socket connection to remote step", e);
                }
              }
              gzipInputStream = null;
              baseStep.logDetailed(
                  "Closed connection to server socket to read rows from remote step on server "
                      + realHostname
                      + " port "
                      + portNumber
                      + " - Local port="
                      + socket.getLocalPort());
            }

            // signal baseStep that nothing else comes from this step.
            //
            rowSet.setDone();
          }
        };
    new Thread(runnable).start();

    return rowSet;
  }
  public Result execute(Result previousResult, int nr) {
    Result result = previousResult;
    result.setResult(false);

    // see PDI-10270, PDI-10644 for details
    boolean oldBehavior =
        "Y"
            .equalsIgnoreCase(
                getVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "N"));

    String countSQLStatement = null;
    long rowsCount = 0;
    long errCount = 0;

    boolean successOK = false;

    int nrRowsLimit = Const.toInt(environmentSubstitute(limit), 0);
    if (log.isDetailed()) {
      logDetailed(
          BaseMessages.getString(
              PKG, "JobEntryEvalTableContent.Log.nrRowsLimit", "" + nrRowsLimit));
    }

    if (connection != null) {
      Database db = new Database(this, connection);
      db.shareVariablesWith(this);
      try {
        db.connect(parentJob.getTransactionId(), null);

        if (iscustomSQL) {
          String realCustomSQL = customSQL;
          if (isUseVars) {
            realCustomSQL = environmentSubstitute(realCustomSQL);
          }
          if (log.isDebug()) {
            logDebug(
                BaseMessages.getString(
                    PKG, "JobEntryEvalTableContent.Log.EnteredCustomSQL", realCustomSQL));
          }

          if (!Const.isEmpty(realCustomSQL)) {
            countSQLStatement = realCustomSQL;
          } else {
            errCount++;
            logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.Error.NoCustomSQL"));
          }

        } else {
          String realTablename = environmentSubstitute(tablename);
          String realSchemaname = environmentSubstitute(schemaname);

          if (!Const.isEmpty(realTablename)) {
            if (!Const.isEmpty(realSchemaname)) {
              countSQLStatement =
                  selectCount
                      + db.getDatabaseMeta()
                          .getQuotedSchemaTableCombination(realSchemaname, realTablename);
            } else {
              countSQLStatement = selectCount + db.getDatabaseMeta().quoteField(realTablename);
            }
          } else {
            errCount++;
            logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.Error.NoTableName"));
          }
        }

        if (countSQLStatement != null) {
          if (log.isDetailed()) {
            logDetailed(
                BaseMessages.getString(
                    PKG, "JobEntryEvalTableContent.Log.RunSQLStatement", countSQLStatement));
          }

          if (iscustomSQL) {
            if (isClearResultList) {
              result.getRows().clear();
            }

            List<Object[]> ar = db.getRows(countSQLStatement, 0);
            if (ar != null) {
              rowsCount = ar.size();

              // ad rows to result
              RowMetaInterface rowMeta = db.getQueryFields(countSQLStatement, false);

              List<RowMetaAndData> rows = new ArrayList<RowMetaAndData>();
              for (int i = 0; i < ar.size(); i++) {
                rows.add(new RowMetaAndData(rowMeta, ar.get(i)));
              }
              if (isAddRowsResult && iscustomSQL) {
                if (rows != null) {
                  result.getRows().addAll(rows);
                }
              }
            } else {
              if (log.isDebug()) {
                logDebug(
                    BaseMessages.getString(
                        PKG,
                        "JobEntryEvalTableContent.Log.customSQLreturnedNothing",
                        countSQLStatement));
              }
            }

          } else {
            RowMetaAndData row = db.getOneRow(countSQLStatement);
            if (row != null) {
              rowsCount = row.getInteger(0);
            }
          }
          if (log.isDetailed()) {
            logDetailed(
                BaseMessages.getString(
                    PKG, "JobEntryEvalTableContent.Log.NrRowsReturned", "" + rowsCount));
          }
          switch (successCondition) {
            case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL:
              successOK = (rowsCount == nrRowsLimit);
              break;
            case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_DIFFERENT:
              successOK = (rowsCount != nrRowsLimit);
              break;
            case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_SMALLER:
              successOK = (rowsCount < nrRowsLimit);
              break;
            case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_SMALLER_EQUAL:
              successOK = (rowsCount <= nrRowsLimit);
              break;
            case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_GREATER:
              successOK = (rowsCount > nrRowsLimit);
              break;
            case JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_GREATER_EQUAL:
              successOK = (rowsCount >= nrRowsLimit);
              break;
            default:
              break;
          }

          if (!successOK && oldBehavior) {
            errCount++;
          }
        } // end if countSQLStatement!=null
      } catch (KettleException dbe) {
        errCount++;
        logError(
            BaseMessages.getString(
                PKG, "JobEntryEvalTableContent.Error.RunningEntry", dbe.getMessage()));
      } finally {
        if (db != null) {
          db.disconnect();
        }
      }
    } else {
      errCount++;
      logError(BaseMessages.getString(PKG, "JobEntryEvalTableContent.NoDbConnection"));
    }

    result.setResult(successOK);
    result.setNrLinesRead(rowsCount);
    result.setNrErrors(errCount);

    return result;
  }