protected void doBatch() throws KettleException {
    WriteConcern concern = null;

    if (log.getLogLevel().getLevel() >= LogLevel.DETAILED.getLevel()) {
      concern = new WriteConcern(1);
    }
    WriteResult result = null;

    if (concern != null) {
      result = m_data.getCollection().insert(m_batch, concern);
    } else {
      result = m_data.getCollection().insert(m_batch);
    }

    CommandResult cmd = result.getLastError();

    if (cmd != null && !cmd.ok()) {
      String message = cmd.getErrorMessage();
      logError(BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.MongoReported", message));
      try {
        cmd.throwOnError();
      } catch (MongoException me) {
        throw new KettleException(me.getMessage(), me);
      }
    }

    m_batch.clear();
  }
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {

    Object[] row = getRow();

    if (row == null) {
      // no more output

      // check any remaining buffered objects
      if (m_batch != null && m_batch.size() > 0) {
        doBatch();
      }

      // INDEXING - http://www.mongodb.org/display/DOCS/Indexes
      // Indexing is computationally expensive - it needs to be
      // done after all data is inserted and done in the BACKGROUND.

      // UNIQUE indexes (prevent duplicates on the
      // keys in the index) and SPARSE indexes (don't index docs that
      // don't have the key field) - current limitation is that SPARSE
      // indexes can only have a single field

      List<MongoDbOutputMeta.MongoIndex> indexes = m_meta.getMongoIndexes();
      if (indexes != null && indexes.size() > 0) {
        logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.ApplyingIndexOpps"));
        m_data.applyIndexes(indexes, log, m_meta.getTruncate());
      }

      disconnect();
      setOutputDone();
      return false;
    }

    if (first) {
      first = false;

      m_batchInsertSize = 100;

      String batchInsert = environmentSubstitute(m_meta.getBatchInsertSize());
      if (!Const.isEmpty(batchInsert)) {
        m_batchInsertSize = Integer.parseInt(batchInsert);
      }
      m_batch = new ArrayList<DBObject>(m_batchInsertSize);

      // output the same as the input
      m_data.setOutputRowMeta(getInputRowMeta());

      m_mongoTopLevelStructure =
          MongoDbOutputData.checkTopLevelConsistency(m_meta.m_mongoFields, this);
      if (m_mongoTopLevelStructure == MongoDbOutputData.MongoTopLevel.INCONSISTENT) {
        throw new KettleException(
            BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.InconsistentMongoTopLevel"));
      }

      // first check our incoming fields against our meta data for fields to insert
      RowMetaInterface rmi = getInputRowMeta();
      List<MongoDbOutputMeta.MongoField> mongoFields = m_meta.getMongoFields();
      List<String> notToBeInserted = new ArrayList<String>();
      for (int i = 0; i < rmi.size(); i++) {
        ValueMetaInterface vm = rmi.getValueMeta(i);
        boolean ok = false;
        for (MongoDbOutputMeta.MongoField field : mongoFields) {
          String mongoMatch = environmentSubstitute(field.m_incomingFieldName);
          if (vm.getName().equals(mongoMatch)) {
            ok = true;
            break;
          }
        }

        if (!ok) {
          notToBeInserted.add(vm.getName());
        }
      }

      if (notToBeInserted.size() == rmi.size()) {
        throw new KettleException(
            BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.NotInsertingAnyFields"));
      }

      if (notToBeInserted.size() > 0) {
        StringBuffer b = new StringBuffer();
        for (String s : notToBeInserted) {
          b.append(s).append(" ");
        }

        logBasic(
            BaseMessages.getString(PKG, "MongoDbOutput.Messages.FieldsNotToBeInserted"),
            b.toString());
      }

      // init mongo fields
      for (MongoDbOutputMeta.MongoField m : m_meta.getMongoFields()) {
        m.init(this);
      }

      // check truncate
      if (m_meta.getTruncate()) {
        try {
          logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.TruncatingCollection"));
          m_data.getCollection().drop();

          // re-establish the collection
          String collection = environmentSubstitute(m_meta.getCollection());
          m_data.createCollection(collection);
          m_data.setCollection(m_data.getDB().getCollection(collection));
        } catch (Exception m) {
          disconnect();
          throw new KettleException(m.getMessage(), m);
        }
      }
    }

    if (!isStopped()) {

      if (m_meta.getUpsert()) {
        /*DBObject updateQuery = MongoDbOutputData.getQueryObject(m_meta.getMongoFields(),
        getInputRowMeta(), row, getParentVariableSpace(), m_mongoTopLevelStructure); */
        DBObject updateQuery =
            MongoDbOutputData.getQueryObject(m_meta.getMongoFields(), getInputRowMeta(), row, this);

        if (log.isDebug()) {
          logDebug(
              BaseMessages.getString(
                  PKG, "MongoDbOutput.Messages.Debug.QueryForUpsert", updateQuery));
        }

        if (updateQuery != null) {
          // i.e. we have some non-null incoming query field values
          DBObject insertUpdate = null;

          // get the record to update the match with
          if (!m_meta.getModifierUpdate()) {
            // complete record replace or insert

            insertUpdate =
                MongoDbOutputData.kettleRowToMongo(
                    m_meta.getMongoFields(),
                    getInputRowMeta(),
                    row,
                    this,
                    m_mongoTopLevelStructure);
          } else {
            // specific field update or insert
            insertUpdate =
                MongoDbOutputData.getModifierUpdateObject(
                    m_meta.getMongoFields(),
                    getInputRowMeta(),
                    row,
                    this,
                    m_mongoTopLevelStructure);
            if (log.isDebug()) {
              logDebug(
                  BaseMessages.getString(
                      PKG, "MongoDbOutput.Messages.Debug.ModifierUpdateObject", insertUpdate));
            }
          }

          if (insertUpdate != null) {
            WriteConcern concern = null;

            if (log.getLogLevel().getLevel() >= LogLevel.DETAILED.getLevel()) {
              concern = new WriteConcern(1);
            }
            WriteResult result = null;
            if (concern != null) {
              result =
                  m_data
                      .getCollection()
                      .update(updateQuery, insertUpdate, true, m_meta.getMulti(), concern);
            } else {
              result =
                  m_data.getCollection().update(updateQuery, insertUpdate, true, m_meta.getMulti());
            }

            CommandResult cmd = result.getLastError();
            if (cmd != null && !cmd.ok()) {
              String message = cmd.getErrorMessage();
              logError(
                  BaseMessages.getString(
                      PKG, "MongoDbOutput.Messages.Error.MongoReported", message));
              try {
                cmd.throwOnError();
              } catch (MongoException me) {
                throw new KettleException(me.getMessage(), me);
              }
            }
          }
        }
      } else {
        // straight insert

        DBObject mongoInsert =
            MongoDbOutputData.kettleRowToMongo(
                m_meta.getMongoFields(), getInputRowMeta(), row, this, m_mongoTopLevelStructure);

        if (mongoInsert != null) {
          m_batch.add(mongoInsert);
        }
        if (m_batch.size() == m_batchInsertSize) {
          logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.CommitingABatch"));
          doBatch();
        }
      }
    }

    return true;
  }