コード例 #1
0
  protected void doBatch() throws KettleException {
    WriteConcern concern = null;

    if (log.getLogLevel().getLevel() >= LogLevel.DETAILED.getLevel()) {
      concern = new WriteConcern(1);
    }
    WriteResult result = null;

    if (concern != null) {
      result = m_data.getCollection().insert(m_batch, concern);
    } else {
      result = m_data.getCollection().insert(m_batch);
    }

    CommandResult cmd = result.getLastError();

    if (cmd != null && !cmd.ok()) {
      String message = cmd.getErrorMessage();
      logError(BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.MongoReported", message));
      try {
        cmd.throwOnError();
      } catch (MongoException me) {
        throw new KettleException(me.getMessage(), me);
      }
    }

    m_batch.clear();
  }
コード例 #2
0
 protected void throwOnError(WriteConcern wc, WriteResult wr) {
   if (wc == null && wr.getLastConcern() == null) {
     CommandResult cr = wr.getLastError();
     if (cr != null && cr.getErrorMessage() != null && cr.getErrorMessage().length() > 0)
       cr.throwOnError();
   }
 }
コード例 #3
0
  @Override // method
  public Role2 add(Role2 roleBaru) {
    final DBObject dbObject = new EntityToDBObject().apply(roleBaru);
    final WriteResult result = rolePersonColl.insert(dbObject);

    log.info("Role '{}' has been added: {}", roleBaru, result.getLastError());
    return roleBaru;
  }
コード例 #4
0
 /** Delete a {<span class="referer">@link</span> Task} for a particular id. */
 public boolean deleteObjectById(String id) {
   WriteResult result = mongoTemplate.remove(new Query(Criteria.where("uid").is(id)), Task.class);
   result.getLastError();
   if (null != result) {
     if (result.getN() > 0) {
       return true;
     }
   }
   return false;
 }
コード例 #5
0
 @Override
 public boolean deleteObject(Task object) {
   WriteResult result = mongoTemplate.remove(object);
   result.getLastError();
   if (null != result) {
     if (result.getN() > 0) {
       return true;
     }
   }
   return false;
 }
コード例 #6
0
  @Test
  @Ignore("Really slow on the delete, not a good unit tests atm")
  public void deleteBatchTest() throws Exception {

    DB db = getDb();

    int count = (int) (OpDelete.BATCH_SIZE * 1.5);

    List<DBObject> docs = new ArrayList<DBObject>(count);

    for (int i = 0; i < count; i++) {
      BasicDBObject doc = new BasicDBObject();

      doc.put("index", i);

      docs.add(doc);
    }

    WriteResult result = db.getCollection("deletebatchtests").insert(docs);

    assertNull(result.getLastError().getErrorMessage());

    // iterate over all the data to make sure it's been inserted

    DBCursor cursor = db.getCollection("deletebatchtests").find();

    for (int i = 0; i < count && cursor.hasNext(); i++) {
      int index = new BasicDBObject(cursor.next().toMap()).getInt("index");

      assertEquals(i, index);
    }

    BasicDBObject query = new BasicDBObject();
    query.put("index", new BasicDBObject("$lte", count));

    // now delete the objects
    db.getCollection("deletebatchtests").remove(query, WriteConcern.SAFE);

    // now  try and iterate, there should be no results
    cursor = db.getCollection("deletebatchtests").find();

    assertFalse(cursor.hasNext());

    // check it has been deleted
    UUID appId = emf.lookupApplication("test-organization/test-app");
    EntityManager em = emf.getEntityManager(appId);

    Results results =
        em.searchCollection(
            new SimpleEntityRef("application", appId), "deletebatchtests", new Query());

    assertEquals(0, results.size());
  }
コード例 #7
0
ファイル: FongoTest.java プロジェクト: jumarko/fongo
 @Test
 public void testUpsertExisting() {
   DBCollection collection = newCollection();
   collection.insert(new BasicDBObject("_id", 1));
   WriteResult result =
       collection.update(
           new BasicDBObject("_id", 1),
           new BasicDBObject("$inc", new BasicDBObject("a", 1)),
           true,
           false);
   assertEquals(new BasicDBObject("_id", 1).append("a", 1), collection.findOne());
   assertTrue(result.getLastError().getBoolean("updatedExisting"));
 }
コード例 #8
0
  private void processAndTransferWriteResult(WriteResult result, Exchange exchange) {
    // if invokeGetLastError is set, or a WriteConcern is set which implicitly calls getLastError,
    // then we have the chance to populate
    // the MONGODB_LAST_ERROR header, as well as setting an exception on the Exchange if one
    // occurred at the MongoDB server
    if (endpoint.isInvokeGetLastError()
        || (endpoint.getWriteConcern() != null
            ? endpoint.getWriteConcern().callGetLastError()
            : false)) {
      CommandResult cr =
          result.getCachedLastError() == null ? result.getLastError() : result.getCachedLastError();
      exchange.getOut().setHeader(MongoDbConstants.LAST_ERROR, cr);
      if (!cr.ok()) {
        exchange.setException(MongoDbComponent.wrapInCamelMongoDbException(cr.getException()));
      }
    }

    // determine where to set the WriteResult: as the OUT body or as an IN message header
    if (endpoint.isWriteResultAsHeader()) {
      exchange.getOut().setHeader(MongoDbConstants.WRITERESULT, result);
    } else {
      exchange.getOut().setBody(result);
    }
  }
コード例 #9
0
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {

    Object[] row = getRow();

    if (row == null) {
      // no more output

      // check any remaining buffered objects
      if (m_batch != null && m_batch.size() > 0) {
        doBatch();
      }

      // INDEXING - http://www.mongodb.org/display/DOCS/Indexes
      // Indexing is computationally expensive - it needs to be
      // done after all data is inserted and done in the BACKGROUND.

      // UNIQUE indexes (prevent duplicates on the
      // keys in the index) and SPARSE indexes (don't index docs that
      // don't have the key field) - current limitation is that SPARSE
      // indexes can only have a single field

      List<MongoDbOutputMeta.MongoIndex> indexes = m_meta.getMongoIndexes();
      if (indexes != null && indexes.size() > 0) {
        logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.ApplyingIndexOpps"));
        m_data.applyIndexes(indexes, log, m_meta.getTruncate());
      }

      disconnect();
      setOutputDone();
      return false;
    }

    if (first) {
      first = false;

      m_batchInsertSize = 100;

      String batchInsert = environmentSubstitute(m_meta.getBatchInsertSize());
      if (!Const.isEmpty(batchInsert)) {
        m_batchInsertSize = Integer.parseInt(batchInsert);
      }
      m_batch = new ArrayList<DBObject>(m_batchInsertSize);

      // output the same as the input
      m_data.setOutputRowMeta(getInputRowMeta());

      m_mongoTopLevelStructure =
          MongoDbOutputData.checkTopLevelConsistency(m_meta.m_mongoFields, this);
      if (m_mongoTopLevelStructure == MongoDbOutputData.MongoTopLevel.INCONSISTENT) {
        throw new KettleException(
            BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.InconsistentMongoTopLevel"));
      }

      // first check our incoming fields against our meta data for fields to insert
      RowMetaInterface rmi = getInputRowMeta();
      List<MongoDbOutputMeta.MongoField> mongoFields = m_meta.getMongoFields();
      List<String> notToBeInserted = new ArrayList<String>();
      for (int i = 0; i < rmi.size(); i++) {
        ValueMetaInterface vm = rmi.getValueMeta(i);
        boolean ok = false;
        for (MongoDbOutputMeta.MongoField field : mongoFields) {
          String mongoMatch = environmentSubstitute(field.m_incomingFieldName);
          if (vm.getName().equals(mongoMatch)) {
            ok = true;
            break;
          }
        }

        if (!ok) {
          notToBeInserted.add(vm.getName());
        }
      }

      if (notToBeInserted.size() == rmi.size()) {
        throw new KettleException(
            BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.NotInsertingAnyFields"));
      }

      if (notToBeInserted.size() > 0) {
        StringBuffer b = new StringBuffer();
        for (String s : notToBeInserted) {
          b.append(s).append(" ");
        }

        logBasic(
            BaseMessages.getString(PKG, "MongoDbOutput.Messages.FieldsNotToBeInserted"),
            b.toString());
      }

      // init mongo fields
      for (MongoDbOutputMeta.MongoField m : m_meta.getMongoFields()) {
        m.init(this);
      }

      // check truncate
      if (m_meta.getTruncate()) {
        try {
          logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.TruncatingCollection"));
          m_data.getCollection().drop();

          // re-establish the collection
          String collection = environmentSubstitute(m_meta.getCollection());
          m_data.createCollection(collection);
          m_data.setCollection(m_data.getDB().getCollection(collection));
        } catch (Exception m) {
          disconnect();
          throw new KettleException(m.getMessage(), m);
        }
      }
    }

    if (!isStopped()) {

      if (m_meta.getUpsert()) {
        /*DBObject updateQuery = MongoDbOutputData.getQueryObject(m_meta.getMongoFields(),
        getInputRowMeta(), row, getParentVariableSpace(), m_mongoTopLevelStructure); */
        DBObject updateQuery =
            MongoDbOutputData.getQueryObject(m_meta.getMongoFields(), getInputRowMeta(), row, this);

        if (log.isDebug()) {
          logDebug(
              BaseMessages.getString(
                  PKG, "MongoDbOutput.Messages.Debug.QueryForUpsert", updateQuery));
        }

        if (updateQuery != null) {
          // i.e. we have some non-null incoming query field values
          DBObject insertUpdate = null;

          // get the record to update the match with
          if (!m_meta.getModifierUpdate()) {
            // complete record replace or insert

            insertUpdate =
                MongoDbOutputData.kettleRowToMongo(
                    m_meta.getMongoFields(),
                    getInputRowMeta(),
                    row,
                    this,
                    m_mongoTopLevelStructure);
          } else {
            // specific field update or insert
            insertUpdate =
                MongoDbOutputData.getModifierUpdateObject(
                    m_meta.getMongoFields(),
                    getInputRowMeta(),
                    row,
                    this,
                    m_mongoTopLevelStructure);
            if (log.isDebug()) {
              logDebug(
                  BaseMessages.getString(
                      PKG, "MongoDbOutput.Messages.Debug.ModifierUpdateObject", insertUpdate));
            }
          }

          if (insertUpdate != null) {
            WriteConcern concern = null;

            if (log.getLogLevel().getLevel() >= LogLevel.DETAILED.getLevel()) {
              concern = new WriteConcern(1);
            }
            WriteResult result = null;
            if (concern != null) {
              result =
                  m_data
                      .getCollection()
                      .update(updateQuery, insertUpdate, true, m_meta.getMulti(), concern);
            } else {
              result =
                  m_data.getCollection().update(updateQuery, insertUpdate, true, m_meta.getMulti());
            }

            CommandResult cmd = result.getLastError();
            if (cmd != null && !cmd.ok()) {
              String message = cmd.getErrorMessage();
              logError(
                  BaseMessages.getString(
                      PKG, "MongoDbOutput.Messages.Error.MongoReported", message));
              try {
                cmd.throwOnError();
              } catch (MongoException me) {
                throw new KettleException(me.getMessage(), me);
              }
            }
          }
        }
      } else {
        // straight insert

        DBObject mongoInsert =
            MongoDbOutputData.kettleRowToMongo(
                m_meta.getMongoFields(), getInputRowMeta(), row, this, m_mongoTopLevelStructure);

        if (mongoInsert != null) {
          m_batch.add(mongoInsert);
        }
        if (m_batch.size() == m_batchInsertSize) {
          logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.CommitingABatch"));
          doBatch();
        }
      }
    }

    return true;
  }