DBCursor testCursor() {
    DBCollection collection = mock(DBCollection.class);
    when(collection.getFullName()).thenReturn(collectionName);
    DBObject keysObject = new BasicDBObject("key", "value");
    DBObject queryObject = new BasicDBObject("query", "value");

    DBCursor cursor = new DBCursorDummy(collection, queryObject, keysObject);

    return cursor;
  }
Ejemplo n.º 2
0
  /**
   * Does an initial sync the same way MongoDB does. https://groups.google.com/
   * forum/?fromgroups=#!topic/mongodb-user/sOKlhD_E2ns
   *
   * @return the last oplog timestamp before the import began
   * @throws InterruptedException if the blocking queue stream is interrupted while waiting
   */
  protected Timestamp<?> doInitialImport(DBCollection collection) throws InterruptedException {
    // TODO: ensure the index type is empty
    // DBCollection slurpedCollection =
    // slurpedDb.getCollection(definition.getMongoCollection());

    logger.info("MongoDBRiver is beginning initial import of " + collection.getFullName());
    Timestamp<?> startTimestamp = getCurrentOplogTimestamp();
    boolean inProgress = true;
    String lastId = null;
    while (inProgress) {
      DBCursor cursor = null;
      try {
        if (definition.isDisableIndexRefresh()) {
          updateIndexRefresh(definition.getIndexName(), -1L);
        }
        if (!definition.isMongoGridFS()) {
          logger.info("Collection {} - count: {}", collection.getName(), collection.count());
          long count = 0;
          cursor =
              collection.find(
                  getFilterForInitialImport(definition.getMongoCollectionFilter(), lastId));
          while (cursor.hasNext()) {
            DBObject object = cursor.next();
            count++;
            if (cursor.hasNext()) {
              lastId = addInsertToStream(null, applyFieldFilter(object), collection.getName());
            } else {
              logger.debug("Last entry for initial import - add timestamp: {}", startTimestamp);
              lastId =
                  addInsertToStream(startTimestamp, applyFieldFilter(object), collection.getName());
            }
          }
          inProgress = false;
          logger.info("Number documents indexed: {}", count);
        } else {
          // TODO: To be optimized.
          // https://github.com/mongodb/mongo-java-driver/pull/48#issuecomment-25241988
          // possible option: Get the object id list from .fs
          // collection
          // then call GriDFS.findOne
          GridFS grid =
              new GridFS(mongo.getDB(definition.getMongoDb()), definition.getMongoCollection());

          cursor = grid.getFileList();
          while (cursor.hasNext()) {
            DBObject object = cursor.next();
            if (object instanceof GridFSDBFile) {
              GridFSDBFile file =
                  grid.findOne(new ObjectId(object.get(MongoDBRiver.MONGODB_ID_FIELD).toString()));
              if (cursor.hasNext()) {
                lastId = addInsertToStream(null, file);
              } else {
                logger.debug("Last entry for initial import - add timestamp: {}", startTimestamp);
                lastId = addInsertToStream(startTimestamp, file);
              }
            }
          }
          inProgress = false;
        }
      } catch (MongoException.CursorNotFound e) {
        logger.info(
            "Initial import - Cursor {} has been closed. About to open a new cusor.",
            cursor.getCursorId());
        logger.debug("Total document inserted [{}]", totalDocuments.get());
      } finally {
        if (cursor != null) {
          logger.trace("Closing initial import cursor");
          cursor.close();
        }
        if (definition.isDisableIndexRefresh()) {
          updateIndexRefresh(definition.getIndexName(), TimeValue.timeValueSeconds(1));
        }
      }
    }
    return startTimestamp;
  }