Ejemplo n.º 1
0
 private Timestamp<?> getCurrentOplogTimestamp() {
   return Timestamp.on(
       oplogCollection
           .find()
           .sort(new BasicDBObject(MongoDBRiver.INSERTION_ORDER_KEY, -1))
           .limit(1)
           .next());
 }
Ejemplo n.º 2
0
 private void isRiverStale(DBCursor cursor, Timestamp<?> time) throws SlurperException {
   if (cursor == null || time == null) {
     return;
   }
   if (definition.getInitialTimestamp() != null && time.equals(definition.getInitialTimestamp())) {
     return;
   }
   DBObject entry = cursor.next();
   Timestamp<?> oplogTimestamp = Timestamp.on(entry);
   if (!time.equals(oplogTimestamp)) {
     MongoDBRiverHelper.setRiverStatus(client, definition.getRiverName(), Status.RIVER_STALE);
     throw new SlurperException("River out of sync with oplog.rs collection");
   }
 }
Ejemplo n.º 3
0
  private boolean isValidOplogEntry(final DBObject entry, final Timestamp<?> startTimestamp) {
    if (MongoDBRiver.OPLOG_NOOP_OPERATION.equals(entry.get(MongoDBRiver.OPLOG_OPERATION))) {
      logger.debug("[No-op Oplog Entry] - can be ignored. {}", entry);
      return false;
    }
    String namespace = (String) entry.get(MongoDBRiver.OPLOG_NAMESPACE);
    // Initial support for sharded collection -
    // https://jira.mongodb.org/browse/SERVER-4333
    // Not interested in operation from migration or sharding
    if (entry.containsField(MongoDBRiver.OPLOG_FROM_MIGRATE)
        && ((BasicBSONObject) entry).getBoolean(MongoDBRiver.OPLOG_FROM_MIGRATE)) {
      logger.debug(
          "[Invalid Oplog Entry] - from migration or sharding operation. Can be ignored. {}",
          entry);
      return false;
    }
    // Not interested by chunks - skip all
    if (namespace.endsWith(MongoDBRiver.GRIDFS_CHUNKS_SUFFIX)) {
      return false;
    }

    if (startTimestamp != null) {
      Timestamp<?> oplogTimestamp = Timestamp.on(entry);
      if (Timestamp.compare(oplogTimestamp, startTimestamp) < 0) {
        logger.debug(
            "[Invalid Oplog Entry] - entry timestamp [{}] before startTimestamp [{}]",
            entry,
            startTimestamp);
        return false;
      }
    }

    boolean validNamespace = false;
    if (definition.isMongoGridFS()) {
      validNamespace = gridfsOplogNamespace.equals(namespace);
    } else {
      if (definition.isImportAllCollections()) {
        // Skip temp entry generated by map / reduce
        if (namespace.startsWith(definition.getMongoDb())
            && !namespace.startsWith(definition.getMongoDb() + ".tmp.mr")) {
          validNamespace = true;
        }
      } else {
        if (definition.getMongoOplogNamespace().equals(namespace)) {
          validNamespace = true;
        }
      }
      if (cmdOplogNamespace.equals(namespace)) {
        validNamespace = true;
      }

      if (MongoDBRiver.OPLOG_ADMIN_COMMAND.equals(namespace)) {
        validNamespace = true;
      }
    }
    if (!validNamespace) {
      logger.debug("[Invalid Oplog Entry] - namespace [{}] is not valid", namespace);
      return false;
    }
    String operation = (String) entry.get(MongoDBRiver.OPLOG_OPERATION);
    if (!oplogOperations.contains(operation)) {
      logger.debug("[Invalid Oplog Entry] - operation [{}] is not valid", operation);
      return false;
    }

    // TODO: implement a better solution
    if (definition.getMongoOplogFilter() != null) {
      DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT);
      BasicDBObject filter = definition.getMongoOplogFilter();
      if (!filterMatch(filter, object)) {
        logger.debug(
            "[Invalid Oplog Entry] - filter [{}] does not match object [{}]", filter, object);
        return false;
      }
    }
    return true;
  }
Ejemplo n.º 4
0
  private Timestamp<?> processOplogEntry(final DBObject entry, final Timestamp<?> startTimestamp)
      throws InterruptedException {
    // To support transactions, TokuMX wraps one or more operations in a single oplog entry, in a
    // list.
    // As long as clients are not transaction-aware, we can pretty safely assume there will only be
    // one operation in the list.
    // Supporting genuine multi-operation transactions will require a bit more logic here.
    flattenOps(entry);

    if (!isValidOplogEntry(entry, startTimestamp)) {
      return startTimestamp;
    }
    Operation operation = Operation.fromString(entry.get(MongoDBRiver.OPLOG_OPERATION).toString());
    String namespace = entry.get(MongoDBRiver.OPLOG_NAMESPACE).toString();
    String collection = null;
    Timestamp<?> oplogTimestamp = Timestamp.on(entry);
    DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT);

    if (definition.isImportAllCollections()) {
      if (namespace.startsWith(definition.getMongoDb()) && !namespace.equals(cmdOplogNamespace)) {
        collection = getCollectionFromNamespace(namespace);
      }
    } else {
      collection = definition.getMongoCollection();
    }

    if (namespace.equals(cmdOplogNamespace)) {
      if (object.containsField(MongoDBRiver.OPLOG_DROP_COMMAND_OPERATION)) {
        operation = Operation.DROP_COLLECTION;
        if (definition.isImportAllCollections()) {
          collection = object.get(MongoDBRiver.OPLOG_DROP_COMMAND_OPERATION).toString();
          if (collection.startsWith("tmp.mr.")) {
            return startTimestamp;
          }
        }
      }
      if (object.containsField(MongoDBRiver.OPLOG_DROP_DATABASE_COMMAND_OPERATION)) {
        operation = Operation.DROP_DATABASE;
      }
    }

    logger.trace("namespace: {} - operation: {}", namespace, operation);
    if (namespace.equals(MongoDBRiver.OPLOG_ADMIN_COMMAND)) {
      if (operation == Operation.COMMAND) {
        processAdminCommandOplogEntry(entry, startTimestamp);
        return startTimestamp;
      }
    }

    if (logger.isTraceEnabled()) {
      logger.trace("MongoDB object deserialized: {}", object.toString());
      logger.trace("collection: {}", collection);
      logger.trace("oplog entry - namespace [{}], operation [{}]", namespace, operation);
      logger.trace("oplog processing item {}", entry);
    }

    String objectId = getObjectIdFromOplogEntry(entry);
    if (operation == Operation.DELETE) {
      // Include only _id in data, as vanilla MongoDB does, so transformation scripts won't be
      // broken by Toku
      if (object.containsField(MongoDBRiver.MONGODB_ID_FIELD)) {
        if (object.keySet().size() > 1) {
          entry.put(
              MongoDBRiver.OPLOG_OBJECT,
              object = new BasicDBObject(MongoDBRiver.MONGODB_ID_FIELD, objectId));
        }
      } else {
        throw new NullPointerException(MongoDBRiver.MONGODB_ID_FIELD);
      }
    }

    if (definition.isMongoGridFS()
        && namespace.endsWith(MongoDBRiver.GRIDFS_FILES_SUFFIX)
        && (operation == Operation.INSERT || operation == Operation.UPDATE)) {
      if (objectId == null) {
        throw new NullPointerException(MongoDBRiver.MONGODB_ID_FIELD);
      }
      GridFS grid = new GridFS(mongo.getDB(definition.getMongoDb()), collection);
      GridFSDBFile file = grid.findOne(new ObjectId(objectId));
      if (file != null) {
        logger.info("Caught file: {} - {}", file.getId(), file.getFilename());
        object = file;
      } else {
        logger.warn("Cannot find file from id: {}", objectId);
      }
    }

    if (object instanceof GridFSDBFile) {
      if (objectId == null) {
        throw new NullPointerException(MongoDBRiver.MONGODB_ID_FIELD);
      }
      if (logger.isTraceEnabled()) {
        logger.trace("Add attachment: {}", objectId);
      }
      addToStream(operation, oplogTimestamp, applyFieldFilter(object), collection);
    } else {
      if (operation == Operation.UPDATE) {
        DBObject update = (DBObject) entry.get(MongoDBRiver.OPLOG_UPDATE);
        logger.debug("Updated item: {}", update);
        addQueryToStream(operation, oplogTimestamp, update, collection);
      } else {
        if (operation == Operation.INSERT) {
          addInsertToStream(oplogTimestamp, applyFieldFilter(object), collection);
        } else {
          addToStream(operation, oplogTimestamp, applyFieldFilter(object), collection);
        }
      }
    }
    return oplogTimestamp;
  }