private Timestamp<?> getCurrentOplogTimestamp() { return Timestamp.on( oplogCollection .find() .sort(new BasicDBObject(MongoDBRiver.INSERTION_ORDER_KEY, -1)) .limit(1) .next()); }
private void isRiverStale(DBCursor cursor, Timestamp<?> time) throws SlurperException { if (cursor == null || time == null) { return; } if (definition.getInitialTimestamp() != null && time.equals(definition.getInitialTimestamp())) { return; } DBObject entry = cursor.next(); Timestamp<?> oplogTimestamp = Timestamp.on(entry); if (!time.equals(oplogTimestamp)) { MongoDBRiverHelper.setRiverStatus(client, definition.getRiverName(), Status.RIVER_STALE); throw new SlurperException("River out of sync with oplog.rs collection"); } }
private boolean isValidOplogEntry(final DBObject entry, final Timestamp<?> startTimestamp) { if (MongoDBRiver.OPLOG_NOOP_OPERATION.equals(entry.get(MongoDBRiver.OPLOG_OPERATION))) { logger.debug("[No-op Oplog Entry] - can be ignored. {}", entry); return false; } String namespace = (String) entry.get(MongoDBRiver.OPLOG_NAMESPACE); // Initial support for sharded collection - // https://jira.mongodb.org/browse/SERVER-4333 // Not interested in operation from migration or sharding if (entry.containsField(MongoDBRiver.OPLOG_FROM_MIGRATE) && ((BasicBSONObject) entry).getBoolean(MongoDBRiver.OPLOG_FROM_MIGRATE)) { logger.debug( "[Invalid Oplog Entry] - from migration or sharding operation. Can be ignored. {}", entry); return false; } // Not interested by chunks - skip all if (namespace.endsWith(MongoDBRiver.GRIDFS_CHUNKS_SUFFIX)) { return false; } if (startTimestamp != null) { Timestamp<?> oplogTimestamp = Timestamp.on(entry); if (Timestamp.compare(oplogTimestamp, startTimestamp) < 0) { logger.debug( "[Invalid Oplog Entry] - entry timestamp [{}] before startTimestamp [{}]", entry, startTimestamp); return false; } } boolean validNamespace = false; if (definition.isMongoGridFS()) { validNamespace = gridfsOplogNamespace.equals(namespace); } else { if (definition.isImportAllCollections()) { // Skip temp entry generated by map / reduce if (namespace.startsWith(definition.getMongoDb()) && !namespace.startsWith(definition.getMongoDb() + ".tmp.mr")) { validNamespace = true; } } else { if (definition.getMongoOplogNamespace().equals(namespace)) { validNamespace = true; } } if (cmdOplogNamespace.equals(namespace)) { validNamespace = true; } if (MongoDBRiver.OPLOG_ADMIN_COMMAND.equals(namespace)) { validNamespace = true; } } if (!validNamespace) { logger.debug("[Invalid Oplog Entry] - namespace [{}] is not valid", namespace); return false; } String operation = (String) entry.get(MongoDBRiver.OPLOG_OPERATION); if (!oplogOperations.contains(operation)) { logger.debug("[Invalid Oplog Entry] - operation [{}] is not valid", operation); return false; } // TODO: implement a better solution if (definition.getMongoOplogFilter() != null) { DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT); BasicDBObject filter = definition.getMongoOplogFilter(); if (!filterMatch(filter, object)) { logger.debug( "[Invalid Oplog Entry] - filter [{}] does not match object [{}]", filter, object); return false; } } return true; }
private Timestamp<?> processOplogEntry(final DBObject entry, final Timestamp<?> startTimestamp) throws InterruptedException { // To support transactions, TokuMX wraps one or more operations in a single oplog entry, in a // list. // As long as clients are not transaction-aware, we can pretty safely assume there will only be // one operation in the list. // Supporting genuine multi-operation transactions will require a bit more logic here. flattenOps(entry); if (!isValidOplogEntry(entry, startTimestamp)) { return startTimestamp; } Operation operation = Operation.fromString(entry.get(MongoDBRiver.OPLOG_OPERATION).toString()); String namespace = entry.get(MongoDBRiver.OPLOG_NAMESPACE).toString(); String collection = null; Timestamp<?> oplogTimestamp = Timestamp.on(entry); DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT); if (definition.isImportAllCollections()) { if (namespace.startsWith(definition.getMongoDb()) && !namespace.equals(cmdOplogNamespace)) { collection = getCollectionFromNamespace(namespace); } } else { collection = definition.getMongoCollection(); } if (namespace.equals(cmdOplogNamespace)) { if (object.containsField(MongoDBRiver.OPLOG_DROP_COMMAND_OPERATION)) { operation = Operation.DROP_COLLECTION; if (definition.isImportAllCollections()) { collection = object.get(MongoDBRiver.OPLOG_DROP_COMMAND_OPERATION).toString(); if (collection.startsWith("tmp.mr.")) { return startTimestamp; } } } if (object.containsField(MongoDBRiver.OPLOG_DROP_DATABASE_COMMAND_OPERATION)) { operation = Operation.DROP_DATABASE; } } logger.trace("namespace: {} - operation: {}", namespace, operation); if (namespace.equals(MongoDBRiver.OPLOG_ADMIN_COMMAND)) { if (operation == Operation.COMMAND) { processAdminCommandOplogEntry(entry, startTimestamp); return startTimestamp; } } if (logger.isTraceEnabled()) { logger.trace("MongoDB object deserialized: {}", object.toString()); logger.trace("collection: {}", collection); logger.trace("oplog entry - namespace [{}], operation [{}]", namespace, operation); logger.trace("oplog processing item {}", entry); } String objectId = getObjectIdFromOplogEntry(entry); if (operation == Operation.DELETE) { // Include only _id in data, as vanilla MongoDB does, so transformation scripts won't be // broken by Toku if (object.containsField(MongoDBRiver.MONGODB_ID_FIELD)) { if (object.keySet().size() > 1) { entry.put( MongoDBRiver.OPLOG_OBJECT, object = new BasicDBObject(MongoDBRiver.MONGODB_ID_FIELD, objectId)); } } else { throw new NullPointerException(MongoDBRiver.MONGODB_ID_FIELD); } } if (definition.isMongoGridFS() && namespace.endsWith(MongoDBRiver.GRIDFS_FILES_SUFFIX) && (operation == Operation.INSERT || operation == Operation.UPDATE)) { if (objectId == null) { throw new NullPointerException(MongoDBRiver.MONGODB_ID_FIELD); } GridFS grid = new GridFS(mongo.getDB(definition.getMongoDb()), collection); GridFSDBFile file = grid.findOne(new ObjectId(objectId)); if (file != null) { logger.info("Caught file: {} - {}", file.getId(), file.getFilename()); object = file; } else { logger.warn("Cannot find file from id: {}", objectId); } } if (object instanceof GridFSDBFile) { if (objectId == null) { throw new NullPointerException(MongoDBRiver.MONGODB_ID_FIELD); } if (logger.isTraceEnabled()) { logger.trace("Add attachment: {}", objectId); } addToStream(operation, oplogTimestamp, applyFieldFilter(object), collection); } else { if (operation == Operation.UPDATE) { DBObject update = (DBObject) entry.get(MongoDBRiver.OPLOG_UPDATE); logger.debug("Updated item: {}", update); addQueryToStream(operation, oplogTimestamp, update, collection); } else { if (operation == Operation.INSERT) { addInsertToStream(oplogTimestamp, applyFieldFilter(object), collection); } else { addToStream(operation, oplogTimestamp, applyFieldFilter(object), collection); } } } return oplogTimestamp; }