public Slurper( List<ServerAddress> mongoServers, MongoDBRiverDefinition definition, SharedContext context, Client client) { this.definition = definition; this.context = context; this.client = client; this.mongo = new MongoClient(mongoServers, definition.getMongoClientOptions()); this.findKeys = new BasicDBObject(); this.gridfsOplogNamespace = definition.getMongoOplogNamespace() + MongoDBRiver.GRIDFS_FILES_SUFFIX; this.cmdOplogNamespace = definition.getMongoDb() + "." + MongoDBRiver.OPLOG_NAMESPACE_COMMAND; if (definition.getExcludeFields() != null) { for (String key : definition.getExcludeFields()) { findKeys.put(key, 0); } } else if (definition.getIncludeFields() != null) { for (String key : definition.getIncludeFields()) { findKeys.put(key, 1); } } }
private boolean isValidOplogEntry(final DBObject entry, final Timestamp<?> startTimestamp) { if (MongoDBRiver.OPLOG_NOOP_OPERATION.equals(entry.get(MongoDBRiver.OPLOG_OPERATION))) { logger.debug("[No-op Oplog Entry] - can be ignored. {}", entry); return false; } String namespace = (String) entry.get(MongoDBRiver.OPLOG_NAMESPACE); // Initial support for sharded collection - // https://jira.mongodb.org/browse/SERVER-4333 // Not interested in operation from migration or sharding if (entry.containsField(MongoDBRiver.OPLOG_FROM_MIGRATE) && ((BasicBSONObject) entry).getBoolean(MongoDBRiver.OPLOG_FROM_MIGRATE)) { logger.debug( "[Invalid Oplog Entry] - from migration or sharding operation. Can be ignored. {}", entry); return false; } // Not interested by chunks - skip all if (namespace.endsWith(MongoDBRiver.GRIDFS_CHUNKS_SUFFIX)) { return false; } if (startTimestamp != null) { Timestamp<?> oplogTimestamp = Timestamp.on(entry); if (Timestamp.compare(oplogTimestamp, startTimestamp) < 0) { logger.debug( "[Invalid Oplog Entry] - entry timestamp [{}] before startTimestamp [{}]", entry, startTimestamp); return false; } } boolean validNamespace = false; if (definition.isMongoGridFS()) { validNamespace = gridfsOplogNamespace.equals(namespace); } else { if (definition.isImportAllCollections()) { // Skip temp entry generated by map / reduce if (namespace.startsWith(definition.getMongoDb()) && !namespace.startsWith(definition.getMongoDb() + ".tmp.mr")) { validNamespace = true; } } else { if (definition.getMongoOplogNamespace().equals(namespace)) { validNamespace = true; } } if (cmdOplogNamespace.equals(namespace)) { validNamespace = true; } if (MongoDBRiver.OPLOG_ADMIN_COMMAND.equals(namespace)) { validNamespace = true; } } if (!validNamespace) { logger.debug("[Invalid Oplog Entry] - namespace [{}] is not valid", namespace); return false; } String operation = (String) entry.get(MongoDBRiver.OPLOG_OPERATION); if (!oplogOperations.contains(operation)) { logger.debug("[Invalid Oplog Entry] - operation [{}] is not valid", operation); return false; } // TODO: implement a better solution if (definition.getMongoOplogFilter() != null) { DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT); BasicDBObject filter = definition.getMongoOplogFilter(); if (!filterMatch(filter, object)) { logger.debug( "[Invalid Oplog Entry] - filter [{}] does not match object [{}]", filter, object); return false; } } return true; }