public Slurper(
List<ServerAddress> mongoServers,
MongoDBRiverDefinition definition,
SharedContext context,
Client client) {
this.definition = definition;
this.context = context;
this.client = client;
this.mongo = new MongoClient(mongoServers, definition.getMongoClientOptions());
this.findKeys = new BasicDBObject();
this.gridfsOplogNamespace =
definition.getMongoOplogNamespace() + MongoDBRiver.GRIDFS_FILES_SUFFIX;
this.cmdOplogNamespace = definition.getMongoDb() + "." + MongoDBRiver.OPLOG_NAMESPACE_COMMAND;
if (definition.getExcludeFields() != null) {
for (String key : definition.getExcludeFields()) {
findKeys.put(key, 0);
}
} else if (definition.getIncludeFields() != null) {
for (String key : definition.getIncludeFields()) {
findKeys.put(key, 1);
}
}
}
private boolean isValidOplogEntry(final DBObject entry, final Timestamp<?> startTimestamp) {
if (MongoDBRiver.OPLOG_NOOP_OPERATION.equals(entry.get(MongoDBRiver.OPLOG_OPERATION))) {
logger.debug("[No-op Oplog Entry] - can be ignored. {}", entry);
return false;
}
String namespace = (String) entry.get(MongoDBRiver.OPLOG_NAMESPACE);
// Initial support for sharded collection -
// https://jira.mongodb.org/browse/SERVER-4333
// Not interested in operation from migration or sharding
if (entry.containsField(MongoDBRiver.OPLOG_FROM_MIGRATE)
&& ((BasicBSONObject) entry).getBoolean(MongoDBRiver.OPLOG_FROM_MIGRATE)) {
logger.debug(
"[Invalid Oplog Entry] - from migration or sharding operation. Can be ignored. {}",
entry);
return false;
}
// Not interested by chunks - skip all
if (namespace.endsWith(MongoDBRiver.GRIDFS_CHUNKS_SUFFIX)) {
return false;
}
if (startTimestamp != null) {
Timestamp<?> oplogTimestamp = Timestamp.on(entry);
if (Timestamp.compare(oplogTimestamp, startTimestamp) < 0) {
logger.debug(
"[Invalid Oplog Entry] - entry timestamp [{}] before startTimestamp [{}]",
entry,
startTimestamp);
return false;
}
}
boolean validNamespace = false;
if (definition.isMongoGridFS()) {
validNamespace = gridfsOplogNamespace.equals(namespace);
} else {
if (definition.isImportAllCollections()) {
// Skip temp entry generated by map / reduce
if (namespace.startsWith(definition.getMongoDb())
&& !namespace.startsWith(definition.getMongoDb() + ".tmp.mr")) {
validNamespace = true;
}
} else {
if (definition.getMongoOplogNamespace().equals(namespace)) {
validNamespace = true;
}
}
if (cmdOplogNamespace.equals(namespace)) {
validNamespace = true;
}
if (MongoDBRiver.OPLOG_ADMIN_COMMAND.equals(namespace)) {
validNamespace = true;
}
}
if (!validNamespace) {
logger.debug("[Invalid Oplog Entry] - namespace [{}] is not valid", namespace);
return false;
}
String operation = (String) entry.get(MongoDBRiver.OPLOG_OPERATION);
if (!oplogOperations.contains(operation)) {
logger.debug("[Invalid Oplog Entry] - operation [{}] is not valid", operation);
return false;
}
// TODO: implement a better solution
if (definition.getMongoOplogFilter() != null) {
DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT);
BasicDBObject filter = definition.getMongoOplogFilter();
if (!filterMatch(filter, object)) {
logger.debug(
"[Invalid Oplog Entry] - filter [{}] does not match object [{}]", filter, object);
return false;
}
}
return true;
}