@Override
public Facet facet() {
TShortIntHashMap facets = aggregator.facets();
if (facets.isEmpty()) {
pushFacets(facets);
return new InternalShortTermsFacet(
facetName,
comparatorType,
size,
ImmutableList.<InternalShortTermsFacet.ShortEntry>of(),
aggregator.missing());
} else {
// we need to fetch facets of "size * numberOfShards" because of problems in how they are
// distributed across shards
BoundedTreeSet<InternalShortTermsFacet.ShortEntry> ordered =
new BoundedTreeSet<InternalShortTermsFacet.ShortEntry>(
comparatorType.comparator(), size * numberOfShards);
for (TShortIntIterator it = facets.iterator(); it.hasNext(); ) {
it.advance();
ordered.add(new InternalShortTermsFacet.ShortEntry(it.key(), it.value()));
}
pushFacets(facets);
return new InternalShortTermsFacet(
facetName, comparatorType, size, ordered, aggregator.missing());
}
}
@Override
public void readFrom(StreamInput in) throws IOException {
clusterName = in.readUTF();
activePrimaryShards = in.readVInt();
activeShards = in.readVInt();
relocatingShards = in.readVInt();
initializingShards = in.readVInt();
unassignedShards = in.readVInt();
numberOfNodes = in.readVInt();
numberOfDataNodes = in.readVInt();
status = ClusterHealthStatus.fromValue(in.readByte());
int size = in.readVInt();
for (int i = 0; i < size; i++) {
ClusterIndexHealth indexHealth = readClusterIndexHealth(in);
indices.put(indexHealth.index(), indexHealth);
}
timedOut = in.readBoolean();
size = in.readVInt();
if (size == 0) {
validationFailures = ImmutableList.of();
} else {
for (int i = 0; i < size; i++) {
validationFailures.add(in.readUTF());
}
}
}
@Override
public List<IntEntry> entries() {
if (!(entries instanceof List)) {
entries = ImmutableList.copyOf(entries);
}
return (List<IntEntry>) entries;
}
/**
* You cannot restore an open existing index, therefore we first check if an index exists that is
* in the snapshot. If the index exists we close the index before we start the restore action.
*
* @param snapshot String containing the name of the snapshot in the configured repository to
* restore
*/
public void restoreSnapshot(String snapshot) {
ClusterAdminClient adminClient = this.clientFactory.client().admin().cluster();
// Obtain the snapshot and check the indices that are in the snapshot
GetSnapshotsRequestBuilder builder = new GetSnapshotsRequestBuilder(adminClient);
builder.setRepository(repoConfig.getName());
builder.setSnapshots(snapshot);
GetSnapshotsResponse getSnapshotsResponse = builder.execute().actionGet();
// Check if the index exists and if so, close it before we can restore it.
ImmutableList<String> indices = getSnapshotsResponse.getSnapshots().get(0).indices();
CloseIndexRequestBuilder closeIndexRequestBuilder =
new CloseIndexRequestBuilder(this.clientFactory.client().admin().indices());
closeIndexRequestBuilder.setIndices(indices.toArray(new String[indices.size()]));
closeIndexRequestBuilder.execute().actionGet();
// Now execute the actual restore action
RestoreSnapshotRequestBuilder restoreBuilder = new RestoreSnapshotRequestBuilder(adminClient);
restoreBuilder.setRepository(repoConfig.getName()).setSnapshot(snapshot);
restoreBuilder.execute().actionGet();
logger.info("The snapshot {} is restored", snapshot);
}
public static Filter wrapSmartNameFilter(
Filter filter,
@Nullable MapperService.SmartNameFieldMappers smartFieldMappers,
QueryParseContext parseContext) {
if (smartFieldMappers == null) {
return filter;
}
if (!smartFieldMappers.hasDocMapper()) {
return filter;
}
DocumentMapper docMapper = smartFieldMappers.docMapper();
return new AndFilter(
ImmutableList.of(parseContext.cacheFilter(docMapper.typeFilter(), null), filter));
}
private BasicDBObject getFilterForInitialImport(BasicDBObject filter, String id) {
if (id == null) {
return filter;
} else {
BasicDBObject filterId =
new BasicDBObject(
MongoDBRiver.MONGODB_ID_FIELD, new BasicBSONObject(QueryOperators.GT, id));
if (filter == null) {
return filterId;
} else {
List<BasicDBObject> values = ImmutableList.of(filter, filterId);
return new BasicDBObject(QueryOperators.AND, values);
}
}
}
@Test
public void throwDarts() {
time(0, 100, 5);
List<List<Integer>> times = new ArrayList<>();
for (int i = 0; i < 20; i++) {
// int maxConvert = randomIntBetween(0, MAX_LIST);
// int minConvert = randomIntBetween(0, maxConvert);
// int maxKeepAsList = randomIntBetween(0, 200);
int maxConvert = MAX_LIST;
int minConvert = randomIntBetween(0, 400);
int maxKeepAsList = randomIntBetween(20, 20);
int time = (int) printTime(minConvert, maxConvert, maxKeepAsList);
times.add(ImmutableList.of(time, minConvert, maxConvert, maxKeepAsList));
}
Collections.sort(times, Ordering.natural().onResultOf(ORDER));
for (Object t : times) {
log.info("{}", t);
}
}
@Override
public Facet facet() {
TIntIntHashMap facets = aggregator.facets();
if (facets.isEmpty()) {
CacheRecycler.pushIntIntMap(facets);
return new InternalIntTermsFacet(
facetName,
comparatorType,
size,
ImmutableList.<InternalIntTermsFacet.IntEntry>of(),
aggregator.missing());
} else {
if (size < EntryPriorityQueue.LIMIT) {
EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
for (TIntIntIterator it = facets.iterator(); it.hasNext(); ) {
it.advance();
ordered.insertWithOverflow(new InternalIntTermsFacet.IntEntry(it.key(), it.value()));
}
InternalIntTermsFacet.IntEntry[] list = new InternalIntTermsFacet.IntEntry[ordered.size()];
for (int i = ordered.size() - 1; i >= 0; i--) {
list[i] = (InternalIntTermsFacet.IntEntry) ordered.pop();
}
CacheRecycler.pushIntIntMap(facets);
return new InternalIntTermsFacet(
facetName, comparatorType, size, Arrays.asList(list), aggregator.missing());
} else {
BoundedTreeSet<InternalIntTermsFacet.IntEntry> ordered =
new BoundedTreeSet<InternalIntTermsFacet.IntEntry>(comparatorType.comparator(), size);
for (TIntIntIterator it = facets.iterator(); it.hasNext(); ) {
it.advance();
ordered.add(new InternalIntTermsFacet.IntEntry(it.key(), it.value()));
}
CacheRecycler.pushIntIntMap(facets);
return new InternalIntTermsFacet(
facetName, comparatorType, size, ordered, aggregator.missing());
}
}
}
public Multimap<String, FacetValue> getStatsByProfileKey(QualityProfileKey key) {
return getStatsByProfileKey(ImmutableList.of(key)).get(key);
}
private boolean isValidOplogEntry(final DBObject entry, final Timestamp<?> startTimestamp) {
if (MongoDBRiver.OPLOG_NOOP_OPERATION.equals(entry.get(MongoDBRiver.OPLOG_OPERATION))) {
logger.debug("[No-op Oplog Entry] - can be ignored. {}", entry);
return false;
}
String namespace = (String) entry.get(MongoDBRiver.OPLOG_NAMESPACE);
// Initial support for sharded collection -
// https://jira.mongodb.org/browse/SERVER-4333
// Not interested in operation from migration or sharding
if (entry.containsField(MongoDBRiver.OPLOG_FROM_MIGRATE)
&& ((BasicBSONObject) entry).getBoolean(MongoDBRiver.OPLOG_FROM_MIGRATE)) {
logger.debug(
"[Invalid Oplog Entry] - from migration or sharding operation. Can be ignored. {}",
entry);
return false;
}
// Not interested by chunks - skip all
if (namespace.endsWith(MongoDBRiver.GRIDFS_CHUNKS_SUFFIX)) {
return false;
}
if (startTimestamp != null) {
Timestamp<?> oplogTimestamp = Timestamp.on(entry);
if (Timestamp.compare(oplogTimestamp, startTimestamp) < 0) {
logger.debug(
"[Invalid Oplog Entry] - entry timestamp [{}] before startTimestamp [{}]",
entry,
startTimestamp);
return false;
}
}
boolean validNamespace = false;
if (definition.isMongoGridFS()) {
validNamespace = gridfsOplogNamespace.equals(namespace);
} else {
if (definition.isImportAllCollections()) {
// Skip temp entry generated by map / reduce
if (namespace.startsWith(definition.getMongoDb())
&& !namespace.startsWith(definition.getMongoDb() + ".tmp.mr")) {
validNamespace = true;
}
} else {
if (definition.getMongoOplogNamespace().equals(namespace)) {
validNamespace = true;
}
}
if (cmdOplogNamespace.equals(namespace)) {
validNamespace = true;
}
if (MongoDBRiver.OPLOG_ADMIN_COMMAND.equals(namespace)) {
validNamespace = true;
}
}
if (!validNamespace) {
logger.debug("[Invalid Oplog Entry] - namespace [{}] is not valid", namespace);
return false;
}
String operation = (String) entry.get(MongoDBRiver.OPLOG_OPERATION);
if (!oplogOperations.contains(operation)) {
logger.debug("[Invalid Oplog Entry] - operation [{}] is not valid", operation);
return false;
}
// TODO: implement a better solution
if (definition.getMongoOplogFilter() != null) {
DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT);
BasicDBObject filter = definition.getMongoOplogFilter();
if (!filterMatch(filter, object)) {
logger.debug(
"[Invalid Oplog Entry] - filter [{}] does not match object [{}]", filter, object);
return false;
}
}
return true;
}
class Slurper implements Runnable {
class SlurperException extends Exception {
/** */
private static final long serialVersionUID = 1L;
SlurperException(String message) {
super(message);
}
}
private static final ESLogger logger = ESLoggerFactory.getLogger(Slurper.class.getName());
private final MongoDBRiverDefinition definition;
private final SharedContext context;
private final BasicDBObject findKeys;
private final String gridfsOplogNamespace;
private final String cmdOplogNamespace;
private final ImmutableList<String> oplogOperations =
ImmutableList.of(
MongoDBRiver.OPLOG_DELETE_OPERATION,
MongoDBRiver.OPLOG_UPDATE_ROW_OPERATION, // from TokuMX
MongoDBRiver.OPLOG_UPDATE_OPERATION,
MongoDBRiver.OPLOG_INSERT_OPERATION,
MongoDBRiver.OPLOG_COMMAND_OPERATION);
private final Client client;
private Mongo mongo;
private DB slurpedDb;
private DB oplogDb;
private DBCollection oplogCollection;
private final AtomicLong totalDocuments = new AtomicLong();
public Slurper(
List<ServerAddress> mongoServers,
MongoDBRiverDefinition definition,
SharedContext context,
Client client) {
this.definition = definition;
this.context = context;
this.client = client;
this.mongo = new MongoClient(mongoServers, definition.getMongoClientOptions());
this.findKeys = new BasicDBObject();
this.gridfsOplogNamespace =
definition.getMongoOplogNamespace() + MongoDBRiver.GRIDFS_FILES_SUFFIX;
this.cmdOplogNamespace = definition.getMongoDb() + "." + MongoDBRiver.OPLOG_NAMESPACE_COMMAND;
if (definition.getExcludeFields() != null) {
for (String key : definition.getExcludeFields()) {
findKeys.put(key, 0);
}
} else if (definition.getIncludeFields() != null) {
for (String key : definition.getIncludeFields()) {
findKeys.put(key, 1);
}
}
}
@Override
public void run() {
while (context.getStatus() == Status.RUNNING) {
try {
if (!assignCollections()) {
break; // failed to assign oplogCollection or
// slurpedCollection
}
Timestamp<?> startTimestamp = null;
if (!definition.isSkipInitialImport()) {
if (!riverHasIndexedFromOplog() && definition.getInitialTimestamp() == null) {
if (!isIndexEmpty()) {
MongoDBRiverHelper.setRiverStatus(
client, definition.getRiverName(), Status.INITIAL_IMPORT_FAILED);
break;
}
if (definition.isImportAllCollections()) {
for (String name : slurpedDb.getCollectionNames()) {
DBCollection collection = slurpedDb.getCollection(name);
startTimestamp = doInitialImport(collection);
}
} else {
DBCollection collection = slurpedDb.getCollection(definition.getMongoCollection());
startTimestamp = doInitialImport(collection);
}
}
} else {
logger.info("Skip initial import from collection {}", definition.getMongoCollection());
}
// Slurp from oplog
DBCursor cursor = null;
try {
cursor = oplogCursor(startTimestamp);
if (cursor == null) {
cursor = processFullOplog();
}
while (cursor.hasNext()) {
DBObject item = cursor.next();
startTimestamp = processOplogEntry(item, startTimestamp);
}
logger.debug("Before waiting for 500 ms");
Thread.sleep(500);
} catch (MongoException.CursorNotFound e) {
logger.info(
"Cursor {} has been closed. About to open a new cusor.", cursor.getCursorId());
logger.debug("Total document inserted [{}]", totalDocuments.get());
} catch (SlurperException sEx) {
logger.warn("Exception in slurper", sEx);
break;
} catch (Exception ex) {
logger.warn("Exception while looping in cursor", ex);
Thread.currentThread().interrupt();
break;
} finally {
if (cursor != null) {
logger.trace("Closing oplog cursor");
cursor.close();
}
}
} catch (MongoInterruptedException mIEx) {
logger.warn("Mongo driver has been interrupted", mIEx);
if (mongo != null) {
mongo.close();
mongo = null;
}
Thread.currentThread().interrupt();
break;
} catch (MongoException e) {
logger.error("Mongo gave an exception", e);
try {
Thread.sleep(5000);
} catch (InterruptedException iEx) {
}
} catch (NoSuchElementException e) {
logger.warn("A mongoDB cursor bug ?", e);
} catch (InterruptedException e) {
logger.info("river-mongodb slurper interrupted");
Thread.currentThread().interrupt();
break;
}
}
}
protected boolean riverHasIndexedFromOplog() {
return MongoDBRiver.getLastTimestamp(client, definition) != null;
}
protected boolean isIndexEmpty() {
return MongoDBRiver.getIndexCount(client, definition) == 0;
}
/**
* Does an initial sync the same way MongoDB does. https://groups.google.com/
* forum/?fromgroups=#!topic/mongodb-user/sOKlhD_E2ns
*
* @return the last oplog timestamp before the import began
* @throws InterruptedException if the blocking queue stream is interrupted while waiting
*/
protected Timestamp<?> doInitialImport(DBCollection collection) throws InterruptedException {
// TODO: ensure the index type is empty
// DBCollection slurpedCollection =
// slurpedDb.getCollection(definition.getMongoCollection());
logger.info("MongoDBRiver is beginning initial import of " + collection.getFullName());
Timestamp<?> startTimestamp = getCurrentOplogTimestamp();
boolean inProgress = true;
String lastId = null;
while (inProgress) {
DBCursor cursor = null;
try {
if (definition.isDisableIndexRefresh()) {
updateIndexRefresh(definition.getIndexName(), -1L);
}
if (!definition.isMongoGridFS()) {
logger.info("Collection {} - count: {}", collection.getName(), collection.count());
long count = 0;
cursor =
collection.find(
getFilterForInitialImport(definition.getMongoCollectionFilter(), lastId));
while (cursor.hasNext()) {
DBObject object = cursor.next();
count++;
if (cursor.hasNext()) {
lastId = addInsertToStream(null, applyFieldFilter(object), collection.getName());
} else {
logger.debug("Last entry for initial import - add timestamp: {}", startTimestamp);
lastId =
addInsertToStream(startTimestamp, applyFieldFilter(object), collection.getName());
}
}
inProgress = false;
logger.info("Number documents indexed: {}", count);
} else {
// TODO: To be optimized.
// https://github.com/mongodb/mongo-java-driver/pull/48#issuecomment-25241988
// possible option: Get the object id list from .fs
// collection
// then call GriDFS.findOne
GridFS grid =
new GridFS(mongo.getDB(definition.getMongoDb()), definition.getMongoCollection());
cursor = grid.getFileList();
while (cursor.hasNext()) {
DBObject object = cursor.next();
if (object instanceof GridFSDBFile) {
GridFSDBFile file =
grid.findOne(new ObjectId(object.get(MongoDBRiver.MONGODB_ID_FIELD).toString()));
if (cursor.hasNext()) {
lastId = addInsertToStream(null, file);
} else {
logger.debug("Last entry for initial import - add timestamp: {}", startTimestamp);
lastId = addInsertToStream(startTimestamp, file);
}
}
}
inProgress = false;
}
} catch (MongoException.CursorNotFound e) {
logger.info(
"Initial import - Cursor {} has been closed. About to open a new cusor.",
cursor.getCursorId());
logger.debug("Total document inserted [{}]", totalDocuments.get());
} finally {
if (cursor != null) {
logger.trace("Closing initial import cursor");
cursor.close();
}
if (definition.isDisableIndexRefresh()) {
updateIndexRefresh(definition.getIndexName(), TimeValue.timeValueSeconds(1));
}
}
}
return startTimestamp;
}
private BasicDBObject getFilterForInitialImport(BasicDBObject filter, String id) {
if (id == null) {
return filter;
} else {
BasicDBObject filterId =
new BasicDBObject(
MongoDBRiver.MONGODB_ID_FIELD, new BasicBSONObject(QueryOperators.GT, id));
if (filter == null) {
return filterId;
} else {
List<BasicDBObject> values = ImmutableList.of(filter, filterId);
return new BasicDBObject(QueryOperators.AND, values);
}
}
}
protected boolean assignCollections() {
DB adminDb = mongo.getDB(MongoDBRiver.MONGODB_ADMIN_DATABASE);
oplogDb = mongo.getDB(MongoDBRiver.MONGODB_LOCAL_DATABASE);
if (!definition.getMongoAdminUser().isEmpty()
&& !definition.getMongoAdminPassword().isEmpty()) {
logger.info(
"Authenticate {} with {}",
MongoDBRiver.MONGODB_ADMIN_DATABASE,
definition.getMongoAdminUser());
CommandResult cmd =
adminDb.authenticateCommand(
definition.getMongoAdminUser(), definition.getMongoAdminPassword().toCharArray());
if (!cmd.ok()) {
logger.error(
"Autenticatication failed for {}: {}",
MongoDBRiver.MONGODB_ADMIN_DATABASE,
cmd.getErrorMessage());
// Can still try with mongoLocal credential if provided.
// return false;
}
oplogDb = adminDb.getMongo().getDB(MongoDBRiver.MONGODB_LOCAL_DATABASE);
}
if (!definition.getMongoLocalUser().isEmpty()
&& !definition.getMongoLocalPassword().isEmpty()
&& !oplogDb.isAuthenticated()) {
logger.info(
"Authenticate {} with {}",
MongoDBRiver.MONGODB_LOCAL_DATABASE,
definition.getMongoLocalUser());
CommandResult cmd =
oplogDb.authenticateCommand(
definition.getMongoLocalUser(), definition.getMongoLocalPassword().toCharArray());
if (!cmd.ok()) {
logger.error(
"Autenticatication failed for {}: {}",
MongoDBRiver.MONGODB_LOCAL_DATABASE,
cmd.getErrorMessage());
return false;
}
}
Set<String> collections = oplogDb.getCollectionNames();
if (!collections.contains(MongoDBRiver.OPLOG_COLLECTION)) {
logger.error(
"Cannot find "
+ MongoDBRiver.OPLOG_COLLECTION
+ " collection. Please check this link: http://goo.gl/2x5IW");
return false;
}
oplogCollection = oplogDb.getCollection(MongoDBRiver.OPLOG_COLLECTION);
slurpedDb = mongo.getDB(definition.getMongoDb());
if (!definition.getMongoAdminUser().isEmpty()
&& !definition.getMongoAdminPassword().isEmpty()
&& adminDb.isAuthenticated()) {
slurpedDb = adminDb.getMongo().getDB(definition.getMongoDb());
}
// Not necessary as local user has access to all databases.
// http://docs.mongodb.org/manual/reference/local-database/
// if (!mongoDbUser.isEmpty() && !mongoDbPassword.isEmpty()
// && !slurpedDb.isAuthenticated()) {
// logger.info("Authenticate {} with {}", mongoDb, mongoDbUser);
// CommandResult cmd = slurpedDb.authenticateCommand(mongoDbUser,
// mongoDbPassword.toCharArray());
// if (!cmd.ok()) {
// logger.error("Authentication failed for {}: {}",
// mongoDb, cmd.getErrorMessage());
// return false;
// }
// }
// slurpedCollection =
// slurpedDb.getCollection(definition.getMongoCollection());
// if (definition.isImportAllCollections()) {
// for (String collection : slurpedDb.getCollectionNames()) {
// slurpedCollections.put(collection,
// slurpedDb.getCollection(collection));
// }
// } else {
// slurpedCollections.put(definition.getMongoCollection(),
// slurpedDb.getCollection(definition.getMongoCollection()));
// }
return true;
}
private void updateIndexRefresh(String name, Object value) {
client
.admin()
.indices()
.prepareUpdateSettings(name)
.setSettings(ImmutableMap.of("index.refresh_interval", value))
.get();
}
private Timestamp<?> getCurrentOplogTimestamp() {
return Timestamp.on(
oplogCollection
.find()
.sort(new BasicDBObject(MongoDBRiver.INSERTION_ORDER_KEY, -1))
.limit(1)
.next());
}
private DBCursor processFullOplog() throws InterruptedException, SlurperException {
Timestamp<?> currentTimestamp = getCurrentOplogTimestamp();
addInsertToStream(currentTimestamp, null);
return oplogCursor(currentTimestamp);
}
private Timestamp<?> processOplogEntry(final DBObject entry, final Timestamp<?> startTimestamp)
throws InterruptedException {
// To support transactions, TokuMX wraps one or more operations in a single oplog entry, in a
// list.
// As long as clients are not transaction-aware, we can pretty safely assume there will only be
// one operation in the list.
// Supporting genuine multi-operation transactions will require a bit more logic here.
flattenOps(entry);
if (!isValidOplogEntry(entry, startTimestamp)) {
return startTimestamp;
}
Operation operation = Operation.fromString(entry.get(MongoDBRiver.OPLOG_OPERATION).toString());
String namespace = entry.get(MongoDBRiver.OPLOG_NAMESPACE).toString();
String collection = null;
Timestamp<?> oplogTimestamp = Timestamp.on(entry);
DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT);
if (definition.isImportAllCollections()) {
if (namespace.startsWith(definition.getMongoDb()) && !namespace.equals(cmdOplogNamespace)) {
collection = getCollectionFromNamespace(namespace);
}
} else {
collection = definition.getMongoCollection();
}
if (namespace.equals(cmdOplogNamespace)) {
if (object.containsField(MongoDBRiver.OPLOG_DROP_COMMAND_OPERATION)) {
operation = Operation.DROP_COLLECTION;
if (definition.isImportAllCollections()) {
collection = object.get(MongoDBRiver.OPLOG_DROP_COMMAND_OPERATION).toString();
if (collection.startsWith("tmp.mr.")) {
return startTimestamp;
}
}
}
if (object.containsField(MongoDBRiver.OPLOG_DROP_DATABASE_COMMAND_OPERATION)) {
operation = Operation.DROP_DATABASE;
}
}
logger.trace("namespace: {} - operation: {}", namespace, operation);
if (namespace.equals(MongoDBRiver.OPLOG_ADMIN_COMMAND)) {
if (operation == Operation.COMMAND) {
processAdminCommandOplogEntry(entry, startTimestamp);
return startTimestamp;
}
}
if (logger.isTraceEnabled()) {
logger.trace("MongoDB object deserialized: {}", object.toString());
logger.trace("collection: {}", collection);
logger.trace("oplog entry - namespace [{}], operation [{}]", namespace, operation);
logger.trace("oplog processing item {}", entry);
}
String objectId = getObjectIdFromOplogEntry(entry);
if (operation == Operation.DELETE) {
// Include only _id in data, as vanilla MongoDB does, so transformation scripts won't be
// broken by Toku
if (object.containsField(MongoDBRiver.MONGODB_ID_FIELD)) {
if (object.keySet().size() > 1) {
entry.put(
MongoDBRiver.OPLOG_OBJECT,
object = new BasicDBObject(MongoDBRiver.MONGODB_ID_FIELD, objectId));
}
} else {
throw new NullPointerException(MongoDBRiver.MONGODB_ID_FIELD);
}
}
if (definition.isMongoGridFS()
&& namespace.endsWith(MongoDBRiver.GRIDFS_FILES_SUFFIX)
&& (operation == Operation.INSERT || operation == Operation.UPDATE)) {
if (objectId == null) {
throw new NullPointerException(MongoDBRiver.MONGODB_ID_FIELD);
}
GridFS grid = new GridFS(mongo.getDB(definition.getMongoDb()), collection);
GridFSDBFile file = grid.findOne(new ObjectId(objectId));
if (file != null) {
logger.info("Caught file: {} - {}", file.getId(), file.getFilename());
object = file;
} else {
logger.warn("Cannot find file from id: {}", objectId);
}
}
if (object instanceof GridFSDBFile) {
if (objectId == null) {
throw new NullPointerException(MongoDBRiver.MONGODB_ID_FIELD);
}
if (logger.isTraceEnabled()) {
logger.trace("Add attachment: {}", objectId);
}
addToStream(operation, oplogTimestamp, applyFieldFilter(object), collection);
} else {
if (operation == Operation.UPDATE) {
DBObject update = (DBObject) entry.get(MongoDBRiver.OPLOG_UPDATE);
logger.debug("Updated item: {}", update);
addQueryToStream(operation, oplogTimestamp, update, collection);
} else {
if (operation == Operation.INSERT) {
addInsertToStream(oplogTimestamp, applyFieldFilter(object), collection);
} else {
addToStream(operation, oplogTimestamp, applyFieldFilter(object), collection);
}
}
}
return oplogTimestamp;
}
@SuppressWarnings("unchecked")
private void flattenOps(DBObject entry) {
Object ops = entry.get(MongoDBRiver.OPLOG_OPS);
if (ops != null) {
try {
for (DBObject op : (List<DBObject>) ops) {
String operation = (String) op.get(MongoDBRiver.OPLOG_OPERATION);
if (operation.equals(MongoDBRiver.OPLOG_COMMAND_OPERATION)) {
DBObject object = (DBObject) op.get(MongoDBRiver.OPLOG_OBJECT);
if (object.containsField(MongoDBRiver.OPLOG_CREATE_COMMAND)) {
continue;
}
}
entry.putAll(op);
}
} catch (ClassCastException e) {
logger.error(e.toString(), e);
}
}
}
private void processAdminCommandOplogEntry(
final DBObject entry, final Timestamp<?> startTimestamp) throws InterruptedException {
if (logger.isTraceEnabled()) {
logger.trace("processAdminCommandOplogEntry - [{}]", entry);
}
DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT);
if (definition.isImportAllCollections()) {
if (object.containsField(MongoDBRiver.OPLOG_RENAME_COLLECTION_COMMAND_OPERATION)
&& object.containsField(MongoDBRiver.OPLOG_TO)) {
String to = object.get(MongoDBRiver.OPLOG_TO).toString();
if (to.startsWith(definition.getMongoDb())) {
String newCollection = getCollectionFromNamespace(to);
DBCollection coll = slurpedDb.getCollection(newCollection);
doInitialImport(coll);
}
}
}
}
private String getCollectionFromNamespace(String namespace) {
if (namespace.startsWith(definition.getMongoDb())
&& CharMatcher.is('.').countIn(namespace) == 1) {
return namespace.substring(definition.getMongoDb().length() + 1);
}
logger.info("Cannot get collection from namespace [{}]", namespace);
return null;
}
private boolean isValidOplogEntry(final DBObject entry, final Timestamp<?> startTimestamp) {
if (MongoDBRiver.OPLOG_NOOP_OPERATION.equals(entry.get(MongoDBRiver.OPLOG_OPERATION))) {
logger.debug("[No-op Oplog Entry] - can be ignored. {}", entry);
return false;
}
String namespace = (String) entry.get(MongoDBRiver.OPLOG_NAMESPACE);
// Initial support for sharded collection -
// https://jira.mongodb.org/browse/SERVER-4333
// Not interested in operation from migration or sharding
if (entry.containsField(MongoDBRiver.OPLOG_FROM_MIGRATE)
&& ((BasicBSONObject) entry).getBoolean(MongoDBRiver.OPLOG_FROM_MIGRATE)) {
logger.debug(
"[Invalid Oplog Entry] - from migration or sharding operation. Can be ignored. {}",
entry);
return false;
}
// Not interested by chunks - skip all
if (namespace.endsWith(MongoDBRiver.GRIDFS_CHUNKS_SUFFIX)) {
return false;
}
if (startTimestamp != null) {
Timestamp<?> oplogTimestamp = Timestamp.on(entry);
if (Timestamp.compare(oplogTimestamp, startTimestamp) < 0) {
logger.debug(
"[Invalid Oplog Entry] - entry timestamp [{}] before startTimestamp [{}]",
entry,
startTimestamp);
return false;
}
}
boolean validNamespace = false;
if (definition.isMongoGridFS()) {
validNamespace = gridfsOplogNamespace.equals(namespace);
} else {
if (definition.isImportAllCollections()) {
// Skip temp entry generated by map / reduce
if (namespace.startsWith(definition.getMongoDb())
&& !namespace.startsWith(definition.getMongoDb() + ".tmp.mr")) {
validNamespace = true;
}
} else {
if (definition.getMongoOplogNamespace().equals(namespace)) {
validNamespace = true;
}
}
if (cmdOplogNamespace.equals(namespace)) {
validNamespace = true;
}
if (MongoDBRiver.OPLOG_ADMIN_COMMAND.equals(namespace)) {
validNamespace = true;
}
}
if (!validNamespace) {
logger.debug("[Invalid Oplog Entry] - namespace [{}] is not valid", namespace);
return false;
}
String operation = (String) entry.get(MongoDBRiver.OPLOG_OPERATION);
if (!oplogOperations.contains(operation)) {
logger.debug("[Invalid Oplog Entry] - operation [{}] is not valid", operation);
return false;
}
// TODO: implement a better solution
if (definition.getMongoOplogFilter() != null) {
DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT);
BasicDBObject filter = definition.getMongoOplogFilter();
if (!filterMatch(filter, object)) {
logger.debug(
"[Invalid Oplog Entry] - filter [{}] does not match object [{}]", filter, object);
return false;
}
}
return true;
}
private boolean filterMatch(DBObject filter, DBObject object) {
for (String key : filter.keySet()) {
if (!object.containsField(key)) {
return false;
}
if (!filter.get(key).equals(object.get(key))) {
return false;
}
}
return true;
}
private DBObject applyFieldFilter(DBObject object) {
if (object instanceof GridFSFile) {
GridFSFile file = (GridFSFile) object;
DBObject metadata = file.getMetaData();
if (metadata != null) {
file.setMetaData(applyFieldFilter(metadata));
}
} else {
object = MongoDBHelper.applyExcludeFields(object, definition.getExcludeFields());
object = MongoDBHelper.applyIncludeFields(object, definition.getIncludeFields());
}
return object;
}
/*
* Extract "_id" from "o" if it fails try to extract from "o2"
*/
private String getObjectIdFromOplogEntry(DBObject entry) {
if (entry.containsField(MongoDBRiver.OPLOG_OBJECT)) {
DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT);
if (object.containsField(MongoDBRiver.MONGODB_ID_FIELD)) {
return object.get(MongoDBRiver.MONGODB_ID_FIELD).toString();
}
}
if (entry.containsField(MongoDBRiver.OPLOG_UPDATE)) {
DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_UPDATE);
if (object.containsField(MongoDBRiver.MONGODB_ID_FIELD)) {
return object.get(MongoDBRiver.MONGODB_ID_FIELD).toString();
}
}
return null;
}
private DBCursor oplogCursor(final Timestamp<?> timestampOverride) throws SlurperException {
Timestamp<?> time =
timestampOverride == null
? MongoDBRiver.getLastTimestamp(client, definition)
: timestampOverride;
DBObject indexFilter = time.getOplogFilter();
if (indexFilter == null) {
return null;
}
int options =
Bytes.QUERYOPTION_TAILABLE | Bytes.QUERYOPTION_AWAITDATA | Bytes.QUERYOPTION_NOTIMEOUT;
// Using OPLOGREPLAY to improve performance:
// https://jira.mongodb.org/browse/JAVA-771
if (indexFilter.containsField(MongoDBRiver.OPLOG_TIMESTAMP)) {
options = options | Bytes.QUERYOPTION_OPLOGREPLAY;
}
DBCursor cursor = oplogCollection.find(indexFilter).setOptions(options);
isRiverStale(cursor, time);
return cursor;
}
private void isRiverStale(DBCursor cursor, Timestamp<?> time) throws SlurperException {
if (cursor == null || time == null) {
return;
}
if (definition.getInitialTimestamp() != null && time.equals(definition.getInitialTimestamp())) {
return;
}
DBObject entry = cursor.next();
Timestamp<?> oplogTimestamp = Timestamp.on(entry);
if (!time.equals(oplogTimestamp)) {
MongoDBRiverHelper.setRiverStatus(client, definition.getRiverName(), Status.RIVER_STALE);
throw new SlurperException("River out of sync with oplog.rs collection");
}
}
private void addQueryToStream(
final Operation operation,
final Timestamp<?> currentTimestamp,
final DBObject update,
final String collection)
throws InterruptedException {
if (logger.isTraceEnabled()) {
logger.trace(
"addQueryToStream - operation [{}], currentTimestamp [{}], update [{}]",
operation,
currentTimestamp,
update);
}
if (collection == null) {
for (String name : slurpedDb.getCollectionNames()) {
DBCollection slurpedCollection = slurpedDb.getCollection(name);
for (DBObject item : slurpedCollection.find(update, findKeys)) {
addToStream(operation, currentTimestamp, item, collection);
}
}
} else {
DBCollection slurpedCollection = slurpedDb.getCollection(collection);
for (DBObject item : slurpedCollection.find(update, findKeys)) {
addToStream(operation, currentTimestamp, item, collection);
}
}
}
private String addInsertToStream(final Timestamp<?> currentTimestamp, final DBObject data)
throws InterruptedException {
return addInsertToStream(currentTimestamp, data, definition.getMongoCollection());
}
private String addInsertToStream(
final Timestamp<?> currentTimestamp, final DBObject data, final String collection)
throws InterruptedException {
totalDocuments.incrementAndGet();
addToStream(Operation.INSERT, currentTimestamp, data, collection);
return data.containsField(MongoDBRiver.MONGODB_ID_FIELD)
? data.get(MongoDBRiver.MONGODB_ID_FIELD).toString()
: null;
}
private void addToStream(
final Operation operation,
final Timestamp<?> currentTimestamp,
final DBObject data,
final String collection)
throws InterruptedException {
if (logger.isTraceEnabled()) {
logger.trace(
"addToStream - operation [{}], currentTimestamp [{}], data [{}], collection [{}]",
operation,
currentTimestamp,
data,
collection);
}
if (operation == Operation.DROP_DATABASE) {
if (definition.isImportAllCollections()) {
for (String name : slurpedDb.getCollectionNames()) {
context
.getStream()
.put(
new MongoDBRiver.QueueEntry(
currentTimestamp, Operation.DROP_COLLECTION, data, name));
}
} else {
context
.getStream()
.put(
new MongoDBRiver.QueueEntry(
currentTimestamp, Operation.DROP_COLLECTION, data, collection));
}
} else {
context
.getStream()
.put(new MongoDBRiver.QueueEntry(currentTimestamp, operation, data, collection));
}
}
}
/** @author kimchy (shay.banon) */
public class InternalIntTermsFacet extends InternalTermsFacet {
private static final String STREAM_TYPE = "iTerms";
public static void registerStream() {
Streams.registerStream(STREAM, STREAM_TYPE);
}
static Stream STREAM =
new Stream() {
@Override
public Facet readFacet(String type, StreamInput in) throws IOException {
return readTermsFacet(in);
}
};
@Override
public String streamType() {
return STREAM_TYPE;
}
public static class IntEntry implements Entry {
int term;
int count;
public IntEntry(int term, int count) {
this.term = term;
this.count = count;
}
public String term() {
return Integer.toString(term);
}
public String getTerm() {
return term();
}
@Override
public Number termAsNumber() {
return term;
}
@Override
public Number getTermAsNumber() {
return termAsNumber();
}
public int count() {
return count;
}
public int getCount() {
return count();
}
@Override
public int compareTo(Entry o) {
int anotherVal = ((IntEntry) o).term;
int i = term - anotherVal;
if (i == 0) {
i = count - o.count();
if (i == 0) {
i = System.identityHashCode(this) - System.identityHashCode(o);
}
}
return i;
}
}
private String name;
int requiredSize;
long missing;
Collection<IntEntry> entries = ImmutableList.of();
ComparatorType comparatorType;
InternalIntTermsFacet() {}
public InternalIntTermsFacet(
String name,
ComparatorType comparatorType,
int requiredSize,
Collection<IntEntry> entries,
long missing) {
this.name = name;
this.comparatorType = comparatorType;
this.requiredSize = requiredSize;
this.entries = entries;
this.missing = missing;
}
@Override
public String name() {
return this.name;
}
@Override
public String getName() {
return this.name;
}
@Override
public String type() {
return TYPE;
}
@Override
public String getType() {
return type();
}
@Override
public List<IntEntry> entries() {
if (!(entries instanceof List)) {
entries = ImmutableList.copyOf(entries);
}
return (List<IntEntry>) entries;
}
@Override
public List<IntEntry> getEntries() {
return entries();
}
@SuppressWarnings({"unchecked"})
@Override
public Iterator<Entry> iterator() {
return (Iterator) entries.iterator();
}
@Override
public long missingCount() {
return this.missing;
}
@Override
public long getMissingCount() {
return missingCount();
}
private static ThreadLocal<ThreadLocals.CleanableValue<TIntIntHashMap>> aggregateCache =
new ThreadLocal<ThreadLocals.CleanableValue<TIntIntHashMap>>() {
@Override
protected ThreadLocals.CleanableValue<TIntIntHashMap> initialValue() {
return new ThreadLocals.CleanableValue<TIntIntHashMap>(new TIntIntHashMap());
}
};
@Override
public Facet reduce(String name, List<Facet> facets) {
if (facets.size() == 1) {
return facets.get(0);
}
InternalIntTermsFacet first = (InternalIntTermsFacet) facets.get(0);
TIntIntHashMap aggregated = aggregateCache.get().get();
aggregated.clear();
long missing = 0;
for (Facet facet : facets) {
InternalIntTermsFacet mFacet = (InternalIntTermsFacet) facet;
missing += mFacet.missingCount();
for (IntEntry entry : mFacet.entries) {
aggregated.adjustOrPutValue(entry.term, entry.count(), entry.count());
}
}
BoundedTreeSet<IntEntry> ordered =
new BoundedTreeSet<IntEntry>(first.comparatorType.comparator(), first.requiredSize);
for (TIntIntIterator it = aggregated.iterator(); it.hasNext(); ) {
it.advance();
ordered.add(new IntEntry(it.key(), it.value()));
}
first.entries = ordered;
first.missing = missing;
return first;
}
static final class Fields {
static final XContentBuilderString _TYPE = new XContentBuilderString("_type");
static final XContentBuilderString MISSING = new XContentBuilderString("missing");
static final XContentBuilderString TERMS = new XContentBuilderString("terms");
static final XContentBuilderString TERM = new XContentBuilderString("term");
static final XContentBuilderString COUNT = new XContentBuilderString("count");
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(name);
builder.field(Fields._TYPE, TermsFacet.TYPE);
builder.field(Fields.MISSING, missing);
builder.startArray(Fields.TERMS);
for (IntEntry entry : entries) {
builder.startObject();
builder.field(Fields.TERM, entry.term);
builder.field(Fields.COUNT, entry.count());
builder.endObject();
}
builder.endArray();
builder.endObject();
return builder;
}
public static InternalIntTermsFacet readTermsFacet(StreamInput in) throws IOException {
InternalIntTermsFacet facet = new InternalIntTermsFacet();
facet.readFrom(in);
return facet;
}
@Override
public void readFrom(StreamInput in) throws IOException {
name = in.readUTF();
comparatorType = ComparatorType.fromId(in.readByte());
requiredSize = in.readVInt();
missing = in.readVLong();
int size = in.readVInt();
entries = new ArrayList<IntEntry>(size);
for (int i = 0; i < size; i++) {
entries.add(new IntEntry(in.readInt(), in.readVInt()));
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeUTF(name);
out.writeByte(comparatorType.id());
out.writeVInt(requiredSize);
out.writeVLong(missing);
out.writeVInt(entries.size());
for (IntEntry entry : entries) {
out.writeInt(entry.term);
out.writeVInt(entry.count());
}
}
}
