@Override
 public Facet facet() {
   TShortIntHashMap facets = aggregator.facets();
   if (facets.isEmpty()) {
     pushFacets(facets);
     return new InternalShortTermsFacet(
         facetName,
         comparatorType,
         size,
         ImmutableList.<InternalShortTermsFacet.ShortEntry>of(),
         aggregator.missing());
   } else {
     // we need to fetch facets of "size * numberOfShards" because of problems in how they are
     // distributed across shards
     BoundedTreeSet<InternalShortTermsFacet.ShortEntry> ordered =
         new BoundedTreeSet<InternalShortTermsFacet.ShortEntry>(
             comparatorType.comparator(), size * numberOfShards);
     for (TShortIntIterator it = facets.iterator(); it.hasNext(); ) {
       it.advance();
       ordered.add(new InternalShortTermsFacet.ShortEntry(it.key(), it.value()));
     }
     pushFacets(facets);
     return new InternalShortTermsFacet(
         facetName, comparatorType, size, ordered, aggregator.missing());
   }
 }
 @Override
 public void readFrom(StreamInput in) throws IOException {
   clusterName = in.readUTF();
   activePrimaryShards = in.readVInt();
   activeShards = in.readVInt();
   relocatingShards = in.readVInt();
   initializingShards = in.readVInt();
   unassignedShards = in.readVInt();
   numberOfNodes = in.readVInt();
   numberOfDataNodes = in.readVInt();
   status = ClusterHealthStatus.fromValue(in.readByte());
   int size = in.readVInt();
   for (int i = 0; i < size; i++) {
     ClusterIndexHealth indexHealth = readClusterIndexHealth(in);
     indices.put(indexHealth.index(), indexHealth);
   }
   timedOut = in.readBoolean();
   size = in.readVInt();
   if (size == 0) {
     validationFailures = ImmutableList.of();
   } else {
     for (int i = 0; i < size; i++) {
       validationFailures.add(in.readUTF());
     }
   }
 }
 @Override
 public List<IntEntry> entries() {
   if (!(entries instanceof List)) {
     entries = ImmutableList.copyOf(entries);
   }
   return (List<IntEntry>) entries;
 }
  /**
   * You cannot restore an open existing index, therefore we first check if an index exists that is
   * in the snapshot. If the index exists we close the index before we start the restore action.
   *
   * @param snapshot String containing the name of the snapshot in the configured repository to
   *     restore
   */
  public void restoreSnapshot(String snapshot) {
    ClusterAdminClient adminClient = this.clientFactory.client().admin().cluster();

    // Obtain the snapshot and check the indices that are in the snapshot
    GetSnapshotsRequestBuilder builder = new GetSnapshotsRequestBuilder(adminClient);
    builder.setRepository(repoConfig.getName());
    builder.setSnapshots(snapshot);
    GetSnapshotsResponse getSnapshotsResponse = builder.execute().actionGet();

    // Check if the index exists and if so, close it before we can restore it.
    ImmutableList<String> indices = getSnapshotsResponse.getSnapshots().get(0).indices();
    CloseIndexRequestBuilder closeIndexRequestBuilder =
        new CloseIndexRequestBuilder(this.clientFactory.client().admin().indices());
    closeIndexRequestBuilder.setIndices(indices.toArray(new String[indices.size()]));
    closeIndexRequestBuilder.execute().actionGet();

    // Now execute the actual restore action
    RestoreSnapshotRequestBuilder restoreBuilder = new RestoreSnapshotRequestBuilder(adminClient);
    restoreBuilder.setRepository(repoConfig.getName()).setSnapshot(snapshot);
    restoreBuilder.execute().actionGet();
    logger.info("The snapshot {} is restored", snapshot);
  }
Esempio n. 5
0
 public static Filter wrapSmartNameFilter(
     Filter filter,
     @Nullable MapperService.SmartNameFieldMappers smartFieldMappers,
     QueryParseContext parseContext) {
   if (smartFieldMappers == null) {
     return filter;
   }
   if (!smartFieldMappers.hasDocMapper()) {
     return filter;
   }
   DocumentMapper docMapper = smartFieldMappers.docMapper();
   return new AndFilter(
       ImmutableList.of(parseContext.cacheFilter(docMapper.typeFilter(), null), filter));
 }
 private BasicDBObject getFilterForInitialImport(BasicDBObject filter, String id) {
   if (id == null) {
     return filter;
   } else {
     BasicDBObject filterId =
         new BasicDBObject(
             MongoDBRiver.MONGODB_ID_FIELD, new BasicBSONObject(QueryOperators.GT, id));
     if (filter == null) {
       return filterId;
     } else {
       List<BasicDBObject> values = ImmutableList.of(filter, filterId);
       return new BasicDBObject(QueryOperators.AND, values);
     }
   }
 }
  @Test
  public void throwDarts() {
    time(0, 100, 5);

    List<List<Integer>> times = new ArrayList<>();
    for (int i = 0; i < 20; i++) {
      // int maxConvert = randomIntBetween(0, MAX_LIST);
      // int minConvert = randomIntBetween(0, maxConvert);
      // int maxKeepAsList = randomIntBetween(0, 200);
      int maxConvert = MAX_LIST;
      int minConvert = randomIntBetween(0, 400);
      int maxKeepAsList = randomIntBetween(20, 20);
      int time = (int) printTime(minConvert, maxConvert, maxKeepAsList);
      times.add(ImmutableList.of(time, minConvert, maxConvert, maxKeepAsList));
    }
    Collections.sort(times, Ordering.natural().onResultOf(ORDER));
    for (Object t : times) {
      log.info("{}", t);
    }
  }
 @Override
 public Facet facet() {
   TIntIntHashMap facets = aggregator.facets();
   if (facets.isEmpty()) {
     CacheRecycler.pushIntIntMap(facets);
     return new InternalIntTermsFacet(
         facetName,
         comparatorType,
         size,
         ImmutableList.<InternalIntTermsFacet.IntEntry>of(),
         aggregator.missing());
   } else {
     if (size < EntryPriorityQueue.LIMIT) {
       EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
       for (TIntIntIterator it = facets.iterator(); it.hasNext(); ) {
         it.advance();
         ordered.insertWithOverflow(new InternalIntTermsFacet.IntEntry(it.key(), it.value()));
       }
       InternalIntTermsFacet.IntEntry[] list = new InternalIntTermsFacet.IntEntry[ordered.size()];
       for (int i = ordered.size() - 1; i >= 0; i--) {
         list[i] = (InternalIntTermsFacet.IntEntry) ordered.pop();
       }
       CacheRecycler.pushIntIntMap(facets);
       return new InternalIntTermsFacet(
           facetName, comparatorType, size, Arrays.asList(list), aggregator.missing());
     } else {
       BoundedTreeSet<InternalIntTermsFacet.IntEntry> ordered =
           new BoundedTreeSet<InternalIntTermsFacet.IntEntry>(comparatorType.comparator(), size);
       for (TIntIntIterator it = facets.iterator(); it.hasNext(); ) {
         it.advance();
         ordered.add(new InternalIntTermsFacet.IntEntry(it.key(), it.value()));
       }
       CacheRecycler.pushIntIntMap(facets);
       return new InternalIntTermsFacet(
           facetName, comparatorType, size, ordered, aggregator.missing());
     }
   }
 }
Esempio n. 9
0
 public Multimap<String, FacetValue> getStatsByProfileKey(QualityProfileKey key) {
   return getStatsByProfileKey(ImmutableList.of(key)).get(key);
 }
  private boolean isValidOplogEntry(final DBObject entry, final Timestamp<?> startTimestamp) {
    if (MongoDBRiver.OPLOG_NOOP_OPERATION.equals(entry.get(MongoDBRiver.OPLOG_OPERATION))) {
      logger.debug("[No-op Oplog Entry] - can be ignored. {}", entry);
      return false;
    }
    String namespace = (String) entry.get(MongoDBRiver.OPLOG_NAMESPACE);
    // Initial support for sharded collection -
    // https://jira.mongodb.org/browse/SERVER-4333
    // Not interested in operation from migration or sharding
    if (entry.containsField(MongoDBRiver.OPLOG_FROM_MIGRATE)
        && ((BasicBSONObject) entry).getBoolean(MongoDBRiver.OPLOG_FROM_MIGRATE)) {
      logger.debug(
          "[Invalid Oplog Entry] - from migration or sharding operation. Can be ignored. {}",
          entry);
      return false;
    }
    // Not interested by chunks - skip all
    if (namespace.endsWith(MongoDBRiver.GRIDFS_CHUNKS_SUFFIX)) {
      return false;
    }

    if (startTimestamp != null) {
      Timestamp<?> oplogTimestamp = Timestamp.on(entry);
      if (Timestamp.compare(oplogTimestamp, startTimestamp) < 0) {
        logger.debug(
            "[Invalid Oplog Entry] - entry timestamp [{}] before startTimestamp [{}]",
            entry,
            startTimestamp);
        return false;
      }
    }

    boolean validNamespace = false;
    if (definition.isMongoGridFS()) {
      validNamespace = gridfsOplogNamespace.equals(namespace);
    } else {
      if (definition.isImportAllCollections()) {
        // Skip temp entry generated by map / reduce
        if (namespace.startsWith(definition.getMongoDb())
            && !namespace.startsWith(definition.getMongoDb() + ".tmp.mr")) {
          validNamespace = true;
        }
      } else {
        if (definition.getMongoOplogNamespace().equals(namespace)) {
          validNamespace = true;
        }
      }
      if (cmdOplogNamespace.equals(namespace)) {
        validNamespace = true;
      }

      if (MongoDBRiver.OPLOG_ADMIN_COMMAND.equals(namespace)) {
        validNamespace = true;
      }
    }
    if (!validNamespace) {
      logger.debug("[Invalid Oplog Entry] - namespace [{}] is not valid", namespace);
      return false;
    }
    String operation = (String) entry.get(MongoDBRiver.OPLOG_OPERATION);
    if (!oplogOperations.contains(operation)) {
      logger.debug("[Invalid Oplog Entry] - operation [{}] is not valid", operation);
      return false;
    }

    // TODO: implement a better solution
    if (definition.getMongoOplogFilter() != null) {
      DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT);
      BasicDBObject filter = definition.getMongoOplogFilter();
      if (!filterMatch(filter, object)) {
        logger.debug(
            "[Invalid Oplog Entry] - filter [{}] does not match object [{}]", filter, object);
        return false;
      }
    }
    return true;
  }
class Slurper implements Runnable {

  class SlurperException extends Exception {
    /** */
    private static final long serialVersionUID = 1L;

    SlurperException(String message) {
      super(message);
    }
  }

  private static final ESLogger logger = ESLoggerFactory.getLogger(Slurper.class.getName());

  private final MongoDBRiverDefinition definition;
  private final SharedContext context;
  private final BasicDBObject findKeys;
  private final String gridfsOplogNamespace;
  private final String cmdOplogNamespace;
  private final ImmutableList<String> oplogOperations =
      ImmutableList.of(
          MongoDBRiver.OPLOG_DELETE_OPERATION,
          MongoDBRiver.OPLOG_UPDATE_ROW_OPERATION, // from TokuMX
          MongoDBRiver.OPLOG_UPDATE_OPERATION,
          MongoDBRiver.OPLOG_INSERT_OPERATION,
          MongoDBRiver.OPLOG_COMMAND_OPERATION);
  private final Client client;
  private Mongo mongo;
  private DB slurpedDb;
  private DB oplogDb;
  private DBCollection oplogCollection;
  private final AtomicLong totalDocuments = new AtomicLong();

  public Slurper(
      List<ServerAddress> mongoServers,
      MongoDBRiverDefinition definition,
      SharedContext context,
      Client client) {
    this.definition = definition;
    this.context = context;
    this.client = client;
    this.mongo = new MongoClient(mongoServers, definition.getMongoClientOptions());
    this.findKeys = new BasicDBObject();
    this.gridfsOplogNamespace =
        definition.getMongoOplogNamespace() + MongoDBRiver.GRIDFS_FILES_SUFFIX;
    this.cmdOplogNamespace = definition.getMongoDb() + "." + MongoDBRiver.OPLOG_NAMESPACE_COMMAND;
    if (definition.getExcludeFields() != null) {
      for (String key : definition.getExcludeFields()) {
        findKeys.put(key, 0);
      }
    } else if (definition.getIncludeFields() != null) {
      for (String key : definition.getIncludeFields()) {
        findKeys.put(key, 1);
      }
    }
  }

  @Override
  public void run() {
    while (context.getStatus() == Status.RUNNING) {
      try {
        if (!assignCollections()) {
          break; // failed to assign oplogCollection or
          // slurpedCollection
        }

        Timestamp<?> startTimestamp = null;
        if (!definition.isSkipInitialImport()) {
          if (!riverHasIndexedFromOplog() && definition.getInitialTimestamp() == null) {
            if (!isIndexEmpty()) {
              MongoDBRiverHelper.setRiverStatus(
                  client, definition.getRiverName(), Status.INITIAL_IMPORT_FAILED);
              break;
            }
            if (definition.isImportAllCollections()) {
              for (String name : slurpedDb.getCollectionNames()) {
                DBCollection collection = slurpedDb.getCollection(name);
                startTimestamp = doInitialImport(collection);
              }
            } else {
              DBCollection collection = slurpedDb.getCollection(definition.getMongoCollection());
              startTimestamp = doInitialImport(collection);
            }
          }
        } else {
          logger.info("Skip initial import from collection {}", definition.getMongoCollection());
        }

        // Slurp from oplog
        DBCursor cursor = null;
        try {
          cursor = oplogCursor(startTimestamp);
          if (cursor == null) {
            cursor = processFullOplog();
          }
          while (cursor.hasNext()) {
            DBObject item = cursor.next();
            startTimestamp = processOplogEntry(item, startTimestamp);
          }
          logger.debug("Before waiting for 500 ms");
          Thread.sleep(500);
        } catch (MongoException.CursorNotFound e) {
          logger.info(
              "Cursor {} has been closed. About to open a new cusor.", cursor.getCursorId());
          logger.debug("Total document inserted [{}]", totalDocuments.get());
        } catch (SlurperException sEx) {
          logger.warn("Exception in slurper", sEx);
          break;
        } catch (Exception ex) {
          logger.warn("Exception while looping in cursor", ex);
          Thread.currentThread().interrupt();
          break;
        } finally {
          if (cursor != null) {
            logger.trace("Closing oplog cursor");
            cursor.close();
          }
        }
      } catch (MongoInterruptedException mIEx) {
        logger.warn("Mongo driver has been interrupted", mIEx);
        if (mongo != null) {
          mongo.close();
          mongo = null;
        }
        Thread.currentThread().interrupt();
        break;
      } catch (MongoException e) {
        logger.error("Mongo gave an exception", e);
        try {
          Thread.sleep(5000);
        } catch (InterruptedException iEx) {
        }
      } catch (NoSuchElementException e) {
        logger.warn("A mongoDB cursor bug ?", e);
      } catch (InterruptedException e) {
        logger.info("river-mongodb slurper interrupted");
        Thread.currentThread().interrupt();
        break;
      }
    }
  }

  protected boolean riverHasIndexedFromOplog() {
    return MongoDBRiver.getLastTimestamp(client, definition) != null;
  }

  protected boolean isIndexEmpty() {
    return MongoDBRiver.getIndexCount(client, definition) == 0;
  }

  /**
   * Does an initial sync the same way MongoDB does. https://groups.google.com/
   * forum/?fromgroups=#!topic/mongodb-user/sOKlhD_E2ns
   *
   * @return the last oplog timestamp before the import began
   * @throws InterruptedException if the blocking queue stream is interrupted while waiting
   */
  protected Timestamp<?> doInitialImport(DBCollection collection) throws InterruptedException {
    // TODO: ensure the index type is empty
    // DBCollection slurpedCollection =
    // slurpedDb.getCollection(definition.getMongoCollection());

    logger.info("MongoDBRiver is beginning initial import of " + collection.getFullName());
    Timestamp<?> startTimestamp = getCurrentOplogTimestamp();
    boolean inProgress = true;
    String lastId = null;
    while (inProgress) {
      DBCursor cursor = null;
      try {
        if (definition.isDisableIndexRefresh()) {
          updateIndexRefresh(definition.getIndexName(), -1L);
        }
        if (!definition.isMongoGridFS()) {
          logger.info("Collection {} - count: {}", collection.getName(), collection.count());
          long count = 0;
          cursor =
              collection.find(
                  getFilterForInitialImport(definition.getMongoCollectionFilter(), lastId));
          while (cursor.hasNext()) {
            DBObject object = cursor.next();
            count++;
            if (cursor.hasNext()) {
              lastId = addInsertToStream(null, applyFieldFilter(object), collection.getName());
            } else {
              logger.debug("Last entry for initial import - add timestamp: {}", startTimestamp);
              lastId =
                  addInsertToStream(startTimestamp, applyFieldFilter(object), collection.getName());
            }
          }
          inProgress = false;
          logger.info("Number documents indexed: {}", count);
        } else {
          // TODO: To be optimized.
          // https://github.com/mongodb/mongo-java-driver/pull/48#issuecomment-25241988
          // possible option: Get the object id list from .fs
          // collection
          // then call GriDFS.findOne
          GridFS grid =
              new GridFS(mongo.getDB(definition.getMongoDb()), definition.getMongoCollection());

          cursor = grid.getFileList();
          while (cursor.hasNext()) {
            DBObject object = cursor.next();
            if (object instanceof GridFSDBFile) {
              GridFSDBFile file =
                  grid.findOne(new ObjectId(object.get(MongoDBRiver.MONGODB_ID_FIELD).toString()));
              if (cursor.hasNext()) {
                lastId = addInsertToStream(null, file);
              } else {
                logger.debug("Last entry for initial import - add timestamp: {}", startTimestamp);
                lastId = addInsertToStream(startTimestamp, file);
              }
            }
          }
          inProgress = false;
        }
      } catch (MongoException.CursorNotFound e) {
        logger.info(
            "Initial import - Cursor {} has been closed. About to open a new cusor.",
            cursor.getCursorId());
        logger.debug("Total document inserted [{}]", totalDocuments.get());
      } finally {
        if (cursor != null) {
          logger.trace("Closing initial import cursor");
          cursor.close();
        }
        if (definition.isDisableIndexRefresh()) {
          updateIndexRefresh(definition.getIndexName(), TimeValue.timeValueSeconds(1));
        }
      }
    }
    return startTimestamp;
  }

  private BasicDBObject getFilterForInitialImport(BasicDBObject filter, String id) {
    if (id == null) {
      return filter;
    } else {
      BasicDBObject filterId =
          new BasicDBObject(
              MongoDBRiver.MONGODB_ID_FIELD, new BasicBSONObject(QueryOperators.GT, id));
      if (filter == null) {
        return filterId;
      } else {
        List<BasicDBObject> values = ImmutableList.of(filter, filterId);
        return new BasicDBObject(QueryOperators.AND, values);
      }
    }
  }

  protected boolean assignCollections() {
    DB adminDb = mongo.getDB(MongoDBRiver.MONGODB_ADMIN_DATABASE);
    oplogDb = mongo.getDB(MongoDBRiver.MONGODB_LOCAL_DATABASE);

    if (!definition.getMongoAdminUser().isEmpty()
        && !definition.getMongoAdminPassword().isEmpty()) {
      logger.info(
          "Authenticate {} with {}",
          MongoDBRiver.MONGODB_ADMIN_DATABASE,
          definition.getMongoAdminUser());

      CommandResult cmd =
          adminDb.authenticateCommand(
              definition.getMongoAdminUser(), definition.getMongoAdminPassword().toCharArray());
      if (!cmd.ok()) {
        logger.error(
            "Autenticatication failed for {}: {}",
            MongoDBRiver.MONGODB_ADMIN_DATABASE,
            cmd.getErrorMessage());
        // Can still try with mongoLocal credential if provided.
        // return false;
      }
      oplogDb = adminDb.getMongo().getDB(MongoDBRiver.MONGODB_LOCAL_DATABASE);
    }

    if (!definition.getMongoLocalUser().isEmpty()
        && !definition.getMongoLocalPassword().isEmpty()
        && !oplogDb.isAuthenticated()) {
      logger.info(
          "Authenticate {} with {}",
          MongoDBRiver.MONGODB_LOCAL_DATABASE,
          definition.getMongoLocalUser());
      CommandResult cmd =
          oplogDb.authenticateCommand(
              definition.getMongoLocalUser(), definition.getMongoLocalPassword().toCharArray());
      if (!cmd.ok()) {
        logger.error(
            "Autenticatication failed for {}: {}",
            MongoDBRiver.MONGODB_LOCAL_DATABASE,
            cmd.getErrorMessage());
        return false;
      }
    }

    Set<String> collections = oplogDb.getCollectionNames();
    if (!collections.contains(MongoDBRiver.OPLOG_COLLECTION)) {
      logger.error(
          "Cannot find "
              + MongoDBRiver.OPLOG_COLLECTION
              + " collection. Please check this link: http://goo.gl/2x5IW");
      return false;
    }
    oplogCollection = oplogDb.getCollection(MongoDBRiver.OPLOG_COLLECTION);

    slurpedDb = mongo.getDB(definition.getMongoDb());
    if (!definition.getMongoAdminUser().isEmpty()
        && !definition.getMongoAdminPassword().isEmpty()
        && adminDb.isAuthenticated()) {
      slurpedDb = adminDb.getMongo().getDB(definition.getMongoDb());
    }

    // Not necessary as local user has access to all databases.
    // http://docs.mongodb.org/manual/reference/local-database/
    // if (!mongoDbUser.isEmpty() && !mongoDbPassword.isEmpty()
    // && !slurpedDb.isAuthenticated()) {
    // logger.info("Authenticate {} with {}", mongoDb, mongoDbUser);
    // CommandResult cmd = slurpedDb.authenticateCommand(mongoDbUser,
    // mongoDbPassword.toCharArray());
    // if (!cmd.ok()) {
    // logger.error("Authentication failed for {}: {}",
    // mongoDb, cmd.getErrorMessage());
    // return false;
    // }
    // }
    // slurpedCollection =
    // slurpedDb.getCollection(definition.getMongoCollection());
    // if (definition.isImportAllCollections()) {
    // for (String collection : slurpedDb.getCollectionNames()) {
    // slurpedCollections.put(collection,
    // slurpedDb.getCollection(collection));
    // }
    // } else {
    // slurpedCollections.put(definition.getMongoCollection(),
    // slurpedDb.getCollection(definition.getMongoCollection()));
    // }

    return true;
  }

  private void updateIndexRefresh(String name, Object value) {
    client
        .admin()
        .indices()
        .prepareUpdateSettings(name)
        .setSettings(ImmutableMap.of("index.refresh_interval", value))
        .get();
  }

  private Timestamp<?> getCurrentOplogTimestamp() {
    return Timestamp.on(
        oplogCollection
            .find()
            .sort(new BasicDBObject(MongoDBRiver.INSERTION_ORDER_KEY, -1))
            .limit(1)
            .next());
  }

  private DBCursor processFullOplog() throws InterruptedException, SlurperException {
    Timestamp<?> currentTimestamp = getCurrentOplogTimestamp();
    addInsertToStream(currentTimestamp, null);
    return oplogCursor(currentTimestamp);
  }

  private Timestamp<?> processOplogEntry(final DBObject entry, final Timestamp<?> startTimestamp)
      throws InterruptedException {
    // To support transactions, TokuMX wraps one or more operations in a single oplog entry, in a
    // list.
    // As long as clients are not transaction-aware, we can pretty safely assume there will only be
    // one operation in the list.
    // Supporting genuine multi-operation transactions will require a bit more logic here.
    flattenOps(entry);

    if (!isValidOplogEntry(entry, startTimestamp)) {
      return startTimestamp;
    }
    Operation operation = Operation.fromString(entry.get(MongoDBRiver.OPLOG_OPERATION).toString());
    String namespace = entry.get(MongoDBRiver.OPLOG_NAMESPACE).toString();
    String collection = null;
    Timestamp<?> oplogTimestamp = Timestamp.on(entry);
    DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT);

    if (definition.isImportAllCollections()) {
      if (namespace.startsWith(definition.getMongoDb()) && !namespace.equals(cmdOplogNamespace)) {
        collection = getCollectionFromNamespace(namespace);
      }
    } else {
      collection = definition.getMongoCollection();
    }

    if (namespace.equals(cmdOplogNamespace)) {
      if (object.containsField(MongoDBRiver.OPLOG_DROP_COMMAND_OPERATION)) {
        operation = Operation.DROP_COLLECTION;
        if (definition.isImportAllCollections()) {
          collection = object.get(MongoDBRiver.OPLOG_DROP_COMMAND_OPERATION).toString();
          if (collection.startsWith("tmp.mr.")) {
            return startTimestamp;
          }
        }
      }
      if (object.containsField(MongoDBRiver.OPLOG_DROP_DATABASE_COMMAND_OPERATION)) {
        operation = Operation.DROP_DATABASE;
      }
    }

    logger.trace("namespace: {} - operation: {}", namespace, operation);
    if (namespace.equals(MongoDBRiver.OPLOG_ADMIN_COMMAND)) {
      if (operation == Operation.COMMAND) {
        processAdminCommandOplogEntry(entry, startTimestamp);
        return startTimestamp;
      }
    }

    if (logger.isTraceEnabled()) {
      logger.trace("MongoDB object deserialized: {}", object.toString());
      logger.trace("collection: {}", collection);
      logger.trace("oplog entry - namespace [{}], operation [{}]", namespace, operation);
      logger.trace("oplog processing item {}", entry);
    }

    String objectId = getObjectIdFromOplogEntry(entry);
    if (operation == Operation.DELETE) {
      // Include only _id in data, as vanilla MongoDB does, so transformation scripts won't be
      // broken by Toku
      if (object.containsField(MongoDBRiver.MONGODB_ID_FIELD)) {
        if (object.keySet().size() > 1) {
          entry.put(
              MongoDBRiver.OPLOG_OBJECT,
              object = new BasicDBObject(MongoDBRiver.MONGODB_ID_FIELD, objectId));
        }
      } else {
        throw new NullPointerException(MongoDBRiver.MONGODB_ID_FIELD);
      }
    }

    if (definition.isMongoGridFS()
        && namespace.endsWith(MongoDBRiver.GRIDFS_FILES_SUFFIX)
        && (operation == Operation.INSERT || operation == Operation.UPDATE)) {
      if (objectId == null) {
        throw new NullPointerException(MongoDBRiver.MONGODB_ID_FIELD);
      }
      GridFS grid = new GridFS(mongo.getDB(definition.getMongoDb()), collection);
      GridFSDBFile file = grid.findOne(new ObjectId(objectId));
      if (file != null) {
        logger.info("Caught file: {} - {}", file.getId(), file.getFilename());
        object = file;
      } else {
        logger.warn("Cannot find file from id: {}", objectId);
      }
    }

    if (object instanceof GridFSDBFile) {
      if (objectId == null) {
        throw new NullPointerException(MongoDBRiver.MONGODB_ID_FIELD);
      }
      if (logger.isTraceEnabled()) {
        logger.trace("Add attachment: {}", objectId);
      }
      addToStream(operation, oplogTimestamp, applyFieldFilter(object), collection);
    } else {
      if (operation == Operation.UPDATE) {
        DBObject update = (DBObject) entry.get(MongoDBRiver.OPLOG_UPDATE);
        logger.debug("Updated item: {}", update);
        addQueryToStream(operation, oplogTimestamp, update, collection);
      } else {
        if (operation == Operation.INSERT) {
          addInsertToStream(oplogTimestamp, applyFieldFilter(object), collection);
        } else {
          addToStream(operation, oplogTimestamp, applyFieldFilter(object), collection);
        }
      }
    }
    return oplogTimestamp;
  }

  @SuppressWarnings("unchecked")
  private void flattenOps(DBObject entry) {
    Object ops = entry.get(MongoDBRiver.OPLOG_OPS);
    if (ops != null) {
      try {
        for (DBObject op : (List<DBObject>) ops) {
          String operation = (String) op.get(MongoDBRiver.OPLOG_OPERATION);
          if (operation.equals(MongoDBRiver.OPLOG_COMMAND_OPERATION)) {
            DBObject object = (DBObject) op.get(MongoDBRiver.OPLOG_OBJECT);
            if (object.containsField(MongoDBRiver.OPLOG_CREATE_COMMAND)) {
              continue;
            }
          }
          entry.putAll(op);
        }
      } catch (ClassCastException e) {
        logger.error(e.toString(), e);
      }
    }
  }

  private void processAdminCommandOplogEntry(
      final DBObject entry, final Timestamp<?> startTimestamp) throws InterruptedException {
    if (logger.isTraceEnabled()) {
      logger.trace("processAdminCommandOplogEntry - [{}]", entry);
    }
    DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT);
    if (definition.isImportAllCollections()) {
      if (object.containsField(MongoDBRiver.OPLOG_RENAME_COLLECTION_COMMAND_OPERATION)
          && object.containsField(MongoDBRiver.OPLOG_TO)) {
        String to = object.get(MongoDBRiver.OPLOG_TO).toString();
        if (to.startsWith(definition.getMongoDb())) {
          String newCollection = getCollectionFromNamespace(to);
          DBCollection coll = slurpedDb.getCollection(newCollection);
          doInitialImport(coll);
        }
      }
    }
  }

  private String getCollectionFromNamespace(String namespace) {
    if (namespace.startsWith(definition.getMongoDb())
        && CharMatcher.is('.').countIn(namespace) == 1) {
      return namespace.substring(definition.getMongoDb().length() + 1);
    }
    logger.info("Cannot get collection from namespace [{}]", namespace);
    return null;
  }

  private boolean isValidOplogEntry(final DBObject entry, final Timestamp<?> startTimestamp) {
    if (MongoDBRiver.OPLOG_NOOP_OPERATION.equals(entry.get(MongoDBRiver.OPLOG_OPERATION))) {
      logger.debug("[No-op Oplog Entry] - can be ignored. {}", entry);
      return false;
    }
    String namespace = (String) entry.get(MongoDBRiver.OPLOG_NAMESPACE);
    // Initial support for sharded collection -
    // https://jira.mongodb.org/browse/SERVER-4333
    // Not interested in operation from migration or sharding
    if (entry.containsField(MongoDBRiver.OPLOG_FROM_MIGRATE)
        && ((BasicBSONObject) entry).getBoolean(MongoDBRiver.OPLOG_FROM_MIGRATE)) {
      logger.debug(
          "[Invalid Oplog Entry] - from migration or sharding operation. Can be ignored. {}",
          entry);
      return false;
    }
    // Not interested by chunks - skip all
    if (namespace.endsWith(MongoDBRiver.GRIDFS_CHUNKS_SUFFIX)) {
      return false;
    }

    if (startTimestamp != null) {
      Timestamp<?> oplogTimestamp = Timestamp.on(entry);
      if (Timestamp.compare(oplogTimestamp, startTimestamp) < 0) {
        logger.debug(
            "[Invalid Oplog Entry] - entry timestamp [{}] before startTimestamp [{}]",
            entry,
            startTimestamp);
        return false;
      }
    }

    boolean validNamespace = false;
    if (definition.isMongoGridFS()) {
      validNamespace = gridfsOplogNamespace.equals(namespace);
    } else {
      if (definition.isImportAllCollections()) {
        // Skip temp entry generated by map / reduce
        if (namespace.startsWith(definition.getMongoDb())
            && !namespace.startsWith(definition.getMongoDb() + ".tmp.mr")) {
          validNamespace = true;
        }
      } else {
        if (definition.getMongoOplogNamespace().equals(namespace)) {
          validNamespace = true;
        }
      }
      if (cmdOplogNamespace.equals(namespace)) {
        validNamespace = true;
      }

      if (MongoDBRiver.OPLOG_ADMIN_COMMAND.equals(namespace)) {
        validNamespace = true;
      }
    }
    if (!validNamespace) {
      logger.debug("[Invalid Oplog Entry] - namespace [{}] is not valid", namespace);
      return false;
    }
    String operation = (String) entry.get(MongoDBRiver.OPLOG_OPERATION);
    if (!oplogOperations.contains(operation)) {
      logger.debug("[Invalid Oplog Entry] - operation [{}] is not valid", operation);
      return false;
    }

    // TODO: implement a better solution
    if (definition.getMongoOplogFilter() != null) {
      DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT);
      BasicDBObject filter = definition.getMongoOplogFilter();
      if (!filterMatch(filter, object)) {
        logger.debug(
            "[Invalid Oplog Entry] - filter [{}] does not match object [{}]", filter, object);
        return false;
      }
    }
    return true;
  }

  private boolean filterMatch(DBObject filter, DBObject object) {
    for (String key : filter.keySet()) {
      if (!object.containsField(key)) {
        return false;
      }
      if (!filter.get(key).equals(object.get(key))) {
        return false;
      }
    }
    return true;
  }

  private DBObject applyFieldFilter(DBObject object) {
    if (object instanceof GridFSFile) {
      GridFSFile file = (GridFSFile) object;
      DBObject metadata = file.getMetaData();
      if (metadata != null) {
        file.setMetaData(applyFieldFilter(metadata));
      }
    } else {
      object = MongoDBHelper.applyExcludeFields(object, definition.getExcludeFields());
      object = MongoDBHelper.applyIncludeFields(object, definition.getIncludeFields());
    }
    return object;
  }

  /*
   * Extract "_id" from "o" if it fails try to extract from "o2"
   */
  private String getObjectIdFromOplogEntry(DBObject entry) {
    if (entry.containsField(MongoDBRiver.OPLOG_OBJECT)) {
      DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_OBJECT);
      if (object.containsField(MongoDBRiver.MONGODB_ID_FIELD)) {
        return object.get(MongoDBRiver.MONGODB_ID_FIELD).toString();
      }
    }
    if (entry.containsField(MongoDBRiver.OPLOG_UPDATE)) {
      DBObject object = (DBObject) entry.get(MongoDBRiver.OPLOG_UPDATE);
      if (object.containsField(MongoDBRiver.MONGODB_ID_FIELD)) {
        return object.get(MongoDBRiver.MONGODB_ID_FIELD).toString();
      }
    }
    return null;
  }

  private DBCursor oplogCursor(final Timestamp<?> timestampOverride) throws SlurperException {
    Timestamp<?> time =
        timestampOverride == null
            ? MongoDBRiver.getLastTimestamp(client, definition)
            : timestampOverride;
    DBObject indexFilter = time.getOplogFilter();
    if (indexFilter == null) {
      return null;
    }

    int options =
        Bytes.QUERYOPTION_TAILABLE | Bytes.QUERYOPTION_AWAITDATA | Bytes.QUERYOPTION_NOTIMEOUT;

    // Using OPLOGREPLAY to improve performance:
    // https://jira.mongodb.org/browse/JAVA-771
    if (indexFilter.containsField(MongoDBRiver.OPLOG_TIMESTAMP)) {
      options = options | Bytes.QUERYOPTION_OPLOGREPLAY;
    }
    DBCursor cursor = oplogCollection.find(indexFilter).setOptions(options);
    isRiverStale(cursor, time);
    return cursor;
  }

  private void isRiverStale(DBCursor cursor, Timestamp<?> time) throws SlurperException {
    if (cursor == null || time == null) {
      return;
    }
    if (definition.getInitialTimestamp() != null && time.equals(definition.getInitialTimestamp())) {
      return;
    }
    DBObject entry = cursor.next();
    Timestamp<?> oplogTimestamp = Timestamp.on(entry);
    if (!time.equals(oplogTimestamp)) {
      MongoDBRiverHelper.setRiverStatus(client, definition.getRiverName(), Status.RIVER_STALE);
      throw new SlurperException("River out of sync with oplog.rs collection");
    }
  }

  private void addQueryToStream(
      final Operation operation,
      final Timestamp<?> currentTimestamp,
      final DBObject update,
      final String collection)
      throws InterruptedException {
    if (logger.isTraceEnabled()) {
      logger.trace(
          "addQueryToStream - operation [{}], currentTimestamp [{}], update [{}]",
          operation,
          currentTimestamp,
          update);
    }

    if (collection == null) {
      for (String name : slurpedDb.getCollectionNames()) {
        DBCollection slurpedCollection = slurpedDb.getCollection(name);
        for (DBObject item : slurpedCollection.find(update, findKeys)) {
          addToStream(operation, currentTimestamp, item, collection);
        }
      }
    } else {
      DBCollection slurpedCollection = slurpedDb.getCollection(collection);
      for (DBObject item : slurpedCollection.find(update, findKeys)) {
        addToStream(operation, currentTimestamp, item, collection);
      }
    }
  }

  private String addInsertToStream(final Timestamp<?> currentTimestamp, final DBObject data)
      throws InterruptedException {
    return addInsertToStream(currentTimestamp, data, definition.getMongoCollection());
  }

  private String addInsertToStream(
      final Timestamp<?> currentTimestamp, final DBObject data, final String collection)
      throws InterruptedException {
    totalDocuments.incrementAndGet();
    addToStream(Operation.INSERT, currentTimestamp, data, collection);
    return data.containsField(MongoDBRiver.MONGODB_ID_FIELD)
        ? data.get(MongoDBRiver.MONGODB_ID_FIELD).toString()
        : null;
  }

  private void addToStream(
      final Operation operation,
      final Timestamp<?> currentTimestamp,
      final DBObject data,
      final String collection)
      throws InterruptedException {
    if (logger.isTraceEnabled()) {
      logger.trace(
          "addToStream - operation [{}], currentTimestamp [{}], data [{}], collection [{}]",
          operation,
          currentTimestamp,
          data,
          collection);
    }

    if (operation == Operation.DROP_DATABASE) {
      if (definition.isImportAllCollections()) {
        for (String name : slurpedDb.getCollectionNames()) {
          context
              .getStream()
              .put(
                  new MongoDBRiver.QueueEntry(
                      currentTimestamp, Operation.DROP_COLLECTION, data, name));
        }
      } else {
        context
            .getStream()
            .put(
                new MongoDBRiver.QueueEntry(
                    currentTimestamp, Operation.DROP_COLLECTION, data, collection));
      }
    } else {
      context
          .getStream()
          .put(new MongoDBRiver.QueueEntry(currentTimestamp, operation, data, collection));
    }
  }
}
/** @author kimchy (shay.banon) */
public class InternalIntTermsFacet extends InternalTermsFacet {

  private static final String STREAM_TYPE = "iTerms";

  public static void registerStream() {
    Streams.registerStream(STREAM, STREAM_TYPE);
  }

  static Stream STREAM =
      new Stream() {
        @Override
        public Facet readFacet(String type, StreamInput in) throws IOException {
          return readTermsFacet(in);
        }
      };

  @Override
  public String streamType() {
    return STREAM_TYPE;
  }

  public static class IntEntry implements Entry {

    int term;
    int count;

    public IntEntry(int term, int count) {
      this.term = term;
      this.count = count;
    }

    public String term() {
      return Integer.toString(term);
    }

    public String getTerm() {
      return term();
    }

    @Override
    public Number termAsNumber() {
      return term;
    }

    @Override
    public Number getTermAsNumber() {
      return termAsNumber();
    }

    public int count() {
      return count;
    }

    public int getCount() {
      return count();
    }

    @Override
    public int compareTo(Entry o) {
      int anotherVal = ((IntEntry) o).term;
      int i = term - anotherVal;
      if (i == 0) {
        i = count - o.count();
        if (i == 0) {
          i = System.identityHashCode(this) - System.identityHashCode(o);
        }
      }
      return i;
    }
  }

  private String name;

  int requiredSize;

  long missing;

  Collection<IntEntry> entries = ImmutableList.of();

  ComparatorType comparatorType;

  InternalIntTermsFacet() {}

  public InternalIntTermsFacet(
      String name,
      ComparatorType comparatorType,
      int requiredSize,
      Collection<IntEntry> entries,
      long missing) {
    this.name = name;
    this.comparatorType = comparatorType;
    this.requiredSize = requiredSize;
    this.entries = entries;
    this.missing = missing;
  }

  @Override
  public String name() {
    return this.name;
  }

  @Override
  public String getName() {
    return this.name;
  }

  @Override
  public String type() {
    return TYPE;
  }

  @Override
  public String getType() {
    return type();
  }

  @Override
  public List<IntEntry> entries() {
    if (!(entries instanceof List)) {
      entries = ImmutableList.copyOf(entries);
    }
    return (List<IntEntry>) entries;
  }

  @Override
  public List<IntEntry> getEntries() {
    return entries();
  }

  @SuppressWarnings({"unchecked"})
  @Override
  public Iterator<Entry> iterator() {
    return (Iterator) entries.iterator();
  }

  @Override
  public long missingCount() {
    return this.missing;
  }

  @Override
  public long getMissingCount() {
    return missingCount();
  }

  private static ThreadLocal<ThreadLocals.CleanableValue<TIntIntHashMap>> aggregateCache =
      new ThreadLocal<ThreadLocals.CleanableValue<TIntIntHashMap>>() {
        @Override
        protected ThreadLocals.CleanableValue<TIntIntHashMap> initialValue() {
          return new ThreadLocals.CleanableValue<TIntIntHashMap>(new TIntIntHashMap());
        }
      };

  @Override
  public Facet reduce(String name, List<Facet> facets) {
    if (facets.size() == 1) {
      return facets.get(0);
    }
    InternalIntTermsFacet first = (InternalIntTermsFacet) facets.get(0);
    TIntIntHashMap aggregated = aggregateCache.get().get();
    aggregated.clear();
    long missing = 0;

    for (Facet facet : facets) {
      InternalIntTermsFacet mFacet = (InternalIntTermsFacet) facet;
      missing += mFacet.missingCount();
      for (IntEntry entry : mFacet.entries) {
        aggregated.adjustOrPutValue(entry.term, entry.count(), entry.count());
      }
    }

    BoundedTreeSet<IntEntry> ordered =
        new BoundedTreeSet<IntEntry>(first.comparatorType.comparator(), first.requiredSize);
    for (TIntIntIterator it = aggregated.iterator(); it.hasNext(); ) {
      it.advance();
      ordered.add(new IntEntry(it.key(), it.value()));
    }
    first.entries = ordered;
    first.missing = missing;
    return first;
  }

  static final class Fields {
    static final XContentBuilderString _TYPE = new XContentBuilderString("_type");
    static final XContentBuilderString MISSING = new XContentBuilderString("missing");
    static final XContentBuilderString TERMS = new XContentBuilderString("terms");
    static final XContentBuilderString TERM = new XContentBuilderString("term");
    static final XContentBuilderString COUNT = new XContentBuilderString("count");
  }

  @Override
  public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
    builder.startObject(name);
    builder.field(Fields._TYPE, TermsFacet.TYPE);
    builder.field(Fields.MISSING, missing);
    builder.startArray(Fields.TERMS);
    for (IntEntry entry : entries) {
      builder.startObject();
      builder.field(Fields.TERM, entry.term);
      builder.field(Fields.COUNT, entry.count());
      builder.endObject();
    }
    builder.endArray();
    builder.endObject();
    return builder;
  }

  public static InternalIntTermsFacet readTermsFacet(StreamInput in) throws IOException {
    InternalIntTermsFacet facet = new InternalIntTermsFacet();
    facet.readFrom(in);
    return facet;
  }

  @Override
  public void readFrom(StreamInput in) throws IOException {
    name = in.readUTF();
    comparatorType = ComparatorType.fromId(in.readByte());
    requiredSize = in.readVInt();
    missing = in.readVLong();

    int size = in.readVInt();
    entries = new ArrayList<IntEntry>(size);
    for (int i = 0; i < size; i++) {
      entries.add(new IntEntry(in.readInt(), in.readVInt()));
    }
  }

  @Override
  public void writeTo(StreamOutput out) throws IOException {
    out.writeUTF(name);
    out.writeByte(comparatorType.id());
    out.writeVInt(requiredSize);
    out.writeVLong(missing);

    out.writeVInt(entries.size());
    for (IntEntry entry : entries) {
      out.writeInt(entry.term);
      out.writeVInt(entry.count());
    }
  }
}