示例#1
0
  @Override
  public void testStarted() {
    if (log.isDebugEnabled()) {
      log.debug(getTitle() + " testStarted");
    }

    MongoClientOptions.Builder builder =
        MongoClientOptions.builder()
            .autoConnectRetry(getAutoConnectRetry())
            .connectTimeout(getConnectTimeout())
            .connectionsPerHost(getConnectionsPerHost())
            .maxAutoConnectRetryTime(getMaxAutoConnectRetryTime())
            .maxWaitTime(getMaxWaitTime())
            .socketKeepAlive(getSocketKeepAlive())
            .socketTimeout(getSocketTimeout())
            .threadsAllowedToBlockForConnectionMultiplier(
                getThreadsAllowedToBlockForConnectionMultiplier());

    if (getSafe()) {
      builder.writeConcern(WriteConcern.SAFE);
    } else {
      builder.writeConcern(
          new WriteConcern(
              getWriteOperationNumberOfServers(),
              getWriteOperationTimeout(),
              getFsync(),
              getWaitForJournaling(),
              getContinueOnInsertError()));
    }
    MongoClientOptions mongoOptions = builder.build();

    if (log.isDebugEnabled()) {
      log.debug("options : " + mongoOptions.toString());
    }

    if (getThreadContext().getVariables().getObject(getSource()) != null) {
      if (log.isWarnEnabled()) {
        log.warn(getSource() + " has already been defined.");
      }
    } else {
      if (log.isDebugEnabled()) {
        log.debug(getSource() + "  is being defined.");
      }
      try {
        getThreadContext()
            .getVariables()
            .putObject(
                getSource(),
                new MongoDB(MongoUtils.toServerAddresses(getConnection()), mongoOptions));
      } catch (UnknownHostException e) {
        throw new IllegalStateException(e);
      }
    }
  }
  protected void connect() {

    if (this.mongoClient == null) {
      try {

        List<ServerAddress> addresses = new ArrayList<ServerAddress>();

        for (String host : this.servers) {
          addresses.add(new ServerAddress(host, 27017));
        }

        this.mongoClient =
            new MongoClient(
                addresses,
                MongoClientOptions.builder()
                    .connectionsPerHost(30)
                    .threadsAllowedToBlockForConnectionMultiplier(20)
                    .build());

        this.db = this.mongoClient.getDB("twitter");

        this.dbCollection = this.db.getCollection("tweets");
      } catch (Exception e) {
        this.log.write(
            "MongoDBConnector - Exception in MongoDBConnector.connect: "
                + e.getMessage()
                + ": "
                + e.toString());
      }
    }
  }
 /**
  * Creates a {@link MongoClient} using the given {@code options} and {@code environment}. If the
  * configured port is zero, the value of the {@code local.mongo.port} property retrieved from the
  * {@code environment} is used to configure the client.
  *
  * @param options the options
  * @param environment the environment
  * @return the Mongo client
  * @throws UnknownHostException if the configured host is unknown
  */
 public MongoClient createMongoClient(MongoClientOptions options, Environment environment)
     throws UnknownHostException {
   try {
     if (hasCustomAddress() || hasCustomCredentials()) {
       if (options == null) {
         options = MongoClientOptions.builder().build();
       }
       List<MongoCredential> credentials = null;
       if (hasCustomCredentials()) {
         String database =
             this.authenticationDatabase == null
                 ? getMongoClientDatabase()
                 : this.authenticationDatabase;
         credentials =
             Arrays.asList(
                 MongoCredential.createMongoCRCredential(this.username, database, this.password));
       }
       String host = this.host == null ? "localhost" : this.host;
       int port = determinePort(environment);
       return new MongoClient(Arrays.asList(new ServerAddress(host, port)), credentials, options);
     }
     // The options and credentials are in the URI
     return new MongoClient(new MongoClientURI(this.uri, builder(options)));
   } finally {
     clearPassword();
   }
 }
  @Override
  @SuppressWarnings("serial")
  public BasicConnectionFactory<MongoDBConnectionImpl> createConnectionFactory()
      throws ResourceException {
    if (this.remoteServerList == null) {
      throw new InvalidPropertyException(UTIL.getString("no_server")); // $NON-NLS-1$
    }
    if (this.database == null) {
      throw new InvalidPropertyException(UTIL.getString("no_database")); // $NON-NLS-1$
    }

    final List<ServerAddress> servers = getServers();

    // TODO: need to define all the properties on the ra.xml and build this correctly
    final MongoClientOptions options = MongoClientOptions.builder().build();

    return new BasicConnectionFactory<MongoDBConnectionImpl>() {
      @Override
      public MongoDBConnectionImpl getConnection() throws ResourceException {
        MongoCredential credential = null;
        if (MongoDBManagedConnectionFactory.this.username != null
            && MongoDBManagedConnectionFactory.this.password != null) {
          credential =
              MongoCredential.createMongoCRCredential(
                  MongoDBManagedConnectionFactory.this.username,
                  MongoDBManagedConnectionFactory.this.database,
                  MongoDBManagedConnectionFactory.this.password.toCharArray());
        }
        return new MongoDBConnectionImpl(
            MongoDBManagedConnectionFactory.this.database, servers, credential, options);
      }
    };
  }
示例#5
0
 private MongoClient createClient(final String databaseUri, final int port)
     throws UnknownHostException {
   final MongoClientOptions options =
       MongoClientOptions.builder()
           .connectionsPerHost(100)
           .threadsAllowedToBlockForConnectionMultiplier(1500)
           .build();
   final MongoClient client = new MongoClient(new ServerAddress(databaseUri, port), options);
   return client;
 }
示例#6
0
 /** Creates a connection to a running MongoDB instance using the required codecs. */
 public DutyScheduleDB() {
   // Create codec registry with LocalDateCodec.
   CodecRegistry codecRegistry =
       CodecRegistries.fromRegistries(
           CodecRegistries.fromCodecs(new LocalDateCodec()),
           CodecRegistries.fromProviders(
               new RaCodecProvider(),
               new DutyBlockCodecProvider(),
               new ScheduledDutyCodecProvider()),
           MongoClient.getDefaultCodecRegistry());
   MongoClientOptions options = MongoClientOptions.builder().codecRegistry(codecRegistry).build();
   mongoClient = new MongoClient(new ServerAddress(), options);
   db = mongoClient.getDatabase("DutySchedulerDB");
 }
  private static MongoClientOptions configureMongoOptions() {
    final MongoClientOptions.Builder options = MongoClientOptions.builder();

    for (ApplicationClass clazz : Play.classes.getAssignableClasses(MongoConfigurator.class)) {
      try {
        final MongoConfigurator configurator = (MongoConfigurator) clazz.javaClass.newInstance();
        configurator.configureOptions(options);
      } catch (InstantiationException e) {
        throw new ConfigurationException(
            String.format("failed to create configurator: %s", clazz.name));
      } catch (IllegalAccessException e) {
        throw new ConfigurationException(
            String.format("failed to create configurator: %s", clazz.name));
      }
    }

    return options.build();
  }
  @Before
  public void beforeEachTest() {

    CodecRegistry codecRegistry =
        CodecRegistries.fromRegistries(
            MongoClient.getDefaultCodecRegistry(),
            CodecRegistries.fromCodecs(
                new LocalDateTimeCodec(), new HostAndPortCodec(), new LocalDateTimeCodec()));

    MongoProperties properties = new MongoProperties();

    MongoClient mongoClient =
        new MongoClient(
            new MongoClientURI(
                properties.getUri(), MongoClientOptions.builder().codecRegistry(codecRegistry)));

    mongoMetricsRepository =
        new MongoMetricsRepository(mongoClient, properties, new ObjectMapper());

    mongoMetricsRepository.init();

    metricsService = new MetricsService(mongoMetricsRepository);
  }
示例#9
0
 @SuppressWarnings("deprecation")
 @Override
 public Mongo get() {
   try {
     String mongodbAddress = configuration.getString("mongodb.address");
     String[] addresses = mongodbAddress.split(",");
     ArrayList<ServerAddress> servers = new ArrayList<ServerAddress>();
     for (String address : addresses) {
       String[] addPort = address.split(":");
       String add = addPort[0];
       int port = addPort.length > 1 ? Integer.parseInt(addPort[1]) : 27017;
       servers.add(new ServerAddress(add, port));
     }
     LOGGER.info("mongo servers:" + mongodbAddress);
     //            MongoOptions mongoOptions = new MongoOptions();
     //            mongoOptions.connectionsPerHost =
     // this.configuration.getInt("mongodb.connectionsPerHost", 60);
     //            mongoOptions.socketTimeout = 60000;
     //            mongoOptions.connectTimeout = 30000;
     //            mongoOptions.autoConnectRetry = true;
     //            mongoOptions.socketKeepAlive = true;
     //            Mongo mongo = new Mongo(servers, mongoOptions);
     MongoClientOptions.Builder builder = MongoClientOptions.builder();
     builder
         .socketKeepAlive(true)
         .socketTimeout(60000)
         .connectTimeout(30000)
         .autoConnectRetry(true)
         .connectionsPerHost(this.configuration.getInt("mongodb.connectionsPerHost", 60));
     MongoClient mongoClient = new MongoClient(servers, builder.build());
     if (isReadSecondary) mongoClient.setReadPreference(ReadPreference.secondaryPreferred());
     return mongoClient;
   } catch (Throwable e) {
     LOGGER.error("failed to init mongodb", e);
     throw new ExceptionInInitializerError(e);
   }
 }
  @SuppressWarnings("unchecked")
  public static synchronized MongoDBRiverDefinition parseSettings(
      String riverName,
      String riverIndexName,
      RiverSettings settings,
      ScriptService scriptService) {

    logger.trace("Parse river settings for {}", riverName);
    Preconditions.checkNotNull(riverName, "No riverName specified");
    Preconditions.checkNotNull(riverIndexName, "No riverIndexName specified");
    Preconditions.checkNotNull(settings, "No settings specified");

    Builder builder = new Builder();
    builder.riverName(riverName);
    builder.riverIndexName(riverIndexName);

    List<ServerAddress> mongoServers = new ArrayList<ServerAddress>();
    String mongoHost;
    int mongoPort;

    if (settings.settings().containsKey(MongoDBRiver.TYPE)) {
      Map<String, Object> mongoSettings =
          (Map<String, Object>) settings.settings().get(MongoDBRiver.TYPE);
      if (mongoSettings.containsKey(SERVERS_FIELD)) {
        Object mongoServersSettings = mongoSettings.get(SERVERS_FIELD);
        logger.trace("mongoServersSettings: " + mongoServersSettings);
        boolean array = XContentMapValues.isArray(mongoServersSettings);

        if (array) {
          ArrayList<Map<String, Object>> feeds =
              (ArrayList<Map<String, Object>>) mongoServersSettings;
          for (Map<String, Object> feed : feeds) {
            mongoHost = XContentMapValues.nodeStringValue(feed.get(HOST_FIELD), null);
            mongoPort = XContentMapValues.nodeIntegerValue(feed.get(PORT_FIELD), DEFAULT_DB_PORT);
            logger.trace("Server: " + mongoHost + " - " + mongoPort);
            try {
              mongoServers.add(new ServerAddress(mongoHost, mongoPort));
            } catch (UnknownHostException uhEx) {
              logger.warn("Cannot add mongo server {}:{}", uhEx, mongoHost, mongoPort);
            }
          }
        }
      } else {
        mongoHost =
            XContentMapValues.nodeStringValue(mongoSettings.get(HOST_FIELD), DEFAULT_DB_HOST);
        mongoPort =
            XContentMapValues.nodeIntegerValue(mongoSettings.get(PORT_FIELD), DEFAULT_DB_PORT);
        try {
          mongoServers.add(new ServerAddress(mongoHost, mongoPort));
        } catch (UnknownHostException uhEx) {
          logger.warn("Cannot add mongo server {}:{}", uhEx, mongoHost, mongoPort);
        }
      }
      builder.mongoServers(mongoServers);

      MongoClientOptions.Builder mongoClientOptionsBuilder =
          MongoClientOptions.builder().socketKeepAlive(true);

      // MongoDB options
      if (mongoSettings.containsKey(OPTIONS_FIELD)) {
        Map<String, Object> mongoOptionsSettings =
            (Map<String, Object>) mongoSettings.get(OPTIONS_FIELD);
        logger.trace("mongoOptionsSettings: " + mongoOptionsSettings);
        builder.mongoSecondaryReadPreference(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(SECONDARY_READ_PREFERENCE_FIELD), false));
        builder.connectTimeout(
            XContentMapValues.nodeIntegerValue(
                mongoOptionsSettings.get(CONNECT_TIMEOUT), DEFAULT_CONNECT_TIMEOUT));
        builder.socketTimeout(
            XContentMapValues.nodeIntegerValue(
                mongoOptionsSettings.get(SOCKET_TIMEOUT), DEFAULT_SOCKET_TIMEOUT));
        builder.dropCollection(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(DROP_COLLECTION_FIELD), false));
        String isMongos =
            XContentMapValues.nodeStringValue(mongoOptionsSettings.get(IS_MONGOS_FIELD), null);
        if (isMongos != null) {
          builder.isMongos(Boolean.valueOf(isMongos));
        }
        builder.mongoUseSSL(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(SSL_CONNECTION_FIELD), false));
        builder.mongoSSLVerifyCertificate(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(SSL_VERIFY_CERT_FIELD), true));
        builder.advancedTransformation(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(ADVANCED_TRANSFORMATION_FIELD), false));
        builder.skipInitialImport(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(SKIP_INITIAL_IMPORT_FIELD), false));
        builder.connectionsPerHost(
            XContentMapValues.nodeIntegerValue(
                mongoOptionsSettings.get(CONNECTIONS_PER_HOST), DEFAULT_CONNECTIONS_PER_HOST));
        builder.threadsAllowedToBlockForConnectionMultiplier(
            XContentMapValues.nodeIntegerValue(
                mongoOptionsSettings.get(THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER),
                DEFAULT_THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER));

        mongoClientOptionsBuilder
            .connectTimeout(builder.connectTimeout)
            .socketTimeout(builder.socketTimeout)
            .connectionsPerHost(builder.connectionsPerHost)
            .threadsAllowedToBlockForConnectionMultiplier(
                builder.threadsAllowedToBlockForConnectionMultiplier);

        if (builder.mongoSecondaryReadPreference) {
          mongoClientOptionsBuilder.readPreference(ReadPreference.secondaryPreferred());
        }

        if (builder.mongoUseSSL) {
          mongoClientOptionsBuilder.socketFactory(getSSLSocketFactory());
        }

        if (mongoOptionsSettings.containsKey(PARENT_TYPES_FIELD)) {
          Set<String> parentTypes = new HashSet<String>();
          Object parentTypesSettings = mongoOptionsSettings.get(PARENT_TYPES_FIELD);
          logger.trace("parentTypesSettings: " + parentTypesSettings);
          boolean array = XContentMapValues.isArray(parentTypesSettings);

          if (array) {
            ArrayList<String> fields = (ArrayList<String>) parentTypesSettings;
            for (String field : fields) {
              logger.trace("Field: " + field);
              parentTypes.add(field);
            }
          }

          builder.parentTypes(parentTypes);
        }

        if (mongoOptionsSettings.containsKey(STORE_STATISTICS_FIELD)) {
          Object storeStatistics = mongoOptionsSettings.get(STORE_STATISTICS_FIELD);
          boolean object = XContentMapValues.isObject(storeStatistics);
          if (object) {
            Map<String, Object> storeStatisticsSettings = (Map<String, Object>) storeStatistics;
            builder.storeStatistics(true);
            builder.statisticsIndexName(
                XContentMapValues.nodeStringValue(
                    storeStatisticsSettings.get(INDEX_OBJECT), riverName + "-stats"));
            builder.statisticsTypeName(
                XContentMapValues.nodeStringValue(
                    storeStatisticsSettings.get(TYPE_FIELD), "stats"));
          } else {
            builder.storeStatistics(XContentMapValues.nodeBooleanValue(storeStatistics, false));
            if (builder.storeStatistics) {
              builder.statisticsIndexName(riverName + "-stats");
              builder.statisticsTypeName("stats");
            }
          }
        }
        // builder.storeStatistics(XContentMapValues.nodeBooleanValue(mongoOptionsSettings.get(STORE_STATISTICS_FIELD),
        // false));
        builder.importAllCollections(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(IMPORT_ALL_COLLECTIONS_FIELD), false));
        builder.disableIndexRefresh(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(DISABLE_INDEX_REFRESH_FIELD), false));
        builder.includeCollection(
            XContentMapValues.nodeStringValue(
                mongoOptionsSettings.get(INCLUDE_COLLECTION_FIELD), ""));

        if (mongoOptionsSettings.containsKey(INCLUDE_FIELDS_FIELD)) {
          Set<String> includeFields = new HashSet<String>();
          Object includeFieldsSettings = mongoOptionsSettings.get(INCLUDE_FIELDS_FIELD);
          logger.trace("includeFieldsSettings: " + includeFieldsSettings);
          boolean array = XContentMapValues.isArray(includeFieldsSettings);

          if (array) {
            ArrayList<String> fields = (ArrayList<String>) includeFieldsSettings;
            for (String field : fields) {
              logger.trace("Field: " + field);
              includeFields.add(field);
            }
          }

          if (!includeFields.contains(MongoDBRiver.MONGODB_ID_FIELD)) {
            includeFields.add(MongoDBRiver.MONGODB_ID_FIELD);
          }
          builder.includeFields(includeFields);
        } else if (mongoOptionsSettings.containsKey(EXCLUDE_FIELDS_FIELD)) {
          Set<String> excludeFields = new HashSet<String>();
          Object excludeFieldsSettings = mongoOptionsSettings.get(EXCLUDE_FIELDS_FIELD);
          logger.trace("excludeFieldsSettings: " + excludeFieldsSettings);
          boolean array = XContentMapValues.isArray(excludeFieldsSettings);

          if (array) {
            ArrayList<String> fields = (ArrayList<String>) excludeFieldsSettings;
            for (String field : fields) {
              logger.trace("Field: " + field);
              excludeFields.add(field);
            }
          }

          builder.excludeFields(excludeFields);
        }

        if (mongoOptionsSettings.containsKey(INITIAL_TIMESTAMP_FIELD)) {
          BSONTimestamp timeStamp = null;
          try {
            Map<String, Object> initalTimestampSettings =
                (Map<String, Object>) mongoOptionsSettings.get(INITIAL_TIMESTAMP_FIELD);
            String scriptType = "js";
            if (initalTimestampSettings.containsKey(INITIAL_TIMESTAMP_SCRIPT_TYPE_FIELD)) {
              scriptType =
                  initalTimestampSettings.get(INITIAL_TIMESTAMP_SCRIPT_TYPE_FIELD).toString();
            }
            if (initalTimestampSettings.containsKey(INITIAL_TIMESTAMP_SCRIPT_FIELD)) {

              ExecutableScript scriptExecutable =
                  scriptService.executable(
                      scriptType,
                      initalTimestampSettings.get(INITIAL_TIMESTAMP_SCRIPT_FIELD).toString(),
                      ScriptService.ScriptType.INLINE,
                      Maps.newHashMap());
              Object ctx = scriptExecutable.run();
              logger.trace("initialTimestamp script returned: {}", ctx);
              if (ctx != null) {
                long timestamp = Long.parseLong(ctx.toString());
                timeStamp = new BSONTimestamp((int) (new Date(timestamp).getTime() / 1000), 1);
              }
            }
          } catch (Throwable t) {
            logger.error("Could not set initial timestamp", t);
          } finally {
            builder.initialTimestamp(timeStamp);
          }
        }
      }
      builder.mongoClientOptions(mongoClientOptionsBuilder.build());

      // Credentials
      if (mongoSettings.containsKey(CREDENTIALS_FIELD)) {
        String dbCredential;
        String mau = "";
        String map = "";
        String maad = "";
        String mlu = "";
        String mlp = "";
        String mlad = "";
        // String mdu = "";
        // String mdp = "";
        Object mongoCredentialsSettings = mongoSettings.get(CREDENTIALS_FIELD);
        boolean array = XContentMapValues.isArray(mongoCredentialsSettings);

        if (array) {
          ArrayList<Map<String, Object>> credentials =
              (ArrayList<Map<String, Object>>) mongoCredentialsSettings;
          for (Map<String, Object> credential : credentials) {
            dbCredential = XContentMapValues.nodeStringValue(credential.get(DB_FIELD), null);
            if (ADMIN_DB_FIELD.equals(dbCredential)) {
              mau = XContentMapValues.nodeStringValue(credential.get(USER_FIELD), null);
              map = XContentMapValues.nodeStringValue(credential.get(PASSWORD_FIELD), null);
              maad = XContentMapValues.nodeStringValue(credential.get(AUTH_FIELD), null);
            } else if (LOCAL_DB_FIELD.equals(dbCredential)) {
              mlu = XContentMapValues.nodeStringValue(credential.get(USER_FIELD), null);
              mlp = XContentMapValues.nodeStringValue(credential.get(PASSWORD_FIELD), null);
              mlad = XContentMapValues.nodeStringValue(credential.get(AUTH_FIELD), null);
              // } else {
              // mdu = XContentMapValues.nodeStringValue(
              // credential.get(USER_FIELD), null);
              // mdp = XContentMapValues.nodeStringValue(
              // credential.get(PASSWORD_FIELD), null);
            }
          }
        }
        builder.mongoAdminUser(mau);
        builder.mongoAdminPassword(map);
        builder.mongoAdminAuthDatabase(maad);
        builder.mongoLocalUser(mlu);
        builder.mongoLocalPassword(mlp);
        builder.mongoLocalAuthDatabase(mlad);
        // mongoDbUser = mdu;
        // mongoDbPassword = mdp;
      }

      builder.mongoDb(XContentMapValues.nodeStringValue(mongoSettings.get(DB_FIELD), riverName));
      builder.mongoCollection(
          XContentMapValues.nodeStringValue(mongoSettings.get(COLLECTION_FIELD), riverName));
      builder.mongoGridFS(
          XContentMapValues.nodeBooleanValue(mongoSettings.get(GRIDFS_FIELD), false));
      if (mongoSettings.containsKey(FILTER_FIELD)) {
        String filter = XContentMapValues.nodeStringValue(mongoSettings.get(FILTER_FIELD), "");
        filter = removePrefix("o.", filter);
        builder.mongoCollectionFilter(convertToBasicDBObject(filter));
        // DBObject bsonObject = (DBObject) JSON.parse(filter);
        // builder.mongoOplogFilter(convertToBasicDBObject(addPrefix("o.",
        // filter)));
        builder.mongoOplogFilter(convertToBasicDBObject(removePrefix("o.", filter)));
        // } else {
        // builder.mongoOplogFilter("");
      }

      if (mongoSettings.containsKey(SCRIPT_FIELD)) {
        String scriptType = "js";
        builder.script(mongoSettings.get(SCRIPT_FIELD).toString());
        if (mongoSettings.containsKey("scriptType")) {
          scriptType = mongoSettings.get("scriptType").toString();
        } else if (mongoSettings.containsKey(SCRIPT_TYPE_FIELD)) {
          scriptType = mongoSettings.get(SCRIPT_TYPE_FIELD).toString();
        }
        builder.scriptType(scriptType);
      }
    } else {
      mongoHost = DEFAULT_DB_HOST;
      mongoPort = DEFAULT_DB_PORT;
      try {
        mongoServers.add(new ServerAddress(mongoHost, mongoPort));
        builder.mongoServers(mongoServers);
      } catch (UnknownHostException e) {
        e.printStackTrace();
      }
      builder.mongoDb(riverName);
      builder.mongoCollection(riverName);
    }

    if (settings.settings().containsKey(INDEX_OBJECT)) {
      Map<String, Object> indexSettings =
          (Map<String, Object>) settings.settings().get(INDEX_OBJECT);
      builder.indexName(
          XContentMapValues.nodeStringValue(indexSettings.get(NAME_FIELD), builder.mongoDb));
      builder.typeName(
          XContentMapValues.nodeStringValue(indexSettings.get(TYPE_FIELD), builder.mongoDb));

      Bulk.Builder bulkBuilder = new Bulk.Builder();
      if (indexSettings.containsKey(BULK_FIELD)) {
        Map<String, Object> bulkSettings = (Map<String, Object>) indexSettings.get(BULK_FIELD);
        int bulkActions =
            XContentMapValues.nodeIntegerValue(
                bulkSettings.get(ACTIONS_FIELD), DEFAULT_BULK_ACTIONS);
        bulkBuilder.bulkActions(bulkActions);
        String size =
            XContentMapValues.nodeStringValue(
                bulkSettings.get(SIZE_FIELD), DEFAULT_BULK_SIZE.toString());
        bulkBuilder.bulkSize(ByteSizeValue.parseBytesSizeValue(size));
        bulkBuilder.concurrentRequests(
            XContentMapValues.nodeIntegerValue(
                bulkSettings.get(CONCURRENT_REQUESTS_FIELD),
                EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY)));
        bulkBuilder.flushInterval(
            XContentMapValues.nodeTimeValue(
                bulkSettings.get(FLUSH_INTERVAL_FIELD), DEFAULT_FLUSH_INTERVAL));
        builder.throttleSize(
            XContentMapValues.nodeIntegerValue(
                indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5));
      } else {
        int bulkActions =
            XContentMapValues.nodeIntegerValue(
                indexSettings.get(BULK_SIZE_FIELD), DEFAULT_BULK_ACTIONS);
        bulkBuilder.bulkActions(bulkActions);
        bulkBuilder.bulkSize(DEFAULT_BULK_SIZE);
        bulkBuilder.flushInterval(
            XContentMapValues.nodeTimeValue(
                indexSettings.get(BULK_TIMEOUT_FIELD), DEFAULT_FLUSH_INTERVAL));
        bulkBuilder.concurrentRequests(
            XContentMapValues.nodeIntegerValue(
                indexSettings.get(CONCURRENT_BULK_REQUESTS_FIELD),
                EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY)));
        builder.throttleSize(
            XContentMapValues.nodeIntegerValue(
                indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5));
      }
      builder.bulk(bulkBuilder.build());
    } else {
      builder.indexName(builder.mongoDb);
      builder.typeName(builder.mongoDb);
      builder.bulk(new Bulk.Builder().build());
    }
    return builder.build();
  }
  public void initialiseConnections() throws UnknownHostException {
    MongoClientOptions options =
        MongoClientOptions.builder().connectTimeout(15000).socketKeepAlive(true).build();

    Mongo mongo = new MongoClient(new ServerAddress(sourceHost, sourcePort), options);

    sourceDB = mongo.getDB(sourceDBName);

    // conceptTermListColl_source = JacksonDBCollection.wrap(sourceDB.getCollection("TermList"),
    // ConceptTermList.class,
    //		String.class);

    placeTermListColl_source =
        JacksonDBCollection.wrap(
            sourceDB.getCollection("TermList"), PlaceTermList.class, String.class);

    // timespanTermListColl_source = JacksonDBCollection.wrap(sourceDB.getCollection("TermList"),
    //		TimespanTermList.class, String.class);

    // agentTermListColl_source = JacksonDBCollection.wrap(sourceDB.getCollection("TermList"),
    // AgentTermList.class,
    //		String.class);

    // conceptTermColl_source = JacksonDBCollection.wrap(sourceDB.getCollection("concept"),
    // MongoTerm.class,
    //		String.class);

    placeTermColl_source =
        JacksonDBCollection.wrap(sourceDB.getCollection("place"), MongoTerm.class, String.class);

    // timespanTermColl_source = JacksonDBCollection.wrap(sourceDB.getCollection("period"),
    // MongoTerm.class,
    //		String.class);

    // agentTermColl_source = JacksonDBCollection.wrap(sourceDB.getCollection("people"),
    // MongoTerm.class,
    //		String.class);

    if (!sourceHost.equals(targetHost) || sourcePort != targetPort) {
      mongo = new MongoClient(new ServerAddress(targetHost, targetPort), options);
    }

    targetDB = mongo.getDB(targetDBName);

    /*
    		conceptTermListColl_target = JacksonDBCollection.wrap(targetDB.getCollection("TermList"), ConceptTermList.class,
    				String.class);
    		conceptTermListColl_target.createIndex(new BasicDBObject("codeUri", 1), new BasicDBObject("unique", true));
    */
    placeTermListColl_target =
        JacksonDBCollection.wrap(
            targetDB.getCollection("TermList"), PlaceTermList.class, String.class);
    placeTermListColl_target.createIndex(
        new BasicDBObject("codeUri", 1), new BasicDBObject("unique", true));
    /*
    		timespanTermListColl_target = JacksonDBCollection.wrap(targetDB.getCollection("TermList"),
    				TimespanTermList.class, String.class);
    		timespanTermListColl_target.createIndex(new BasicDBObject("codeUri", 1), new BasicDBObject("unique", true));

    		agentTermListColl_target = JacksonDBCollection.wrap(targetDB.getCollection("TermList"), AgentTermList.class,
    				String.class);
    		agentTermListColl_target.createIndex(new BasicDBObject("codeUri", 1), new BasicDBObject("unique", true));

    		conceptTermColl_target = JacksonDBCollection.wrap(targetDB.getCollection("concept"), MongoTerm.class,
    				String.class);
    		conceptTermColl_target.createIndex(new BasicDBObject("label", 1).append("lang", 1).append("codeUri", 1),
    				new BasicDBObject("unique", true));
    		conceptTermColl_target.createIndex(new BasicDBObject("codeUri", 1));
    */
    placeTermColl_target =
        JacksonDBCollection.wrap(targetDB.getCollection("place"), MongoTerm.class, String.class);
    placeTermColl_target.createIndex(
        new BasicDBObject("label", 1).append("lang", 1).append("codeUri", 1),
        new BasicDBObject("unique", true));
    placeTermColl_target.createIndex(new BasicDBObject("codeUri", 1));
    /*
    		timespanTermColl_target = JacksonDBCollection.wrap(targetDB.getCollection("period"), MongoTerm.class,
    				String.class);
    		timespanTermColl_target.createIndex(new BasicDBObject("label", 1).append("lang", 1).append("codeUri", 1),
    				new BasicDBObject("unique", true));
    		timespanTermColl_target.createIndex(new BasicDBObject("codeUri", 1));

    		agentTermColl_target = JacksonDBCollection.wrap(targetDB.getCollection("people"), MongoTerm.class,
    				String.class);
    		agentTermColl_target.createIndex(new BasicDBObject("label", 1).append("lang", 1).append("codeUri", 1),
    				new BasicDBObject("unique", true));
    		agentTermColl_target.createIndex(new BasicDBObject("codeUri", 1));
    */
    lookupColl_target =
        JacksonDBCollection.wrap(
            targetDB.getCollection("lookup"), MongoCodeLookup.class, String.class);
    lookupColl_target.createIndex(
        new BasicDBObject("codeUri", 1).append("originalCodeUri", 1),
        new BasicDBObject("unique", true));
    lookupColl_target.createIndex(new BasicDBObject("originalCodeUri", 1)); // for
    // reverse
    // lookup

    sequenceColl_target =
        JacksonDBCollection.wrap(
            targetDB.getCollection("sequence"), MongoSequence.class, String.class);
  }
  private void initMongoInstances() throws Exception {
    logger.debug("*** initMongoInstances ***");
    CommandResult cr;

    // Create 3 mongod processes
    mongodConfig1 =
        new MongodConfig(
            new GenericVersion(mongoVersion),
            new Net(mongoPort1, Network.localhostIsIPv6()),
            new Storage(null, REPLICA_SET_NAME, 20),
            new Timeout());
    MongodStarter starter = MongodStarter.getDefaultInstance();
    mongodExe1 = starter.prepare(mongodConfig1);
    mongod1 = mongodExe1.start();
    mongodConfig2 =
        new MongodConfig(
            new GenericVersion(mongoVersion),
            new Net(mongoPort2, Network.localhostIsIPv6()),
            new Storage(null, REPLICA_SET_NAME, 20),
            new Timeout());
    mongodExe2 = starter.prepare(mongodConfig2);
    mongod2 = mongodExe2.start();
    mongodConfig3 =
        new MongodConfig(
            new GenericVersion(mongoVersion),
            new Net(mongoPort3, Network.localhostIsIPv6()),
            new Storage(null, REPLICA_SET_NAME, 20),
            new Timeout());
    mongodExe3 = starter.prepare(mongodConfig3);
    mongod3 = mongodExe3.start();
    String server1 = Network.getLocalHost().getHostName() + ":" + mongodConfig1.net().getPort();
    String server2 = Network.getLocalHost().getHostName() + ":" + mongodConfig2.net().getPort();
    String server3 = Network.getLocalHost().getHostName() + ":" + mongodConfig3.net().getPort();
    logger.debug("Server #1: {}", server1);
    logger.debug("Server #2: {}", server2);
    logger.debug("Server #3: {}", server3);
    Thread.sleep(2000);
    MongoClientOptions mco =
        MongoClientOptions.builder()
            .autoConnectRetry(true)
            .connectTimeout(15000)
            .socketTimeout(60000)
            .build();
    mongo =
        new MongoClient(
            new ServerAddress(Network.getLocalHost().getHostName(), mongodConfig1.net().getPort()),
            mco);
    mongoAdminDB = mongo.getDB(ADMIN_DATABASE_NAME);

    cr = mongoAdminDB.command(new BasicDBObject("isMaster", 1));
    logger.debug("isMaster: " + cr);

    // Initialize replica set
    cr =
        mongoAdminDB.command(
            new BasicDBObject(
                "replSetInitiate",
                (DBObject)
                    JSON.parse(
                        "{'_id': '"
                            + REPLICA_SET_NAME
                            + "', 'members': [{'_id': 0, 'host': '"
                            + server1
                            + "'}, {'_id': 1, 'host': '"
                            + server2
                            + "'}, {'_id': 2, 'host': '"
                            + server3
                            + "', 'arbiterOnly' : true}]} }")));
    logger.debug("replSetInitiate: " + cr);

    Thread.sleep(5000);
    cr = mongoAdminDB.command(new BasicDBObject("replSetGetStatus", 1));
    logger.info("replSetGetStatus: " + cr);

    // Check replica set status before to proceed
    while (!isReplicaSetStarted(cr)) {
      logger.debug("Waiting for 3 seconds...");
      Thread.sleep(3000);
      cr = mongoAdminDB.command(new BasicDBObject("replSetGetStatus", 1));
      logger.debug("replSetGetStatus: " + cr);
    }

    mongo.close();
    mongo = null;

    // Initialize a new client using all instances.
    List<ServerAddress> mongoServers = new ArrayList<ServerAddress>();
    mongoServers.add(
        new ServerAddress(Network.getLocalHost().getHostName(), mongodConfig1.net().getPort()));
    mongoServers.add(
        new ServerAddress(Network.getLocalHost().getHostName(), mongodConfig2.net().getPort()));
    mongoServers.add(
        new ServerAddress(Network.getLocalHost().getHostName(), mongodConfig3.net().getPort()));
    mongo = new MongoClient(mongoServers, mco);
    Assert.assertNotNull(mongo);
    mongo.setReadPreference(ReadPreference.secondaryPreferred());
    mongo.setWriteConcern(WriteConcern.REPLICAS_SAFE);
  }
 private Builder builder(MongoClientOptions options) {
   Builder builder = MongoClientOptions.builder();
   if (options != null) {
     builder.alwaysUseMBeans(options.isAlwaysUseMBeans());
     builder.connectionsPerHost(options.getConnectionsPerHost());
     builder.connectTimeout(options.getConnectTimeout());
     builder.cursorFinalizerEnabled(options.isCursorFinalizerEnabled());
     builder.dbDecoderFactory(options.getDbDecoderFactory());
     builder.dbEncoderFactory(options.getDbEncoderFactory());
     builder.description(options.getDescription());
     builder.maxWaitTime(options.getMaxWaitTime());
     builder.readPreference(options.getReadPreference());
     builder.socketFactory(options.getSocketFactory());
     builder.socketKeepAlive(options.isSocketKeepAlive());
     builder.socketTimeout(options.getSocketTimeout());
     builder.threadsAllowedToBlockForConnectionMultiplier(
         options.getThreadsAllowedToBlockForConnectionMultiplier());
     builder.writeConcern(options.getWriteConcern());
   }
   return builder;
 }
示例#14
0
 /**
  * 构件MongoDB连接选项<br>
  *
  * @param group 分组,当分组对应的选项不存在时会读取根选项,如果也不存在使用默认值
  * @return MongoClientOptions
  */
 private MongoClientOptions buildMongoClientOptions(String group) {
   return buildMongoClientOptions(MongoClientOptions.builder(), group).build();
 }