@Override
public void testStarted() {
if (log.isDebugEnabled()) {
log.debug(getTitle() + " testStarted");
}
MongoClientOptions.Builder builder =
MongoClientOptions.builder()
.autoConnectRetry(getAutoConnectRetry())
.connectTimeout(getConnectTimeout())
.connectionsPerHost(getConnectionsPerHost())
.maxAutoConnectRetryTime(getMaxAutoConnectRetryTime())
.maxWaitTime(getMaxWaitTime())
.socketKeepAlive(getSocketKeepAlive())
.socketTimeout(getSocketTimeout())
.threadsAllowedToBlockForConnectionMultiplier(
getThreadsAllowedToBlockForConnectionMultiplier());
if (getSafe()) {
builder.writeConcern(WriteConcern.SAFE);
} else {
builder.writeConcern(
new WriteConcern(
getWriteOperationNumberOfServers(),
getWriteOperationTimeout(),
getFsync(),
getWaitForJournaling(),
getContinueOnInsertError()));
}
MongoClientOptions mongoOptions = builder.build();
if (log.isDebugEnabled()) {
log.debug("options : " + mongoOptions.toString());
}
if (getThreadContext().getVariables().getObject(getSource()) != null) {
if (log.isWarnEnabled()) {
log.warn(getSource() + " has already been defined.");
}
} else {
if (log.isDebugEnabled()) {
log.debug(getSource() + " is being defined.");
}
try {
getThreadContext()
.getVariables()
.putObject(
getSource(),
new MongoDB(MongoUtils.toServerAddresses(getConnection()), mongoOptions));
} catch (UnknownHostException e) {
throw new IllegalStateException(e);
}
}
}
protected void connect() {
if (this.mongoClient == null) {
try {
List<ServerAddress> addresses = new ArrayList<ServerAddress>();
for (String host : this.servers) {
addresses.add(new ServerAddress(host, 27017));
}
this.mongoClient =
new MongoClient(
addresses,
MongoClientOptions.builder()
.connectionsPerHost(30)
.threadsAllowedToBlockForConnectionMultiplier(20)
.build());
this.db = this.mongoClient.getDB("twitter");
this.dbCollection = this.db.getCollection("tweets");
} catch (Exception e) {
this.log.write(
"MongoDBConnector - Exception in MongoDBConnector.connect: "
+ e.getMessage()
+ ": "
+ e.toString());
}
}
}
/**
* Creates a {@link MongoClient} using the given {@code options} and {@code environment}. If the
* configured port is zero, the value of the {@code local.mongo.port} property retrieved from the
* {@code environment} is used to configure the client.
*
* @param options the options
* @param environment the environment
* @return the Mongo client
* @throws UnknownHostException if the configured host is unknown
*/
public MongoClient createMongoClient(MongoClientOptions options, Environment environment)
throws UnknownHostException {
try {
if (hasCustomAddress() || hasCustomCredentials()) {
if (options == null) {
options = MongoClientOptions.builder().build();
}
List<MongoCredential> credentials = null;
if (hasCustomCredentials()) {
String database =
this.authenticationDatabase == null
? getMongoClientDatabase()
: this.authenticationDatabase;
credentials =
Arrays.asList(
MongoCredential.createMongoCRCredential(this.username, database, this.password));
}
String host = this.host == null ? "localhost" : this.host;
int port = determinePort(environment);
return new MongoClient(Arrays.asList(new ServerAddress(host, port)), credentials, options);
}
// The options and credentials are in the URI
return new MongoClient(new MongoClientURI(this.uri, builder(options)));
} finally {
clearPassword();
}
}
@Override
@SuppressWarnings("serial")
public BasicConnectionFactory<MongoDBConnectionImpl> createConnectionFactory()
throws ResourceException {
if (this.remoteServerList == null) {
throw new InvalidPropertyException(UTIL.getString("no_server")); // $NON-NLS-1$
}
if (this.database == null) {
throw new InvalidPropertyException(UTIL.getString("no_database")); // $NON-NLS-1$
}
final List<ServerAddress> servers = getServers();
// TODO: need to define all the properties on the ra.xml and build this correctly
final MongoClientOptions options = MongoClientOptions.builder().build();
return new BasicConnectionFactory<MongoDBConnectionImpl>() {
@Override
public MongoDBConnectionImpl getConnection() throws ResourceException {
MongoCredential credential = null;
if (MongoDBManagedConnectionFactory.this.username != null
&& MongoDBManagedConnectionFactory.this.password != null) {
credential =
MongoCredential.createMongoCRCredential(
MongoDBManagedConnectionFactory.this.username,
MongoDBManagedConnectionFactory.this.database,
MongoDBManagedConnectionFactory.this.password.toCharArray());
}
return new MongoDBConnectionImpl(
MongoDBManagedConnectionFactory.this.database, servers, credential, options);
}
};
}
private MongoClient createClient(final String databaseUri, final int port)
throws UnknownHostException {
final MongoClientOptions options =
MongoClientOptions.builder()
.connectionsPerHost(100)
.threadsAllowedToBlockForConnectionMultiplier(1500)
.build();
final MongoClient client = new MongoClient(new ServerAddress(databaseUri, port), options);
return client;
}
/** Creates a connection to a running MongoDB instance using the required codecs. */
public DutyScheduleDB() {
// Create codec registry with LocalDateCodec.
CodecRegistry codecRegistry =
CodecRegistries.fromRegistries(
CodecRegistries.fromCodecs(new LocalDateCodec()),
CodecRegistries.fromProviders(
new RaCodecProvider(),
new DutyBlockCodecProvider(),
new ScheduledDutyCodecProvider()),
MongoClient.getDefaultCodecRegistry());
MongoClientOptions options = MongoClientOptions.builder().codecRegistry(codecRegistry).build();
mongoClient = new MongoClient(new ServerAddress(), options);
db = mongoClient.getDatabase("DutySchedulerDB");
}
private static MongoClientOptions configureMongoOptions() {
final MongoClientOptions.Builder options = MongoClientOptions.builder();
for (ApplicationClass clazz : Play.classes.getAssignableClasses(MongoConfigurator.class)) {
try {
final MongoConfigurator configurator = (MongoConfigurator) clazz.javaClass.newInstance();
configurator.configureOptions(options);
} catch (InstantiationException e) {
throw new ConfigurationException(
String.format("failed to create configurator: %s", clazz.name));
} catch (IllegalAccessException e) {
throw new ConfigurationException(
String.format("failed to create configurator: %s", clazz.name));
}
}
return options.build();
}
@Before
public void beforeEachTest() {
CodecRegistry codecRegistry =
CodecRegistries.fromRegistries(
MongoClient.getDefaultCodecRegistry(),
CodecRegistries.fromCodecs(
new LocalDateTimeCodec(), new HostAndPortCodec(), new LocalDateTimeCodec()));
MongoProperties properties = new MongoProperties();
MongoClient mongoClient =
new MongoClient(
new MongoClientURI(
properties.getUri(), MongoClientOptions.builder().codecRegistry(codecRegistry)));
mongoMetricsRepository =
new MongoMetricsRepository(mongoClient, properties, new ObjectMapper());
mongoMetricsRepository.init();
metricsService = new MetricsService(mongoMetricsRepository);
}
@SuppressWarnings("deprecation")
@Override
public Mongo get() {
try {
String mongodbAddress = configuration.getString("mongodb.address");
String[] addresses = mongodbAddress.split(",");
ArrayList<ServerAddress> servers = new ArrayList<ServerAddress>();
for (String address : addresses) {
String[] addPort = address.split(":");
String add = addPort[0];
int port = addPort.length > 1 ? Integer.parseInt(addPort[1]) : 27017;
servers.add(new ServerAddress(add, port));
}
LOGGER.info("mongo servers:" + mongodbAddress);
// MongoOptions mongoOptions = new MongoOptions();
// mongoOptions.connectionsPerHost =
// this.configuration.getInt("mongodb.connectionsPerHost", 60);
// mongoOptions.socketTimeout = 60000;
// mongoOptions.connectTimeout = 30000;
// mongoOptions.autoConnectRetry = true;
// mongoOptions.socketKeepAlive = true;
// Mongo mongo = new Mongo(servers, mongoOptions);
MongoClientOptions.Builder builder = MongoClientOptions.builder();
builder
.socketKeepAlive(true)
.socketTimeout(60000)
.connectTimeout(30000)
.autoConnectRetry(true)
.connectionsPerHost(this.configuration.getInt("mongodb.connectionsPerHost", 60));
MongoClient mongoClient = new MongoClient(servers, builder.build());
if (isReadSecondary) mongoClient.setReadPreference(ReadPreference.secondaryPreferred());
return mongoClient;
} catch (Throwable e) {
LOGGER.error("failed to init mongodb", e);
throw new ExceptionInInitializerError(e);
}
}
@SuppressWarnings("unchecked")
public static synchronized MongoDBRiverDefinition parseSettings(
String riverName,
String riverIndexName,
RiverSettings settings,
ScriptService scriptService) {
logger.trace("Parse river settings for {}", riverName);
Preconditions.checkNotNull(riverName, "No riverName specified");
Preconditions.checkNotNull(riverIndexName, "No riverIndexName specified");
Preconditions.checkNotNull(settings, "No settings specified");
Builder builder = new Builder();
builder.riverName(riverName);
builder.riverIndexName(riverIndexName);
List<ServerAddress> mongoServers = new ArrayList<ServerAddress>();
String mongoHost;
int mongoPort;
if (settings.settings().containsKey(MongoDBRiver.TYPE)) {
Map<String, Object> mongoSettings =
(Map<String, Object>) settings.settings().get(MongoDBRiver.TYPE);
if (mongoSettings.containsKey(SERVERS_FIELD)) {
Object mongoServersSettings = mongoSettings.get(SERVERS_FIELD);
logger.trace("mongoServersSettings: " + mongoServersSettings);
boolean array = XContentMapValues.isArray(mongoServersSettings);
if (array) {
ArrayList<Map<String, Object>> feeds =
(ArrayList<Map<String, Object>>) mongoServersSettings;
for (Map<String, Object> feed : feeds) {
mongoHost = XContentMapValues.nodeStringValue(feed.get(HOST_FIELD), null);
mongoPort = XContentMapValues.nodeIntegerValue(feed.get(PORT_FIELD), DEFAULT_DB_PORT);
logger.trace("Server: " + mongoHost + " - " + mongoPort);
try {
mongoServers.add(new ServerAddress(mongoHost, mongoPort));
} catch (UnknownHostException uhEx) {
logger.warn("Cannot add mongo server {}:{}", uhEx, mongoHost, mongoPort);
}
}
}
} else {
mongoHost =
XContentMapValues.nodeStringValue(mongoSettings.get(HOST_FIELD), DEFAULT_DB_HOST);
mongoPort =
XContentMapValues.nodeIntegerValue(mongoSettings.get(PORT_FIELD), DEFAULT_DB_PORT);
try {
mongoServers.add(new ServerAddress(mongoHost, mongoPort));
} catch (UnknownHostException uhEx) {
logger.warn("Cannot add mongo server {}:{}", uhEx, mongoHost, mongoPort);
}
}
builder.mongoServers(mongoServers);
MongoClientOptions.Builder mongoClientOptionsBuilder =
MongoClientOptions.builder().socketKeepAlive(true);
// MongoDB options
if (mongoSettings.containsKey(OPTIONS_FIELD)) {
Map<String, Object> mongoOptionsSettings =
(Map<String, Object>) mongoSettings.get(OPTIONS_FIELD);
logger.trace("mongoOptionsSettings: " + mongoOptionsSettings);
builder.mongoSecondaryReadPreference(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SECONDARY_READ_PREFERENCE_FIELD), false));
builder.connectTimeout(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(CONNECT_TIMEOUT), DEFAULT_CONNECT_TIMEOUT));
builder.socketTimeout(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(SOCKET_TIMEOUT), DEFAULT_SOCKET_TIMEOUT));
builder.dropCollection(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(DROP_COLLECTION_FIELD), false));
String isMongos =
XContentMapValues.nodeStringValue(mongoOptionsSettings.get(IS_MONGOS_FIELD), null);
if (isMongos != null) {
builder.isMongos(Boolean.valueOf(isMongos));
}
builder.mongoUseSSL(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SSL_CONNECTION_FIELD), false));
builder.mongoSSLVerifyCertificate(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SSL_VERIFY_CERT_FIELD), true));
builder.advancedTransformation(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(ADVANCED_TRANSFORMATION_FIELD), false));
builder.skipInitialImport(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SKIP_INITIAL_IMPORT_FIELD), false));
builder.connectionsPerHost(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(CONNECTIONS_PER_HOST), DEFAULT_CONNECTIONS_PER_HOST));
builder.threadsAllowedToBlockForConnectionMultiplier(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER),
DEFAULT_THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER));
mongoClientOptionsBuilder
.connectTimeout(builder.connectTimeout)
.socketTimeout(builder.socketTimeout)
.connectionsPerHost(builder.connectionsPerHost)
.threadsAllowedToBlockForConnectionMultiplier(
builder.threadsAllowedToBlockForConnectionMultiplier);
if (builder.mongoSecondaryReadPreference) {
mongoClientOptionsBuilder.readPreference(ReadPreference.secondaryPreferred());
}
if (builder.mongoUseSSL) {
mongoClientOptionsBuilder.socketFactory(getSSLSocketFactory());
}
if (mongoOptionsSettings.containsKey(PARENT_TYPES_FIELD)) {
Set<String> parentTypes = new HashSet<String>();
Object parentTypesSettings = mongoOptionsSettings.get(PARENT_TYPES_FIELD);
logger.trace("parentTypesSettings: " + parentTypesSettings);
boolean array = XContentMapValues.isArray(parentTypesSettings);
if (array) {
ArrayList<String> fields = (ArrayList<String>) parentTypesSettings;
for (String field : fields) {
logger.trace("Field: " + field);
parentTypes.add(field);
}
}
builder.parentTypes(parentTypes);
}
if (mongoOptionsSettings.containsKey(STORE_STATISTICS_FIELD)) {
Object storeStatistics = mongoOptionsSettings.get(STORE_STATISTICS_FIELD);
boolean object = XContentMapValues.isObject(storeStatistics);
if (object) {
Map<String, Object> storeStatisticsSettings = (Map<String, Object>) storeStatistics;
builder.storeStatistics(true);
builder.statisticsIndexName(
XContentMapValues.nodeStringValue(
storeStatisticsSettings.get(INDEX_OBJECT), riverName + "-stats"));
builder.statisticsTypeName(
XContentMapValues.nodeStringValue(
storeStatisticsSettings.get(TYPE_FIELD), "stats"));
} else {
builder.storeStatistics(XContentMapValues.nodeBooleanValue(storeStatistics, false));
if (builder.storeStatistics) {
builder.statisticsIndexName(riverName + "-stats");
builder.statisticsTypeName("stats");
}
}
}
// builder.storeStatistics(XContentMapValues.nodeBooleanValue(mongoOptionsSettings.get(STORE_STATISTICS_FIELD),
// false));
builder.importAllCollections(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(IMPORT_ALL_COLLECTIONS_FIELD), false));
builder.disableIndexRefresh(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(DISABLE_INDEX_REFRESH_FIELD), false));
builder.includeCollection(
XContentMapValues.nodeStringValue(
mongoOptionsSettings.get(INCLUDE_COLLECTION_FIELD), ""));
if (mongoOptionsSettings.containsKey(INCLUDE_FIELDS_FIELD)) {
Set<String> includeFields = new HashSet<String>();
Object includeFieldsSettings = mongoOptionsSettings.get(INCLUDE_FIELDS_FIELD);
logger.trace("includeFieldsSettings: " + includeFieldsSettings);
boolean array = XContentMapValues.isArray(includeFieldsSettings);
if (array) {
ArrayList<String> fields = (ArrayList<String>) includeFieldsSettings;
for (String field : fields) {
logger.trace("Field: " + field);
includeFields.add(field);
}
}
if (!includeFields.contains(MongoDBRiver.MONGODB_ID_FIELD)) {
includeFields.add(MongoDBRiver.MONGODB_ID_FIELD);
}
builder.includeFields(includeFields);
} else if (mongoOptionsSettings.containsKey(EXCLUDE_FIELDS_FIELD)) {
Set<String> excludeFields = new HashSet<String>();
Object excludeFieldsSettings = mongoOptionsSettings.get(EXCLUDE_FIELDS_FIELD);
logger.trace("excludeFieldsSettings: " + excludeFieldsSettings);
boolean array = XContentMapValues.isArray(excludeFieldsSettings);
if (array) {
ArrayList<String> fields = (ArrayList<String>) excludeFieldsSettings;
for (String field : fields) {
logger.trace("Field: " + field);
excludeFields.add(field);
}
}
builder.excludeFields(excludeFields);
}
if (mongoOptionsSettings.containsKey(INITIAL_TIMESTAMP_FIELD)) {
BSONTimestamp timeStamp = null;
try {
Map<String, Object> initalTimestampSettings =
(Map<String, Object>) mongoOptionsSettings.get(INITIAL_TIMESTAMP_FIELD);
String scriptType = "js";
if (initalTimestampSettings.containsKey(INITIAL_TIMESTAMP_SCRIPT_TYPE_FIELD)) {
scriptType =
initalTimestampSettings.get(INITIAL_TIMESTAMP_SCRIPT_TYPE_FIELD).toString();
}
if (initalTimestampSettings.containsKey(INITIAL_TIMESTAMP_SCRIPT_FIELD)) {
ExecutableScript scriptExecutable =
scriptService.executable(
scriptType,
initalTimestampSettings.get(INITIAL_TIMESTAMP_SCRIPT_FIELD).toString(),
ScriptService.ScriptType.INLINE,
Maps.newHashMap());
Object ctx = scriptExecutable.run();
logger.trace("initialTimestamp script returned: {}", ctx);
if (ctx != null) {
long timestamp = Long.parseLong(ctx.toString());
timeStamp = new BSONTimestamp((int) (new Date(timestamp).getTime() / 1000), 1);
}
}
} catch (Throwable t) {
logger.error("Could not set initial timestamp", t);
} finally {
builder.initialTimestamp(timeStamp);
}
}
}
builder.mongoClientOptions(mongoClientOptionsBuilder.build());
// Credentials
if (mongoSettings.containsKey(CREDENTIALS_FIELD)) {
String dbCredential;
String mau = "";
String map = "";
String maad = "";
String mlu = "";
String mlp = "";
String mlad = "";
// String mdu = "";
// String mdp = "";
Object mongoCredentialsSettings = mongoSettings.get(CREDENTIALS_FIELD);
boolean array = XContentMapValues.isArray(mongoCredentialsSettings);
if (array) {
ArrayList<Map<String, Object>> credentials =
(ArrayList<Map<String, Object>>) mongoCredentialsSettings;
for (Map<String, Object> credential : credentials) {
dbCredential = XContentMapValues.nodeStringValue(credential.get(DB_FIELD), null);
if (ADMIN_DB_FIELD.equals(dbCredential)) {
mau = XContentMapValues.nodeStringValue(credential.get(USER_FIELD), null);
map = XContentMapValues.nodeStringValue(credential.get(PASSWORD_FIELD), null);
maad = XContentMapValues.nodeStringValue(credential.get(AUTH_FIELD), null);
} else if (LOCAL_DB_FIELD.equals(dbCredential)) {
mlu = XContentMapValues.nodeStringValue(credential.get(USER_FIELD), null);
mlp = XContentMapValues.nodeStringValue(credential.get(PASSWORD_FIELD), null);
mlad = XContentMapValues.nodeStringValue(credential.get(AUTH_FIELD), null);
// } else {
// mdu = XContentMapValues.nodeStringValue(
// credential.get(USER_FIELD), null);
// mdp = XContentMapValues.nodeStringValue(
// credential.get(PASSWORD_FIELD), null);
}
}
}
builder.mongoAdminUser(mau);
builder.mongoAdminPassword(map);
builder.mongoAdminAuthDatabase(maad);
builder.mongoLocalUser(mlu);
builder.mongoLocalPassword(mlp);
builder.mongoLocalAuthDatabase(mlad);
// mongoDbUser = mdu;
// mongoDbPassword = mdp;
}
builder.mongoDb(XContentMapValues.nodeStringValue(mongoSettings.get(DB_FIELD), riverName));
builder.mongoCollection(
XContentMapValues.nodeStringValue(mongoSettings.get(COLLECTION_FIELD), riverName));
builder.mongoGridFS(
XContentMapValues.nodeBooleanValue(mongoSettings.get(GRIDFS_FIELD), false));
if (mongoSettings.containsKey(FILTER_FIELD)) {
String filter = XContentMapValues.nodeStringValue(mongoSettings.get(FILTER_FIELD), "");
filter = removePrefix("o.", filter);
builder.mongoCollectionFilter(convertToBasicDBObject(filter));
// DBObject bsonObject = (DBObject) JSON.parse(filter);
// builder.mongoOplogFilter(convertToBasicDBObject(addPrefix("o.",
// filter)));
builder.mongoOplogFilter(convertToBasicDBObject(removePrefix("o.", filter)));
// } else {
// builder.mongoOplogFilter("");
}
if (mongoSettings.containsKey(SCRIPT_FIELD)) {
String scriptType = "js";
builder.script(mongoSettings.get(SCRIPT_FIELD).toString());
if (mongoSettings.containsKey("scriptType")) {
scriptType = mongoSettings.get("scriptType").toString();
} else if (mongoSettings.containsKey(SCRIPT_TYPE_FIELD)) {
scriptType = mongoSettings.get(SCRIPT_TYPE_FIELD).toString();
}
builder.scriptType(scriptType);
}
} else {
mongoHost = DEFAULT_DB_HOST;
mongoPort = DEFAULT_DB_PORT;
try {
mongoServers.add(new ServerAddress(mongoHost, mongoPort));
builder.mongoServers(mongoServers);
} catch (UnknownHostException e) {
e.printStackTrace();
}
builder.mongoDb(riverName);
builder.mongoCollection(riverName);
}
if (settings.settings().containsKey(INDEX_OBJECT)) {
Map<String, Object> indexSettings =
(Map<String, Object>) settings.settings().get(INDEX_OBJECT);
builder.indexName(
XContentMapValues.nodeStringValue(indexSettings.get(NAME_FIELD), builder.mongoDb));
builder.typeName(
XContentMapValues.nodeStringValue(indexSettings.get(TYPE_FIELD), builder.mongoDb));
Bulk.Builder bulkBuilder = new Bulk.Builder();
if (indexSettings.containsKey(BULK_FIELD)) {
Map<String, Object> bulkSettings = (Map<String, Object>) indexSettings.get(BULK_FIELD);
int bulkActions =
XContentMapValues.nodeIntegerValue(
bulkSettings.get(ACTIONS_FIELD), DEFAULT_BULK_ACTIONS);
bulkBuilder.bulkActions(bulkActions);
String size =
XContentMapValues.nodeStringValue(
bulkSettings.get(SIZE_FIELD), DEFAULT_BULK_SIZE.toString());
bulkBuilder.bulkSize(ByteSizeValue.parseBytesSizeValue(size));
bulkBuilder.concurrentRequests(
XContentMapValues.nodeIntegerValue(
bulkSettings.get(CONCURRENT_REQUESTS_FIELD),
EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY)));
bulkBuilder.flushInterval(
XContentMapValues.nodeTimeValue(
bulkSettings.get(FLUSH_INTERVAL_FIELD), DEFAULT_FLUSH_INTERVAL));
builder.throttleSize(
XContentMapValues.nodeIntegerValue(
indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5));
} else {
int bulkActions =
XContentMapValues.nodeIntegerValue(
indexSettings.get(BULK_SIZE_FIELD), DEFAULT_BULK_ACTIONS);
bulkBuilder.bulkActions(bulkActions);
bulkBuilder.bulkSize(DEFAULT_BULK_SIZE);
bulkBuilder.flushInterval(
XContentMapValues.nodeTimeValue(
indexSettings.get(BULK_TIMEOUT_FIELD), DEFAULT_FLUSH_INTERVAL));
bulkBuilder.concurrentRequests(
XContentMapValues.nodeIntegerValue(
indexSettings.get(CONCURRENT_BULK_REQUESTS_FIELD),
EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY)));
builder.throttleSize(
XContentMapValues.nodeIntegerValue(
indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5));
}
builder.bulk(bulkBuilder.build());
} else {
builder.indexName(builder.mongoDb);
builder.typeName(builder.mongoDb);
builder.bulk(new Bulk.Builder().build());
}
return builder.build();
}
public void initialiseConnections() throws UnknownHostException {
MongoClientOptions options =
MongoClientOptions.builder().connectTimeout(15000).socketKeepAlive(true).build();
Mongo mongo = new MongoClient(new ServerAddress(sourceHost, sourcePort), options);
sourceDB = mongo.getDB(sourceDBName);
// conceptTermListColl_source = JacksonDBCollection.wrap(sourceDB.getCollection("TermList"),
// ConceptTermList.class,
// String.class);
placeTermListColl_source =
JacksonDBCollection.wrap(
sourceDB.getCollection("TermList"), PlaceTermList.class, String.class);
// timespanTermListColl_source = JacksonDBCollection.wrap(sourceDB.getCollection("TermList"),
// TimespanTermList.class, String.class);
// agentTermListColl_source = JacksonDBCollection.wrap(sourceDB.getCollection("TermList"),
// AgentTermList.class,
// String.class);
// conceptTermColl_source = JacksonDBCollection.wrap(sourceDB.getCollection("concept"),
// MongoTerm.class,
// String.class);
placeTermColl_source =
JacksonDBCollection.wrap(sourceDB.getCollection("place"), MongoTerm.class, String.class);
// timespanTermColl_source = JacksonDBCollection.wrap(sourceDB.getCollection("period"),
// MongoTerm.class,
// String.class);
// agentTermColl_source = JacksonDBCollection.wrap(sourceDB.getCollection("people"),
// MongoTerm.class,
// String.class);
if (!sourceHost.equals(targetHost) || sourcePort != targetPort) {
mongo = new MongoClient(new ServerAddress(targetHost, targetPort), options);
}
targetDB = mongo.getDB(targetDBName);
/*
conceptTermListColl_target = JacksonDBCollection.wrap(targetDB.getCollection("TermList"), ConceptTermList.class,
String.class);
conceptTermListColl_target.createIndex(new BasicDBObject("codeUri", 1), new BasicDBObject("unique", true));
*/
placeTermListColl_target =
JacksonDBCollection.wrap(
targetDB.getCollection("TermList"), PlaceTermList.class, String.class);
placeTermListColl_target.createIndex(
new BasicDBObject("codeUri", 1), new BasicDBObject("unique", true));
/*
timespanTermListColl_target = JacksonDBCollection.wrap(targetDB.getCollection("TermList"),
TimespanTermList.class, String.class);
timespanTermListColl_target.createIndex(new BasicDBObject("codeUri", 1), new BasicDBObject("unique", true));
agentTermListColl_target = JacksonDBCollection.wrap(targetDB.getCollection("TermList"), AgentTermList.class,
String.class);
agentTermListColl_target.createIndex(new BasicDBObject("codeUri", 1), new BasicDBObject("unique", true));
conceptTermColl_target = JacksonDBCollection.wrap(targetDB.getCollection("concept"), MongoTerm.class,
String.class);
conceptTermColl_target.createIndex(new BasicDBObject("label", 1).append("lang", 1).append("codeUri", 1),
new BasicDBObject("unique", true));
conceptTermColl_target.createIndex(new BasicDBObject("codeUri", 1));
*/
placeTermColl_target =
JacksonDBCollection.wrap(targetDB.getCollection("place"), MongoTerm.class, String.class);
placeTermColl_target.createIndex(
new BasicDBObject("label", 1).append("lang", 1).append("codeUri", 1),
new BasicDBObject("unique", true));
placeTermColl_target.createIndex(new BasicDBObject("codeUri", 1));
/*
timespanTermColl_target = JacksonDBCollection.wrap(targetDB.getCollection("period"), MongoTerm.class,
String.class);
timespanTermColl_target.createIndex(new BasicDBObject("label", 1).append("lang", 1).append("codeUri", 1),
new BasicDBObject("unique", true));
timespanTermColl_target.createIndex(new BasicDBObject("codeUri", 1));
agentTermColl_target = JacksonDBCollection.wrap(targetDB.getCollection("people"), MongoTerm.class,
String.class);
agentTermColl_target.createIndex(new BasicDBObject("label", 1).append("lang", 1).append("codeUri", 1),
new BasicDBObject("unique", true));
agentTermColl_target.createIndex(new BasicDBObject("codeUri", 1));
*/
lookupColl_target =
JacksonDBCollection.wrap(
targetDB.getCollection("lookup"), MongoCodeLookup.class, String.class);
lookupColl_target.createIndex(
new BasicDBObject("codeUri", 1).append("originalCodeUri", 1),
new BasicDBObject("unique", true));
lookupColl_target.createIndex(new BasicDBObject("originalCodeUri", 1)); // for
// reverse
// lookup
sequenceColl_target =
JacksonDBCollection.wrap(
targetDB.getCollection("sequence"), MongoSequence.class, String.class);
}
private void initMongoInstances() throws Exception {
logger.debug("*** initMongoInstances ***");
CommandResult cr;
// Create 3 mongod processes
mongodConfig1 =
new MongodConfig(
new GenericVersion(mongoVersion),
new Net(mongoPort1, Network.localhostIsIPv6()),
new Storage(null, REPLICA_SET_NAME, 20),
new Timeout());
MongodStarter starter = MongodStarter.getDefaultInstance();
mongodExe1 = starter.prepare(mongodConfig1);
mongod1 = mongodExe1.start();
mongodConfig2 =
new MongodConfig(
new GenericVersion(mongoVersion),
new Net(mongoPort2, Network.localhostIsIPv6()),
new Storage(null, REPLICA_SET_NAME, 20),
new Timeout());
mongodExe2 = starter.prepare(mongodConfig2);
mongod2 = mongodExe2.start();
mongodConfig3 =
new MongodConfig(
new GenericVersion(mongoVersion),
new Net(mongoPort3, Network.localhostIsIPv6()),
new Storage(null, REPLICA_SET_NAME, 20),
new Timeout());
mongodExe3 = starter.prepare(mongodConfig3);
mongod3 = mongodExe3.start();
String server1 = Network.getLocalHost().getHostName() + ":" + mongodConfig1.net().getPort();
String server2 = Network.getLocalHost().getHostName() + ":" + mongodConfig2.net().getPort();
String server3 = Network.getLocalHost().getHostName() + ":" + mongodConfig3.net().getPort();
logger.debug("Server #1: {}", server1);
logger.debug("Server #2: {}", server2);
logger.debug("Server #3: {}", server3);
Thread.sleep(2000);
MongoClientOptions mco =
MongoClientOptions.builder()
.autoConnectRetry(true)
.connectTimeout(15000)
.socketTimeout(60000)
.build();
mongo =
new MongoClient(
new ServerAddress(Network.getLocalHost().getHostName(), mongodConfig1.net().getPort()),
mco);
mongoAdminDB = mongo.getDB(ADMIN_DATABASE_NAME);
cr = mongoAdminDB.command(new BasicDBObject("isMaster", 1));
logger.debug("isMaster: " + cr);
// Initialize replica set
cr =
mongoAdminDB.command(
new BasicDBObject(
"replSetInitiate",
(DBObject)
JSON.parse(
"{'_id': '"
+ REPLICA_SET_NAME
+ "', 'members': [{'_id': 0, 'host': '"
+ server1
+ "'}, {'_id': 1, 'host': '"
+ server2
+ "'}, {'_id': 2, 'host': '"
+ server3
+ "', 'arbiterOnly' : true}]} }")));
logger.debug("replSetInitiate: " + cr);
Thread.sleep(5000);
cr = mongoAdminDB.command(new BasicDBObject("replSetGetStatus", 1));
logger.info("replSetGetStatus: " + cr);
// Check replica set status before to proceed
while (!isReplicaSetStarted(cr)) {
logger.debug("Waiting for 3 seconds...");
Thread.sleep(3000);
cr = mongoAdminDB.command(new BasicDBObject("replSetGetStatus", 1));
logger.debug("replSetGetStatus: " + cr);
}
mongo.close();
mongo = null;
// Initialize a new client using all instances.
List<ServerAddress> mongoServers = new ArrayList<ServerAddress>();
mongoServers.add(
new ServerAddress(Network.getLocalHost().getHostName(), mongodConfig1.net().getPort()));
mongoServers.add(
new ServerAddress(Network.getLocalHost().getHostName(), mongodConfig2.net().getPort()));
mongoServers.add(
new ServerAddress(Network.getLocalHost().getHostName(), mongodConfig3.net().getPort()));
mongo = new MongoClient(mongoServers, mco);
Assert.assertNotNull(mongo);
mongo.setReadPreference(ReadPreference.secondaryPreferred());
mongo.setWriteConcern(WriteConcern.REPLICAS_SAFE);
}
private Builder builder(MongoClientOptions options) {
Builder builder = MongoClientOptions.builder();
if (options != null) {
builder.alwaysUseMBeans(options.isAlwaysUseMBeans());
builder.connectionsPerHost(options.getConnectionsPerHost());
builder.connectTimeout(options.getConnectTimeout());
builder.cursorFinalizerEnabled(options.isCursorFinalizerEnabled());
builder.dbDecoderFactory(options.getDbDecoderFactory());
builder.dbEncoderFactory(options.getDbEncoderFactory());
builder.description(options.getDescription());
builder.maxWaitTime(options.getMaxWaitTime());
builder.readPreference(options.getReadPreference());
builder.socketFactory(options.getSocketFactory());
builder.socketKeepAlive(options.isSocketKeepAlive());
builder.socketTimeout(options.getSocketTimeout());
builder.threadsAllowedToBlockForConnectionMultiplier(
options.getThreadsAllowedToBlockForConnectionMultiplier());
builder.writeConcern(options.getWriteConcern());
}
return builder;
}
/**
* 构件MongoDB连接选项<br>
*
* @param group 分组,当分组对应的选项不存在时会读取根选项，如果也不存在使用默认值
* @return MongoClientOptions
*/
private MongoClientOptions buildMongoClientOptions(String group) {
return buildMongoClientOptions(MongoClientOptions.builder(), group).build();
}
