@Override
public Mapper.Builder parse(
String fieldName, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
TextFieldMapper.Builder builder = new TextFieldMapper.Builder(fieldName);
builder.fieldType().setIndexAnalyzer(parserContext.analysisService().defaultIndexAnalyzer());
builder
.fieldType()
.setSearchAnalyzer(parserContext.analysisService().defaultSearchAnalyzer());
builder
.fieldType()
.setSearchQuoteAnalyzer(parserContext.analysisService().defaultSearchQuoteAnalyzer());
parseTextField(builder, fieldName, node, parserContext);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
iterator.hasNext(); ) {
Map.Entry<String, Object> entry = iterator.next();
String propName = entry.getKey();
Object propNode = entry.getValue();
if (propName.equals("position_increment_gap")) {
int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1);
builder.positionIncrementGap(newPositionIncrementGap);
iterator.remove();
} else if (propName.equals("fielddata")) {
builder.fielddata(XContentMapValues.nodeBooleanValue(propNode));
iterator.remove();
} else if (propName.equals("eager_global_ordinals")) {
builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode));
iterator.remove();
} else if (propName.equals("fielddata_frequency_filter")) {
Map<?, ?> frequencyFilter = (Map<?, ?>) propNode;
double minFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("min"), 0);
double maxFrequency =
XContentMapValues.nodeDoubleValue(frequencyFilter.remove("max"), Integer.MAX_VALUE);
int minSegmentSize =
XContentMapValues.nodeIntegerValue(frequencyFilter.remove("min_segment_size"), 0);
builder.fielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize);
DocumentMapperParser.checkNoRemainingFields(
propName, frequencyFilter, parserContext.indexVersionCreated());
iterator.remove();
}
}
return builder;
}
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
Builder builder = geoShapeField(name);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
iterator.hasNext(); ) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (Names.TREE.equals(fieldName)) {
builder.fieldType().setTree(fieldNode.toString());
iterator.remove();
} else if (Names.TREE_LEVELS.equals(fieldName)) {
builder.fieldType().setTreeLevels(Integer.parseInt(fieldNode.toString()));
iterator.remove();
} else if (Names.TREE_PRESISION.equals(fieldName)) {
builder
.fieldType()
.setPrecisionInMeters(
DistanceUnit.parse(
fieldNode.toString(), DistanceUnit.DEFAULT, DistanceUnit.DEFAULT));
iterator.remove();
} else if (Names.DISTANCE_ERROR_PCT.equals(fieldName)) {
builder.fieldType().setDistanceErrorPct(Double.parseDouble(fieldNode.toString()));
iterator.remove();
} else if (Names.ORIENTATION.equals(fieldName)) {
builder
.fieldType()
.setOrientation(ShapeBuilder.orientationFromString(fieldNode.toString()));
iterator.remove();
} else if (Names.STRATEGY.equals(fieldName)) {
builder.fieldType().setStrategyName(fieldNode.toString());
iterator.remove();
} else if (Names.COERCE.equals(fieldName)) {
builder.coerce(nodeBooleanValue(fieldNode));
iterator.remove();
} else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName)
&& builder.fieldType().strategyName.equals(SpatialStrategy.TERM.getStrategyName())
== false) {
builder.fieldType().setPointsOnly(XContentMapValues.nodeBooleanValue(fieldNode));
iterator.remove();
}
}
return builder;
}
/**
* Configure the river.
*
* @param settings used for configuration.
*/
@SuppressWarnings({"unchecked"})
protected void configure(Map<String, Object> settings) {
if (!closed) throw new IllegalStateException("Remote River must be stopped to configure it!");
if (settings.containsKey("remote")) {
Map<String, Object> remoteSettings = (Map<String, Object>) settings.get("remote");
maxIndexingThreads =
XContentMapValues.nodeIntegerValue(remoteSettings.get("maxIndexingThreads"), 1);
SpaceIndexingMode sim =
SpaceIndexingMode.parseConfiguration((String) remoteSettings.get("listDocumentsMode"));
if (sim != null) spaceIndexingMode = sim;
else if (XContentMapValues.nodeBooleanValue(remoteSettings.get("simpleGetDocuments"), false))
spaceIndexingMode = SpaceIndexingMode.SIMPLE;
indexFullUpdatePeriod =
Utils.parseTimeValue(remoteSettings, "indexFullUpdatePeriod", 12, TimeUnit.HOURS);
String ifuce = Utils.trimToNull((String) remoteSettings.get("indexFullUpdateCronExpression"));
if (ifuce != null) {
try {
this.indexFullUpdateCronExpression = new CronExpression(ifuce);
} catch (ParseException e) {
throw new SettingsException(
"Cron expression in indexFullUpdateCronExpression is invalid: " + e.getMessage());
}
}
if (spaceIndexingMode.isIncrementalUpdateSupported()
|| (indexFullUpdatePeriod < 1 && indexFullUpdateCronExpression == null))
indexUpdatePeriod =
Utils.parseTimeValue(remoteSettings, "indexUpdatePeriod", 5, TimeUnit.MINUTES);
else indexUpdatePeriod = 0;
if (remoteSettings.containsKey("spacesIndexed")) {
allIndexedSpacesKeys =
Utils.parseCsvString(
XContentMapValues.nodeStringValue(remoteSettings.get("spacesIndexed"), null));
if (allIndexedSpacesKeys != null) {
// stop spaces loading from remote system
allIndexedSpacesKeysNextRefresh = Long.MAX_VALUE;
}
}
if (remoteSettings.containsKey("spaceKeysExcluded")) {
spaceKeysExcluded =
Utils.parseCsvString(
XContentMapValues.nodeStringValue(remoteSettings.get("spaceKeysExcluded"), null));
}
String remoteClientClass =
Utils.trimToNull(
XContentMapValues.nodeStringValue(remoteSettings.get("remoteClientClass"), null));
if (remoteClientClass != null) {
try {
remoteSystemClient = (IRemoteSystemClient) Class.forName(remoteClientClass).newInstance();
} catch (Exception e) {
throw new SettingsException(
"Unable to instantiate class defined by 'remote/remoteClientClass': "
+ e.getMessage());
}
} else {
remoteSystemClient = new GetJSONClient();
}
remoteSystemClient.init(
this, remoteSettings, allIndexedSpacesKeysNextRefresh != Long.MAX_VALUE, this);
} else {
throw new SettingsException("'remote' element of river configuration structure not found");
}
Map<String, Object> indexSettings = null;
if (settings.containsKey("index")) {
indexSettings = (Map<String, Object>) settings.get("index");
indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name());
typeName =
XContentMapValues.nodeStringValue(
indexSettings.get("type"), INDEX_DOCUMENT_TYPE_NAME_DEFAULT);
} else {
indexName = riverName.name();
typeName = INDEX_DOCUMENT_TYPE_NAME_DEFAULT;
}
Map<String, Object> activityLogSettings = null;
if (settings.containsKey("activity_log")) {
activityLogSettings = (Map<String, Object>) settings.get("activity_log");
activityLogIndexName =
Utils.trimToNull(
XContentMapValues.nodeStringValue(activityLogSettings.get("index"), null));
if (activityLogIndexName == null) {
throw new SettingsException(
"'activity_log/index' element of river configuration structure must be defined with some string");
}
activityLogTypeName =
Utils.trimToNull(
XContentMapValues.nodeStringValue(
activityLogSettings.get("type"), INDEX_ACTIVITY_TYPE_NAME_DEFAULT));
}
documentIndexStructureBuilder =
new DocumentWithCommentsIndexStructureBuilder(
this, indexName, typeName, indexSettings, spaceIndexingMode.isUpdateDateMandatory());
preparePreprocessors(indexSettings, documentIndexStructureBuilder);
remoteSystemClient.setIndexStructureBuilder(documentIndexStructureBuilder);
logger.info(
"Configured Remote River '{}'. Search index name '{}', document type for issues '{}'. Indexing mode '{}'.",
riverName.getName(),
indexName,
typeName,
spaceIndexingMode.getConfigValue());
if (activityLogIndexName != null) {
logger.info(
"Activity log for Remote River '{}' is enabled. Search index name '{}', document type for index updates '{}'.",
riverName.getName(),
activityLogIndexName,
activityLogTypeName);
}
}
@SuppressWarnings({"unchecked"})
@Inject
public RabbitmqRiver(
RiverName riverName, RiverSettings settings, Client client, ScriptService scriptService) {
super(riverName, settings);
this.client = client;
if (settings.settings().containsKey("rabbitmq")) {
Map<String, Object> rabbitSettings =
(Map<String, Object>) settings.settings().get("rabbitmq");
if (rabbitSettings.containsKey("addresses")) {
List<Address> addresses = new ArrayList<Address>();
for (Map<String, Object> address :
(List<Map<String, Object>>) rabbitSettings.get("addresses")) {
addresses.add(
new Address(
XContentMapValues.nodeStringValue(address.get("host"), "localhost"),
XContentMapValues.nodeIntegerValue(address.get("port"), AMQP.PROTOCOL.PORT)));
}
rabbitAddresses = addresses.toArray(new Address[addresses.size()]);
} else {
String rabbitHost =
XContentMapValues.nodeStringValue(rabbitSettings.get("host"), "localhost");
int rabbitPort =
XContentMapValues.nodeIntegerValue(rabbitSettings.get("port"), AMQP.PROTOCOL.PORT);
rabbitAddresses = new Address[] {new Address(rabbitHost, rabbitPort)};
}
rabbitUser = XContentMapValues.nodeStringValue(rabbitSettings.get("user"), "guest");
rabbitPassword = XContentMapValues.nodeStringValue(rabbitSettings.get("pass"), "guest");
rabbitVhost = XContentMapValues.nodeStringValue(rabbitSettings.get("vhost"), "/");
rabbitQueue = XContentMapValues.nodeStringValue(rabbitSettings.get("queue"), "elasticsearch");
rabbitExchange =
XContentMapValues.nodeStringValue(rabbitSettings.get("exchange"), "elasticsearch");
rabbitRoutingKey =
XContentMapValues.nodeStringValue(rabbitSettings.get("routing_key"), "elasticsearch");
rabbitExchangeDeclare =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("exchange_declare"), true);
if (rabbitExchangeDeclare) {
rabbitExchangeType =
XContentMapValues.nodeStringValue(rabbitSettings.get("exchange_type"), "direct");
rabbitExchangeDurable =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("exchange_durable"), true);
} else {
rabbitExchangeType = "direct";
rabbitExchangeDurable = true;
}
rabbitQueueDeclare =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_declare"), true);
if (rabbitQueueDeclare) {
rabbitQueueDurable =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_durable"), true);
rabbitQueueAutoDelete =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_auto_delete"), false);
if (rabbitSettings.containsKey("args")) {
rabbitQueueArgs = (Map<String, Object>) rabbitSettings.get("args");
}
} else {
rabbitQueueDurable = true;
rabbitQueueAutoDelete = false;
}
rabbitQueueBind = XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_bind"), true);
rabbitQosPrefetchSize =
XContentMapValues.nodeIntegerValue(rabbitSettings.get("qos_prefetch_size"), 0);
rabbitQosPrefetchCount =
XContentMapValues.nodeIntegerValue(rabbitSettings.get("qos_prefetch_count"), 10);
rabbitHeartbeat =
TimeValue.parseTimeValue(
XContentMapValues.nodeStringValue(rabbitSettings.get("heartbeat"), "30m"),
TimeValue.timeValueMinutes(30));
} else {
rabbitAddresses = new Address[] {new Address("localhost", AMQP.PROTOCOL.PORT)};
rabbitUser = "******";
rabbitPassword = "******";
rabbitVhost = "/";
rabbitQueue = "elasticsearch";
rabbitQueueAutoDelete = false;
rabbitQueueDurable = true;
rabbitExchange = "elasticsearch";
rabbitExchangeType = "direct";
rabbitExchangeDurable = true;
rabbitRoutingKey = "elasticsearch";
rabbitExchangeDeclare = true;
rabbitQueueDeclare = true;
rabbitQueueBind = true;
rabbitQosPrefetchSize = 0;
rabbitQosPrefetchCount = 10;
rabbitHeartbeat = TimeValue.timeValueMinutes(30);
}
if (settings.settings().containsKey("index")) {
Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index");
bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size"), 100);
if (indexSettings.containsKey("bulk_timeout")) {
bulkTimeout =
TimeValue.parseTimeValue(
XContentMapValues.nodeStringValue(indexSettings.get("bulk_timeout"), "10ms"),
TimeValue.timeValueMillis(10));
} else {
bulkTimeout = TimeValue.timeValueMillis(10);
}
ordered = XContentMapValues.nodeBooleanValue(indexSettings.get("ordered"), false);
} else {
bulkSize = 100;
bulkTimeout = TimeValue.timeValueMillis(10);
ordered = false;
}
if (settings.settings().containsKey("bulk_script_filter")) {
Map<String, Object> scriptSettings =
(Map<String, Object>) settings.settings().get("bulk_script_filter");
if (scriptSettings.containsKey("script")) {
String scriptLang = "native";
if (scriptSettings.containsKey("script_lang")) {
scriptLang = scriptSettings.get("script_lang").toString();
}
Map<String, Object> scriptParams = null;
if (scriptSettings.containsKey("script_params")) {
scriptParams = (Map<String, Object>) scriptSettings.get("script_params");
} else {
scriptParams = Maps.newHashMap();
}
bulkScript =
scriptService.executable(
scriptLang, scriptSettings.get("script").toString(), scriptParams);
} else {
bulkScript = null;
}
} else {
bulkScript = null;
}
if (settings.settings().containsKey("script_filter")) {
Map<String, Object> scriptSettings =
(Map<String, Object>) settings.settings().get("script_filter");
if (scriptSettings.containsKey("script")) {
String scriptLang = "mvel";
if (scriptSettings.containsKey("script_lang")) {
scriptLang = scriptSettings.get("script_lang").toString();
}
Map<String, Object> scriptParams = null;
if (scriptSettings.containsKey("script_params")) {
scriptParams = (Map<String, Object>) scriptSettings.get("script_params");
} else {
scriptParams = Maps.newHashMap();
}
script =
scriptService.executable(
scriptLang, scriptSettings.get("script").toString(), scriptParams);
} else {
script = null;
}
} else {
script = null;
}
}
@SuppressWarnings("unchecked")
public static synchronized MongoDBRiverDefinition parseSettings(
String riverName,
String riverIndexName,
RiverSettings settings,
ScriptService scriptService) {
logger.trace("Parse river settings for {}", riverName);
Preconditions.checkNotNull(riverName, "No riverName specified");
Preconditions.checkNotNull(riverIndexName, "No riverIndexName specified");
Preconditions.checkNotNull(settings, "No settings specified");
Builder builder = new Builder();
builder.riverName(riverName);
builder.riverIndexName(riverIndexName);
List<ServerAddress> mongoServers = new ArrayList<ServerAddress>();
String mongoHost;
int mongoPort;
if (settings.settings().containsKey(MongoDBRiver.TYPE)) {
Map<String, Object> mongoSettings =
(Map<String, Object>) settings.settings().get(MongoDBRiver.TYPE);
if (mongoSettings.containsKey(SERVERS_FIELD)) {
Object mongoServersSettings = mongoSettings.get(SERVERS_FIELD);
logger.trace("mongoServersSettings: " + mongoServersSettings);
boolean array = XContentMapValues.isArray(mongoServersSettings);
if (array) {
ArrayList<Map<String, Object>> feeds =
(ArrayList<Map<String, Object>>) mongoServersSettings;
for (Map<String, Object> feed : feeds) {
mongoHost = XContentMapValues.nodeStringValue(feed.get(HOST_FIELD), null);
mongoPort = XContentMapValues.nodeIntegerValue(feed.get(PORT_FIELD), DEFAULT_DB_PORT);
logger.trace("Server: " + mongoHost + " - " + mongoPort);
try {
mongoServers.add(new ServerAddress(mongoHost, mongoPort));
} catch (UnknownHostException uhEx) {
logger.warn("Cannot add mongo server {}:{}", uhEx, mongoHost, mongoPort);
}
}
}
} else {
mongoHost =
XContentMapValues.nodeStringValue(mongoSettings.get(HOST_FIELD), DEFAULT_DB_HOST);
mongoPort =
XContentMapValues.nodeIntegerValue(mongoSettings.get(PORT_FIELD), DEFAULT_DB_PORT);
try {
mongoServers.add(new ServerAddress(mongoHost, mongoPort));
} catch (UnknownHostException uhEx) {
logger.warn("Cannot add mongo server {}:{}", uhEx, mongoHost, mongoPort);
}
}
builder.mongoServers(mongoServers);
MongoClientOptions.Builder mongoClientOptionsBuilder =
MongoClientOptions.builder().socketKeepAlive(true);
// MongoDB options
if (mongoSettings.containsKey(OPTIONS_FIELD)) {
Map<String, Object> mongoOptionsSettings =
(Map<String, Object>) mongoSettings.get(OPTIONS_FIELD);
logger.trace("mongoOptionsSettings: " + mongoOptionsSettings);
builder.mongoSecondaryReadPreference(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SECONDARY_READ_PREFERENCE_FIELD), false));
builder.connectTimeout(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(CONNECT_TIMEOUT), DEFAULT_CONNECT_TIMEOUT));
builder.socketTimeout(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(SOCKET_TIMEOUT), DEFAULT_SOCKET_TIMEOUT));
builder.dropCollection(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(DROP_COLLECTION_FIELD), false));
String isMongos =
XContentMapValues.nodeStringValue(mongoOptionsSettings.get(IS_MONGOS_FIELD), null);
if (isMongos != null) {
builder.isMongos(Boolean.valueOf(isMongos));
}
builder.mongoUseSSL(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SSL_CONNECTION_FIELD), false));
builder.mongoSSLVerifyCertificate(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SSL_VERIFY_CERT_FIELD), true));
builder.advancedTransformation(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(ADVANCED_TRANSFORMATION_FIELD), false));
builder.skipInitialImport(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SKIP_INITIAL_IMPORT_FIELD), false));
builder.connectionsPerHost(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(CONNECTIONS_PER_HOST), DEFAULT_CONNECTIONS_PER_HOST));
builder.threadsAllowedToBlockForConnectionMultiplier(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER),
DEFAULT_THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER));
mongoClientOptionsBuilder
.connectTimeout(builder.connectTimeout)
.socketTimeout(builder.socketTimeout)
.connectionsPerHost(builder.connectionsPerHost)
.threadsAllowedToBlockForConnectionMultiplier(
builder.threadsAllowedToBlockForConnectionMultiplier);
if (builder.mongoSecondaryReadPreference) {
mongoClientOptionsBuilder.readPreference(ReadPreference.secondaryPreferred());
}
if (builder.mongoUseSSL) {
mongoClientOptionsBuilder.socketFactory(getSSLSocketFactory());
}
if (mongoOptionsSettings.containsKey(PARENT_TYPES_FIELD)) {
Set<String> parentTypes = new HashSet<String>();
Object parentTypesSettings = mongoOptionsSettings.get(PARENT_TYPES_FIELD);
logger.trace("parentTypesSettings: " + parentTypesSettings);
boolean array = XContentMapValues.isArray(parentTypesSettings);
if (array) {
ArrayList<String> fields = (ArrayList<String>) parentTypesSettings;
for (String field : fields) {
logger.trace("Field: " + field);
parentTypes.add(field);
}
}
builder.parentTypes(parentTypes);
}
if (mongoOptionsSettings.containsKey(STORE_STATISTICS_FIELD)) {
Object storeStatistics = mongoOptionsSettings.get(STORE_STATISTICS_FIELD);
boolean object = XContentMapValues.isObject(storeStatistics);
if (object) {
Map<String, Object> storeStatisticsSettings = (Map<String, Object>) storeStatistics;
builder.storeStatistics(true);
builder.statisticsIndexName(
XContentMapValues.nodeStringValue(
storeStatisticsSettings.get(INDEX_OBJECT), riverName + "-stats"));
builder.statisticsTypeName(
XContentMapValues.nodeStringValue(
storeStatisticsSettings.get(TYPE_FIELD), "stats"));
} else {
builder.storeStatistics(XContentMapValues.nodeBooleanValue(storeStatistics, false));
if (builder.storeStatistics) {
builder.statisticsIndexName(riverName + "-stats");
builder.statisticsTypeName("stats");
}
}
}
// builder.storeStatistics(XContentMapValues.nodeBooleanValue(mongoOptionsSettings.get(STORE_STATISTICS_FIELD),
// false));
builder.importAllCollections(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(IMPORT_ALL_COLLECTIONS_FIELD), false));
builder.disableIndexRefresh(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(DISABLE_INDEX_REFRESH_FIELD), false));
builder.includeCollection(
XContentMapValues.nodeStringValue(
mongoOptionsSettings.get(INCLUDE_COLLECTION_FIELD), ""));
if (mongoOptionsSettings.containsKey(INCLUDE_FIELDS_FIELD)) {
Set<String> includeFields = new HashSet<String>();
Object includeFieldsSettings = mongoOptionsSettings.get(INCLUDE_FIELDS_FIELD);
logger.trace("includeFieldsSettings: " + includeFieldsSettings);
boolean array = XContentMapValues.isArray(includeFieldsSettings);
if (array) {
ArrayList<String> fields = (ArrayList<String>) includeFieldsSettings;
for (String field : fields) {
logger.trace("Field: " + field);
includeFields.add(field);
}
}
if (!includeFields.contains(MongoDBRiver.MONGODB_ID_FIELD)) {
includeFields.add(MongoDBRiver.MONGODB_ID_FIELD);
}
builder.includeFields(includeFields);
} else if (mongoOptionsSettings.containsKey(EXCLUDE_FIELDS_FIELD)) {
Set<String> excludeFields = new HashSet<String>();
Object excludeFieldsSettings = mongoOptionsSettings.get(EXCLUDE_FIELDS_FIELD);
logger.trace("excludeFieldsSettings: " + excludeFieldsSettings);
boolean array = XContentMapValues.isArray(excludeFieldsSettings);
if (array) {
ArrayList<String> fields = (ArrayList<String>) excludeFieldsSettings;
for (String field : fields) {
logger.trace("Field: " + field);
excludeFields.add(field);
}
}
builder.excludeFields(excludeFields);
}
if (mongoOptionsSettings.containsKey(INITIAL_TIMESTAMP_FIELD)) {
BSONTimestamp timeStamp = null;
try {
Map<String, Object> initalTimestampSettings =
(Map<String, Object>) mongoOptionsSettings.get(INITIAL_TIMESTAMP_FIELD);
String scriptType = "js";
if (initalTimestampSettings.containsKey(INITIAL_TIMESTAMP_SCRIPT_TYPE_FIELD)) {
scriptType =
initalTimestampSettings.get(INITIAL_TIMESTAMP_SCRIPT_TYPE_FIELD).toString();
}
if (initalTimestampSettings.containsKey(INITIAL_TIMESTAMP_SCRIPT_FIELD)) {
ExecutableScript scriptExecutable =
scriptService.executable(
scriptType,
initalTimestampSettings.get(INITIAL_TIMESTAMP_SCRIPT_FIELD).toString(),
ScriptService.ScriptType.INLINE,
Maps.newHashMap());
Object ctx = scriptExecutable.run();
logger.trace("initialTimestamp script returned: {}", ctx);
if (ctx != null) {
long timestamp = Long.parseLong(ctx.toString());
timeStamp = new BSONTimestamp((int) (new Date(timestamp).getTime() / 1000), 1);
}
}
} catch (Throwable t) {
logger.error("Could not set initial timestamp", t);
} finally {
builder.initialTimestamp(timeStamp);
}
}
}
builder.mongoClientOptions(mongoClientOptionsBuilder.build());
// Credentials
if (mongoSettings.containsKey(CREDENTIALS_FIELD)) {
String dbCredential;
String mau = "";
String map = "";
String maad = "";
String mlu = "";
String mlp = "";
String mlad = "";
// String mdu = "";
// String mdp = "";
Object mongoCredentialsSettings = mongoSettings.get(CREDENTIALS_FIELD);
boolean array = XContentMapValues.isArray(mongoCredentialsSettings);
if (array) {
ArrayList<Map<String, Object>> credentials =
(ArrayList<Map<String, Object>>) mongoCredentialsSettings;
for (Map<String, Object> credential : credentials) {
dbCredential = XContentMapValues.nodeStringValue(credential.get(DB_FIELD), null);
if (ADMIN_DB_FIELD.equals(dbCredential)) {
mau = XContentMapValues.nodeStringValue(credential.get(USER_FIELD), null);
map = XContentMapValues.nodeStringValue(credential.get(PASSWORD_FIELD), null);
maad = XContentMapValues.nodeStringValue(credential.get(AUTH_FIELD), null);
} else if (LOCAL_DB_FIELD.equals(dbCredential)) {
mlu = XContentMapValues.nodeStringValue(credential.get(USER_FIELD), null);
mlp = XContentMapValues.nodeStringValue(credential.get(PASSWORD_FIELD), null);
mlad = XContentMapValues.nodeStringValue(credential.get(AUTH_FIELD), null);
// } else {
// mdu = XContentMapValues.nodeStringValue(
// credential.get(USER_FIELD), null);
// mdp = XContentMapValues.nodeStringValue(
// credential.get(PASSWORD_FIELD), null);
}
}
}
builder.mongoAdminUser(mau);
builder.mongoAdminPassword(map);
builder.mongoAdminAuthDatabase(maad);
builder.mongoLocalUser(mlu);
builder.mongoLocalPassword(mlp);
builder.mongoLocalAuthDatabase(mlad);
// mongoDbUser = mdu;
// mongoDbPassword = mdp;
}
builder.mongoDb(XContentMapValues.nodeStringValue(mongoSettings.get(DB_FIELD), riverName));
builder.mongoCollection(
XContentMapValues.nodeStringValue(mongoSettings.get(COLLECTION_FIELD), riverName));
builder.mongoGridFS(
XContentMapValues.nodeBooleanValue(mongoSettings.get(GRIDFS_FIELD), false));
if (mongoSettings.containsKey(FILTER_FIELD)) {
String filter = XContentMapValues.nodeStringValue(mongoSettings.get(FILTER_FIELD), "");
filter = removePrefix("o.", filter);
builder.mongoCollectionFilter(convertToBasicDBObject(filter));
// DBObject bsonObject = (DBObject) JSON.parse(filter);
// builder.mongoOplogFilter(convertToBasicDBObject(addPrefix("o.",
// filter)));
builder.mongoOplogFilter(convertToBasicDBObject(removePrefix("o.", filter)));
// } else {
// builder.mongoOplogFilter("");
}
if (mongoSettings.containsKey(SCRIPT_FIELD)) {
String scriptType = "js";
builder.script(mongoSettings.get(SCRIPT_FIELD).toString());
if (mongoSettings.containsKey("scriptType")) {
scriptType = mongoSettings.get("scriptType").toString();
} else if (mongoSettings.containsKey(SCRIPT_TYPE_FIELD)) {
scriptType = mongoSettings.get(SCRIPT_TYPE_FIELD).toString();
}
builder.scriptType(scriptType);
}
} else {
mongoHost = DEFAULT_DB_HOST;
mongoPort = DEFAULT_DB_PORT;
try {
mongoServers.add(new ServerAddress(mongoHost, mongoPort));
builder.mongoServers(mongoServers);
} catch (UnknownHostException e) {
e.printStackTrace();
}
builder.mongoDb(riverName);
builder.mongoCollection(riverName);
}
if (settings.settings().containsKey(INDEX_OBJECT)) {
Map<String, Object> indexSettings =
(Map<String, Object>) settings.settings().get(INDEX_OBJECT);
builder.indexName(
XContentMapValues.nodeStringValue(indexSettings.get(NAME_FIELD), builder.mongoDb));
builder.typeName(
XContentMapValues.nodeStringValue(indexSettings.get(TYPE_FIELD), builder.mongoDb));
Bulk.Builder bulkBuilder = new Bulk.Builder();
if (indexSettings.containsKey(BULK_FIELD)) {
Map<String, Object> bulkSettings = (Map<String, Object>) indexSettings.get(BULK_FIELD);
int bulkActions =
XContentMapValues.nodeIntegerValue(
bulkSettings.get(ACTIONS_FIELD), DEFAULT_BULK_ACTIONS);
bulkBuilder.bulkActions(bulkActions);
String size =
XContentMapValues.nodeStringValue(
bulkSettings.get(SIZE_FIELD), DEFAULT_BULK_SIZE.toString());
bulkBuilder.bulkSize(ByteSizeValue.parseBytesSizeValue(size));
bulkBuilder.concurrentRequests(
XContentMapValues.nodeIntegerValue(
bulkSettings.get(CONCURRENT_REQUESTS_FIELD),
EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY)));
bulkBuilder.flushInterval(
XContentMapValues.nodeTimeValue(
bulkSettings.get(FLUSH_INTERVAL_FIELD), DEFAULT_FLUSH_INTERVAL));
builder.throttleSize(
XContentMapValues.nodeIntegerValue(
indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5));
} else {
int bulkActions =
XContentMapValues.nodeIntegerValue(
indexSettings.get(BULK_SIZE_FIELD), DEFAULT_BULK_ACTIONS);
bulkBuilder.bulkActions(bulkActions);
bulkBuilder.bulkSize(DEFAULT_BULK_SIZE);
bulkBuilder.flushInterval(
XContentMapValues.nodeTimeValue(
indexSettings.get(BULK_TIMEOUT_FIELD), DEFAULT_FLUSH_INTERVAL));
bulkBuilder.concurrentRequests(
XContentMapValues.nodeIntegerValue(
indexSettings.get(CONCURRENT_BULK_REQUESTS_FIELD),
EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY)));
builder.throttleSize(
XContentMapValues.nodeIntegerValue(
indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5));
}
builder.bulk(bulkBuilder.build());
} else {
builder.indexName(builder.mongoDb);
builder.typeName(builder.mongoDb);
builder.bulk(new Bulk.Builder().build());
}
return builder.build();
}
@Override
public Mapper.Builder<?, ?> parse(
String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
Builder builder = geoPointField(name);
final boolean indexCreatedBeforeV2_0 =
parserContext.indexVersionCreated().before(Version.V_2_0_0);
parseField(builder, name, node, parserContext);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
iterator.hasNext(); ) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("path")
&& parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) {
builder.multiFieldPathType(parsePathType(name, fieldNode.toString()));
iterator.remove();
} else if (fieldName.equals("lat_lon")) {
builder.enableLatLon(XContentMapValues.nodeBooleanValue(fieldNode));
iterator.remove();
} else if (fieldName.equals("geohash")) {
builder.enableGeoHash(XContentMapValues.nodeBooleanValue(fieldNode));
iterator.remove();
} else if (fieldName.equals("geohash_prefix")) {
builder.geohashPrefix(XContentMapValues.nodeBooleanValue(fieldNode));
if (XContentMapValues.nodeBooleanValue(fieldNode)) {
builder.enableGeoHash(true);
}
iterator.remove();
} else if (fieldName.equals("precision_step")) {
builder.precisionStep(XContentMapValues.nodeIntegerValue(fieldNode));
iterator.remove();
} else if (fieldName.equals("geohash_precision")) {
if (fieldNode instanceof Integer) {
builder.geoHashPrecision(XContentMapValues.nodeIntegerValue(fieldNode));
} else {
builder.geoHashPrecision(GeoUtils.geoHashLevelsForPrecision(fieldNode.toString()));
}
iterator.remove();
} else if (fieldName.equals(Names.IGNORE_MALFORMED)) {
if (builder.fieldType().coerce == false) {
builder.fieldType().ignoreMalformed = XContentMapValues.nodeBooleanValue(fieldNode);
}
iterator.remove();
} else if (indexCreatedBeforeV2_0 && fieldName.equals("validate")) {
if (builder.fieldType().ignoreMalformed == false) {
builder.fieldType().ignoreMalformed = !XContentMapValues.nodeBooleanValue(fieldNode);
}
iterator.remove();
} else if (indexCreatedBeforeV2_0 && fieldName.equals("validate_lon")) {
if (builder.fieldType().ignoreMalformed() == false) {
builder.fieldType().ignoreMalformed = !XContentMapValues.nodeBooleanValue(fieldNode);
}
iterator.remove();
} else if (indexCreatedBeforeV2_0 && fieldName.equals("validate_lat")) {
if (builder.fieldType().ignoreMalformed == false) {
builder.fieldType().ignoreMalformed = !XContentMapValues.nodeBooleanValue(fieldNode);
}
iterator.remove();
} else if (fieldName.equals(Names.COERCE)) {
builder.fieldType().coerce = XContentMapValues.nodeBooleanValue(fieldNode);
if (builder.fieldType().coerce == true) {
builder.fieldType().ignoreMalformed = true;
}
iterator.remove();
} else if (indexCreatedBeforeV2_0 && fieldName.equals("normalize")) {
builder.fieldType().coerce = XContentMapValues.nodeBooleanValue(fieldNode);
iterator.remove();
} else if (indexCreatedBeforeV2_0 && fieldName.equals("normalize_lat")) {
builder.fieldType().coerce = XContentMapValues.nodeBooleanValue(fieldNode);
iterator.remove();
} else if (indexCreatedBeforeV2_0 && fieldName.equals("normalize_lon")) {
if (builder.fieldType().coerce == false) {
builder.fieldType().coerce = XContentMapValues.nodeBooleanValue(fieldNode);
}
iterator.remove();
} else if (parseMultiField(builder, name, parserContext, fieldName, fieldNode)) {
iterator.remove();
}
}
return builder;
}
@SuppressWarnings({"unchecked"})
@Inject
public AmazonsqsRiver(RiverName riverName, RiverSettings settings, Client client) {
super(riverName, settings);
this.client = client;
if (settings.settings().containsKey("amazonsqs")) {
Map<String, Object> sqsSettings = (Map<String, Object>) settings.settings().get("amazonsqs");
REGION = XContentMapValues.nodeStringValue(sqsSettings.get("region"), "null");
ENDPOINT = XContentMapValues.nodeStringValue(sqsSettings.get("endpoint"), "null");
ACCESS_KEY = XContentMapValues.nodeStringValue(sqsSettings.get("access_key"), "null");
SECRET_KEY = XContentMapValues.nodeStringValue(sqsSettings.get("secret_key"), "null");
QUEUE_URL = XContentMapValues.nodeStringValue(sqsSettings.get("queue_url"), "null");
SLEEP = XContentMapValues.nodeIntegerValue(sqsSettings.get("sleep"), DEFAULT_SLEEP);
LONGPOLLING_INTERVAL =
XContentMapValues.nodeIntegerValue(
sqsSettings.get("longpolling_interval"), DEFAULT_LONGPOLLING_INTERVAL);
DEBUG = XContentMapValues.nodeBooleanValue(sqsSettings.get("debug"), DEFAULT_DEBUG);
} else {
REGION = settings.globalSettings().get("cloud.aws.region");
ENDPOINT = settings.globalSettings().get("cloud.aws.endpoint");
ACCESS_KEY = settings.globalSettings().get("cloud.aws.access_key");
SECRET_KEY = settings.globalSettings().get("cloud.aws.secret_key");
QUEUE_URL = settings.globalSettings().get("cloud.aws.sqs.queue_url");
SLEEP = settings.globalSettings().getAsInt("cloud.aws.sqs.sleep", DEFAULT_SLEEP);
LONGPOLLING_INTERVAL =
settings
.globalSettings()
.getAsInt("cloud.aws.sqs.longpolling_interval", DEFAULT_LONGPOLLING_INTERVAL);
DEBUG = settings.globalSettings().getAsBoolean("cloud.aws.sqs.debug", DEFAULT_DEBUG);
}
if (settings.settings().containsKey("index")) {
Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index");
INDEX = XContentMapValues.nodeStringValue(indexSettings.get("index"), DEFAULT_INDEX);
MAX_MESSAGES =
XContentMapValues.nodeIntegerValue(
indexSettings.get("max_messages"), DEFAULT_MAX_MESSAGES);
} else {
INDEX = DEFAULT_INDEX;
MAX_MESSAGES = DEFAULT_MAX_MESSAGES;
}
if (ACCESS_KEY == null || ACCESS_KEY.trim().isEmpty() || ACCESS_KEY.equals("null")) {
sqs = new AmazonSQSAsyncClient();
} else {
sqs = new AmazonSQSAsyncClient(new BasicAWSCredentials(ACCESS_KEY, SECRET_KEY));
}
String endpoint = ENDPOINT;
if (ENDPOINT == null || ENDPOINT.trim().isEmpty() || ENDPOINT.equals("null")) {
endpoint = "https://sqs.".concat(REGION).concat(".amazonaws.com");
}
sqs.setEndpoint(endpoint);
if (DEBUG) {
if (ACCESS_KEY == null
|| ACCESS_KEY.length() < 2
|| SECRET_KEY == null
|| SECRET_KEY.length() < 2) {
logger.warn(
"AWS Credentials are not correct! "
.concat("Access Key: ")
.concat(ACCESS_KEY + " ")
.concat("Secret Key: ")
.concat(SECRET_KEY + " ")
.concat("Endpoint: ")
.concat(endpoint));
} else {
logger.info(
"AWS Credentials: "
.concat("Access Key length: ")
.concat(ACCESS_KEY.length() + " ")
.concat("Secret Key length: ")
.concat(SECRET_KEY.length() + " ")
.concat("Endpoint: ")
.concat(endpoint));
}
}
mapper = new ObjectMapper();
}