@SuppressWarnings({"unchecked"})
@Inject
public SimpleRiver(
RiverName riverName, RiverSettings settings, Client client, ThreadPool threadPool) {
super(riverName, settings);
this.client = client;
if (settings.settings().containsKey("simple")) {
Map<String, Object> simpleSettings = (Map<String, Object>) settings.settings().get("simple");
simpleNumber = XContentMapValues.nodeIntegerValue(simpleSettings.get("number"), 100);
fieldName = XContentMapValues.nodeStringValue(simpleSettings.get("field"), "test");
poll =
XContentMapValues.nodeTimeValue(
simpleSettings.get("poll"), TimeValue.timeValueMinutes(60));
}
logger.info(
"creating simple stream river for [{} numbers] with field [{}]", simpleNumber, fieldName);
if (settings.settings().containsKey("index")) {
Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index");
indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name());
typeName = XContentMapValues.nodeStringValue(indexSettings.get("type"), "simple_type");
bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size"), 100);
bulkThreshold = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_threshold"), 10);
} else {
indexName = riverName.name();
typeName = "simple_type";
bulkSize = 100;
bulkThreshold = 10;
}
}
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
StringFieldMapper.Builder builder = stringField(name);
parseField(builder, name, node, parserContext);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
iterator.hasNext(); ) {
Map.Entry<String, Object> entry = iterator.next();
String propName = Strings.toUnderscoreCase(entry.getKey());
Object propNode = entry.getValue();
if (propName.equals("null_value")) {
if (propNode == null) {
throw new MapperParsingException("Property [null_value] cannot be null.");
}
builder.nullValue(propNode.toString());
iterator.remove();
} else if (propName.equals("search_quote_analyzer")) {
NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
if (analyzer == null) {
throw new MapperParsingException(
"Analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
}
builder.searchQuotedAnalyzer(analyzer);
iterator.remove();
} else if (propName.equals("position_offset_gap")) {
builder.positionOffsetGap(XContentMapValues.nodeIntegerValue(propNode, -1));
// we need to update to actual analyzers if they are not set in this case...
// so we can inject the position offset gap...
if (builder.indexAnalyzer == null) {
builder.indexAnalyzer = parserContext.analysisService().defaultIndexAnalyzer();
}
if (builder.searchAnalyzer == null) {
builder.searchAnalyzer = parserContext.analysisService().defaultSearchAnalyzer();
}
if (builder.searchQuotedAnalyzer == null) {
builder.searchQuotedAnalyzer =
parserContext.analysisService().defaultSearchQuoteAnalyzer();
}
iterator.remove();
} else if (propName.equals("ignore_above")) {
builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
iterator.remove();
} else if (parseMultiField(builder, name, parserContext, propName, propNode)) {
iterator.remove();
}
}
return builder;
}
/** The template source definition. */
public PutIndexTemplateRequest source(Map templateSource) {
Map<String, Object> source = templateSource;
if (source.containsKey("template")) {
template(source.get("template").toString());
}
if (source.containsKey("order")) {
order(XContentMapValues.nodeIntegerValue(source.get("order"), order()));
}
if (source.containsKey("settings")) {
if (!(source.get("settings") instanceof Map)) {
throw new ElasticSearchIllegalArgumentException(
"Malformed settings section, should include an inner object");
}
settings((Map<String, Object>) source.get("settings"));
}
if (source.containsKey("mappings")) {
Map<String, Object> mappings = (Map<String, Object>) source.get("mappings");
for (Map.Entry<String, Object> entry : mappings.entrySet()) {
if (!(entry.getValue() instanceof Map)) {
throw new ElasticSearchIllegalArgumentException(
"Malformed mappings section for type ["
+ entry.getKey()
+ "], should include an inner object describing the mapping");
}
mapping(entry.getKey(), (Map<String, Object>) entry.getValue());
}
}
return this;
}
@Override
public Mapper.Builder parse(
String fieldName, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
TextFieldMapper.Builder builder = new TextFieldMapper.Builder(fieldName);
builder.fieldType().setIndexAnalyzer(parserContext.analysisService().defaultIndexAnalyzer());
builder
.fieldType()
.setSearchAnalyzer(parserContext.analysisService().defaultSearchAnalyzer());
builder
.fieldType()
.setSearchQuoteAnalyzer(parserContext.analysisService().defaultSearchQuoteAnalyzer());
parseTextField(builder, fieldName, node, parserContext);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
iterator.hasNext(); ) {
Map.Entry<String, Object> entry = iterator.next();
String propName = entry.getKey();
Object propNode = entry.getValue();
if (propName.equals("position_increment_gap")) {
int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1);
builder.positionIncrementGap(newPositionIncrementGap);
iterator.remove();
} else if (propName.equals("fielddata")) {
builder.fielddata(XContentMapValues.nodeBooleanValue(propNode));
iterator.remove();
} else if (propName.equals("eager_global_ordinals")) {
builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode));
iterator.remove();
} else if (propName.equals("fielddata_frequency_filter")) {
Map<?, ?> frequencyFilter = (Map<?, ?>) propNode;
double minFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("min"), 0);
double maxFrequency =
XContentMapValues.nodeDoubleValue(frequencyFilter.remove("max"), Integer.MAX_VALUE);
int minSegmentSize =
XContentMapValues.nodeIntegerValue(frequencyFilter.remove("min_segment_size"), 0);
builder.fielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize);
DocumentMapperParser.checkNoRemainingFields(
propName, frequencyFilter, parserContext.indexVersionCreated());
iterator.remove();
}
}
return builder;
}
/** The template source definition. */
public PutIndexTemplateRequest source(Map templateSource) {
Map<String, Object> source = templateSource;
for (Map.Entry<String, Object> entry : source.entrySet()) {
String name = entry.getKey();
if (name.equals("template")) {
template(entry.getValue().toString());
} else if (name.equals("order")) {
order(XContentMapValues.nodeIntegerValue(entry.getValue(), order()));
} else if (name.equals("settings")) {
if (!(entry.getValue() instanceof Map)) {
throw new ElasticSearchIllegalArgumentException(
"Malformed settings section, should include an inner object");
}
settings((Map<String, Object>) entry.getValue());
} else if (name.equals("mappings")) {
Map<String, Object> mappings = (Map<String, Object>) entry.getValue();
for (Map.Entry<String, Object> entry1 : mappings.entrySet()) {
if (!(entry1.getValue() instanceof Map)) {
throw new ElasticSearchIllegalArgumentException(
"Malformed mappings section for type ["
+ entry1.getKey()
+ "], should include an inner object describing the mapping");
}
mapping(entry1.getKey(), (Map<String, Object>) entry1.getValue());
}
} else {
// maybe custom?
IndexMetaData.Custom.Factory factory = IndexMetaData.lookupFactory(name);
if (factory != null) {
try {
customs.put(name, factory.fromMap((Map<String, Object>) entry.getValue()));
} catch (IOException e) {
throw new ElasticSearchParseException(
"failed to parse custom metadata for [" + name + "]");
}
}
}
}
return this;
}
/**
* Configure the river.
*
* @param settings used for configuration.
*/
@SuppressWarnings({"unchecked"})
protected void configure(Map<String, Object> settings) {
if (!closed) throw new IllegalStateException("Remote River must be stopped to configure it!");
if (settings.containsKey("remote")) {
Map<String, Object> remoteSettings = (Map<String, Object>) settings.get("remote");
maxIndexingThreads =
XContentMapValues.nodeIntegerValue(remoteSettings.get("maxIndexingThreads"), 1);
SpaceIndexingMode sim =
SpaceIndexingMode.parseConfiguration((String) remoteSettings.get("listDocumentsMode"));
if (sim != null) spaceIndexingMode = sim;
else if (XContentMapValues.nodeBooleanValue(remoteSettings.get("simpleGetDocuments"), false))
spaceIndexingMode = SpaceIndexingMode.SIMPLE;
indexFullUpdatePeriod =
Utils.parseTimeValue(remoteSettings, "indexFullUpdatePeriod", 12, TimeUnit.HOURS);
String ifuce = Utils.trimToNull((String) remoteSettings.get("indexFullUpdateCronExpression"));
if (ifuce != null) {
try {
this.indexFullUpdateCronExpression = new CronExpression(ifuce);
} catch (ParseException e) {
throw new SettingsException(
"Cron expression in indexFullUpdateCronExpression is invalid: " + e.getMessage());
}
}
if (spaceIndexingMode.isIncrementalUpdateSupported()
|| (indexFullUpdatePeriod < 1 && indexFullUpdateCronExpression == null))
indexUpdatePeriod =
Utils.parseTimeValue(remoteSettings, "indexUpdatePeriod", 5, TimeUnit.MINUTES);
else indexUpdatePeriod = 0;
if (remoteSettings.containsKey("spacesIndexed")) {
allIndexedSpacesKeys =
Utils.parseCsvString(
XContentMapValues.nodeStringValue(remoteSettings.get("spacesIndexed"), null));
if (allIndexedSpacesKeys != null) {
// stop spaces loading from remote system
allIndexedSpacesKeysNextRefresh = Long.MAX_VALUE;
}
}
if (remoteSettings.containsKey("spaceKeysExcluded")) {
spaceKeysExcluded =
Utils.parseCsvString(
XContentMapValues.nodeStringValue(remoteSettings.get("spaceKeysExcluded"), null));
}
String remoteClientClass =
Utils.trimToNull(
XContentMapValues.nodeStringValue(remoteSettings.get("remoteClientClass"), null));
if (remoteClientClass != null) {
try {
remoteSystemClient = (IRemoteSystemClient) Class.forName(remoteClientClass).newInstance();
} catch (Exception e) {
throw new SettingsException(
"Unable to instantiate class defined by 'remote/remoteClientClass': "
+ e.getMessage());
}
} else {
remoteSystemClient = new GetJSONClient();
}
remoteSystemClient.init(
this, remoteSettings, allIndexedSpacesKeysNextRefresh != Long.MAX_VALUE, this);
} else {
throw new SettingsException("'remote' element of river configuration structure not found");
}
Map<String, Object> indexSettings = null;
if (settings.containsKey("index")) {
indexSettings = (Map<String, Object>) settings.get("index");
indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name());
typeName =
XContentMapValues.nodeStringValue(
indexSettings.get("type"), INDEX_DOCUMENT_TYPE_NAME_DEFAULT);
} else {
indexName = riverName.name();
typeName = INDEX_DOCUMENT_TYPE_NAME_DEFAULT;
}
Map<String, Object> activityLogSettings = null;
if (settings.containsKey("activity_log")) {
activityLogSettings = (Map<String, Object>) settings.get("activity_log");
activityLogIndexName =
Utils.trimToNull(
XContentMapValues.nodeStringValue(activityLogSettings.get("index"), null));
if (activityLogIndexName == null) {
throw new SettingsException(
"'activity_log/index' element of river configuration structure must be defined with some string");
}
activityLogTypeName =
Utils.trimToNull(
XContentMapValues.nodeStringValue(
activityLogSettings.get("type"), INDEX_ACTIVITY_TYPE_NAME_DEFAULT));
}
documentIndexStructureBuilder =
new DocumentWithCommentsIndexStructureBuilder(
this, indexName, typeName, indexSettings, spaceIndexingMode.isUpdateDateMandatory());
preparePreprocessors(indexSettings, documentIndexStructureBuilder);
remoteSystemClient.setIndexStructureBuilder(documentIndexStructureBuilder);
logger.info(
"Configured Remote River '{}'. Search index name '{}', document type for issues '{}'. Indexing mode '{}'.",
riverName.getName(),
indexName,
typeName,
spaceIndexingMode.getConfigValue());
if (activityLogIndexName != null) {
logger.info(
"Activity log for Remote River '{}' is enabled. Search index name '{}', document type for index updates '{}'.",
riverName.getName(),
activityLogIndexName,
activityLogTypeName);
}
}
@SuppressWarnings({"unchecked"})
@Inject
public RabbitmqRiver(
RiverName riverName, RiverSettings settings, Client client, ScriptService scriptService) {
super(riverName, settings);
this.client = client;
if (settings.settings().containsKey("rabbitmq")) {
Map<String, Object> rabbitSettings =
(Map<String, Object>) settings.settings().get("rabbitmq");
if (rabbitSettings.containsKey("addresses")) {
List<Address> addresses = new ArrayList<Address>();
for (Map<String, Object> address :
(List<Map<String, Object>>) rabbitSettings.get("addresses")) {
addresses.add(
new Address(
XContentMapValues.nodeStringValue(address.get("host"), "localhost"),
XContentMapValues.nodeIntegerValue(address.get("port"), AMQP.PROTOCOL.PORT)));
}
rabbitAddresses = addresses.toArray(new Address[addresses.size()]);
} else {
String rabbitHost =
XContentMapValues.nodeStringValue(rabbitSettings.get("host"), "localhost");
int rabbitPort =
XContentMapValues.nodeIntegerValue(rabbitSettings.get("port"), AMQP.PROTOCOL.PORT);
rabbitAddresses = new Address[] {new Address(rabbitHost, rabbitPort)};
}
rabbitUser = XContentMapValues.nodeStringValue(rabbitSettings.get("user"), "guest");
rabbitPassword = XContentMapValues.nodeStringValue(rabbitSettings.get("pass"), "guest");
rabbitVhost = XContentMapValues.nodeStringValue(rabbitSettings.get("vhost"), "/");
rabbitQueue = XContentMapValues.nodeStringValue(rabbitSettings.get("queue"), "elasticsearch");
rabbitExchange =
XContentMapValues.nodeStringValue(rabbitSettings.get("exchange"), "elasticsearch");
rabbitRoutingKey =
XContentMapValues.nodeStringValue(rabbitSettings.get("routing_key"), "elasticsearch");
rabbitExchangeDeclare =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("exchange_declare"), true);
if (rabbitExchangeDeclare) {
rabbitExchangeType =
XContentMapValues.nodeStringValue(rabbitSettings.get("exchange_type"), "direct");
rabbitExchangeDurable =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("exchange_durable"), true);
} else {
rabbitExchangeType = "direct";
rabbitExchangeDurable = true;
}
rabbitQueueDeclare =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_declare"), true);
if (rabbitQueueDeclare) {
rabbitQueueDurable =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_durable"), true);
rabbitQueueAutoDelete =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_auto_delete"), false);
if (rabbitSettings.containsKey("args")) {
rabbitQueueArgs = (Map<String, Object>) rabbitSettings.get("args");
}
} else {
rabbitQueueDurable = true;
rabbitQueueAutoDelete = false;
}
rabbitQueueBind = XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_bind"), true);
rabbitQosPrefetchSize =
XContentMapValues.nodeIntegerValue(rabbitSettings.get("qos_prefetch_size"), 0);
rabbitQosPrefetchCount =
XContentMapValues.nodeIntegerValue(rabbitSettings.get("qos_prefetch_count"), 10);
rabbitHeartbeat =
TimeValue.parseTimeValue(
XContentMapValues.nodeStringValue(rabbitSettings.get("heartbeat"), "30m"),
TimeValue.timeValueMinutes(30));
} else {
rabbitAddresses = new Address[] {new Address("localhost", AMQP.PROTOCOL.PORT)};
rabbitUser = "******";
rabbitPassword = "******";
rabbitVhost = "/";
rabbitQueue = "elasticsearch";
rabbitQueueAutoDelete = false;
rabbitQueueDurable = true;
rabbitExchange = "elasticsearch";
rabbitExchangeType = "direct";
rabbitExchangeDurable = true;
rabbitRoutingKey = "elasticsearch";
rabbitExchangeDeclare = true;
rabbitQueueDeclare = true;
rabbitQueueBind = true;
rabbitQosPrefetchSize = 0;
rabbitQosPrefetchCount = 10;
rabbitHeartbeat = TimeValue.timeValueMinutes(30);
}
if (settings.settings().containsKey("index")) {
Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index");
bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size"), 100);
if (indexSettings.containsKey("bulk_timeout")) {
bulkTimeout =
TimeValue.parseTimeValue(
XContentMapValues.nodeStringValue(indexSettings.get("bulk_timeout"), "10ms"),
TimeValue.timeValueMillis(10));
} else {
bulkTimeout = TimeValue.timeValueMillis(10);
}
ordered = XContentMapValues.nodeBooleanValue(indexSettings.get("ordered"), false);
} else {
bulkSize = 100;
bulkTimeout = TimeValue.timeValueMillis(10);
ordered = false;
}
if (settings.settings().containsKey("bulk_script_filter")) {
Map<String, Object> scriptSettings =
(Map<String, Object>) settings.settings().get("bulk_script_filter");
if (scriptSettings.containsKey("script")) {
String scriptLang = "native";
if (scriptSettings.containsKey("script_lang")) {
scriptLang = scriptSettings.get("script_lang").toString();
}
Map<String, Object> scriptParams = null;
if (scriptSettings.containsKey("script_params")) {
scriptParams = (Map<String, Object>) scriptSettings.get("script_params");
} else {
scriptParams = Maps.newHashMap();
}
bulkScript =
scriptService.executable(
scriptLang, scriptSettings.get("script").toString(), scriptParams);
} else {
bulkScript = null;
}
} else {
bulkScript = null;
}
if (settings.settings().containsKey("script_filter")) {
Map<String, Object> scriptSettings =
(Map<String, Object>) settings.settings().get("script_filter");
if (scriptSettings.containsKey("script")) {
String scriptLang = "mvel";
if (scriptSettings.containsKey("script_lang")) {
scriptLang = scriptSettings.get("script_lang").toString();
}
Map<String, Object> scriptParams = null;
if (scriptSettings.containsKey("script_params")) {
scriptParams = (Map<String, Object>) scriptSettings.get("script_params");
} else {
scriptParams = Maps.newHashMap();
}
script =
scriptService.executable(
scriptLang, scriptSettings.get("script").toString(), scriptParams);
} else {
script = null;
}
} else {
script = null;
}
}
@SuppressWarnings("unchecked")
public static synchronized MongoDBRiverDefinition parseSettings(
String riverName,
String riverIndexName,
RiverSettings settings,
ScriptService scriptService) {
logger.trace("Parse river settings for {}", riverName);
Preconditions.checkNotNull(riverName, "No riverName specified");
Preconditions.checkNotNull(riverIndexName, "No riverIndexName specified");
Preconditions.checkNotNull(settings, "No settings specified");
Builder builder = new Builder();
builder.riverName(riverName);
builder.riverIndexName(riverIndexName);
List<ServerAddress> mongoServers = new ArrayList<ServerAddress>();
String mongoHost;
int mongoPort;
if (settings.settings().containsKey(MongoDBRiver.TYPE)) {
Map<String, Object> mongoSettings =
(Map<String, Object>) settings.settings().get(MongoDBRiver.TYPE);
if (mongoSettings.containsKey(SERVERS_FIELD)) {
Object mongoServersSettings = mongoSettings.get(SERVERS_FIELD);
logger.trace("mongoServersSettings: " + mongoServersSettings);
boolean array = XContentMapValues.isArray(mongoServersSettings);
if (array) {
ArrayList<Map<String, Object>> feeds =
(ArrayList<Map<String, Object>>) mongoServersSettings;
for (Map<String, Object> feed : feeds) {
mongoHost = XContentMapValues.nodeStringValue(feed.get(HOST_FIELD), null);
mongoPort = XContentMapValues.nodeIntegerValue(feed.get(PORT_FIELD), DEFAULT_DB_PORT);
logger.trace("Server: " + mongoHost + " - " + mongoPort);
try {
mongoServers.add(new ServerAddress(mongoHost, mongoPort));
} catch (UnknownHostException uhEx) {
logger.warn("Cannot add mongo server {}:{}", uhEx, mongoHost, mongoPort);
}
}
}
} else {
mongoHost =
XContentMapValues.nodeStringValue(mongoSettings.get(HOST_FIELD), DEFAULT_DB_HOST);
mongoPort =
XContentMapValues.nodeIntegerValue(mongoSettings.get(PORT_FIELD), DEFAULT_DB_PORT);
try {
mongoServers.add(new ServerAddress(mongoHost, mongoPort));
} catch (UnknownHostException uhEx) {
logger.warn("Cannot add mongo server {}:{}", uhEx, mongoHost, mongoPort);
}
}
builder.mongoServers(mongoServers);
MongoClientOptions.Builder mongoClientOptionsBuilder =
MongoClientOptions.builder().socketKeepAlive(true);
// MongoDB options
if (mongoSettings.containsKey(OPTIONS_FIELD)) {
Map<String, Object> mongoOptionsSettings =
(Map<String, Object>) mongoSettings.get(OPTIONS_FIELD);
logger.trace("mongoOptionsSettings: " + mongoOptionsSettings);
builder.mongoSecondaryReadPreference(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SECONDARY_READ_PREFERENCE_FIELD), false));
builder.connectTimeout(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(CONNECT_TIMEOUT), DEFAULT_CONNECT_TIMEOUT));
builder.socketTimeout(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(SOCKET_TIMEOUT), DEFAULT_SOCKET_TIMEOUT));
builder.dropCollection(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(DROP_COLLECTION_FIELD), false));
String isMongos =
XContentMapValues.nodeStringValue(mongoOptionsSettings.get(IS_MONGOS_FIELD), null);
if (isMongos != null) {
builder.isMongos(Boolean.valueOf(isMongos));
}
builder.mongoUseSSL(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SSL_CONNECTION_FIELD), false));
builder.mongoSSLVerifyCertificate(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SSL_VERIFY_CERT_FIELD), true));
builder.advancedTransformation(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(ADVANCED_TRANSFORMATION_FIELD), false));
builder.skipInitialImport(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SKIP_INITIAL_IMPORT_FIELD), false));
builder.connectionsPerHost(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(CONNECTIONS_PER_HOST), DEFAULT_CONNECTIONS_PER_HOST));
builder.threadsAllowedToBlockForConnectionMultiplier(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER),
DEFAULT_THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER));
mongoClientOptionsBuilder
.connectTimeout(builder.connectTimeout)
.socketTimeout(builder.socketTimeout)
.connectionsPerHost(builder.connectionsPerHost)
.threadsAllowedToBlockForConnectionMultiplier(
builder.threadsAllowedToBlockForConnectionMultiplier);
if (builder.mongoSecondaryReadPreference) {
mongoClientOptionsBuilder.readPreference(ReadPreference.secondaryPreferred());
}
if (builder.mongoUseSSL) {
mongoClientOptionsBuilder.socketFactory(getSSLSocketFactory());
}
if (mongoOptionsSettings.containsKey(PARENT_TYPES_FIELD)) {
Set<String> parentTypes = new HashSet<String>();
Object parentTypesSettings = mongoOptionsSettings.get(PARENT_TYPES_FIELD);
logger.trace("parentTypesSettings: " + parentTypesSettings);
boolean array = XContentMapValues.isArray(parentTypesSettings);
if (array) {
ArrayList<String> fields = (ArrayList<String>) parentTypesSettings;
for (String field : fields) {
logger.trace("Field: " + field);
parentTypes.add(field);
}
}
builder.parentTypes(parentTypes);
}
if (mongoOptionsSettings.containsKey(STORE_STATISTICS_FIELD)) {
Object storeStatistics = mongoOptionsSettings.get(STORE_STATISTICS_FIELD);
boolean object = XContentMapValues.isObject(storeStatistics);
if (object) {
Map<String, Object> storeStatisticsSettings = (Map<String, Object>) storeStatistics;
builder.storeStatistics(true);
builder.statisticsIndexName(
XContentMapValues.nodeStringValue(
storeStatisticsSettings.get(INDEX_OBJECT), riverName + "-stats"));
builder.statisticsTypeName(
XContentMapValues.nodeStringValue(
storeStatisticsSettings.get(TYPE_FIELD), "stats"));
} else {
builder.storeStatistics(XContentMapValues.nodeBooleanValue(storeStatistics, false));
if (builder.storeStatistics) {
builder.statisticsIndexName(riverName + "-stats");
builder.statisticsTypeName("stats");
}
}
}
// builder.storeStatistics(XContentMapValues.nodeBooleanValue(mongoOptionsSettings.get(STORE_STATISTICS_FIELD),
// false));
builder.importAllCollections(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(IMPORT_ALL_COLLECTIONS_FIELD), false));
builder.disableIndexRefresh(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(DISABLE_INDEX_REFRESH_FIELD), false));
builder.includeCollection(
XContentMapValues.nodeStringValue(
mongoOptionsSettings.get(INCLUDE_COLLECTION_FIELD), ""));
if (mongoOptionsSettings.containsKey(INCLUDE_FIELDS_FIELD)) {
Set<String> includeFields = new HashSet<String>();
Object includeFieldsSettings = mongoOptionsSettings.get(INCLUDE_FIELDS_FIELD);
logger.trace("includeFieldsSettings: " + includeFieldsSettings);
boolean array = XContentMapValues.isArray(includeFieldsSettings);
if (array) {
ArrayList<String> fields = (ArrayList<String>) includeFieldsSettings;
for (String field : fields) {
logger.trace("Field: " + field);
includeFields.add(field);
}
}
if (!includeFields.contains(MongoDBRiver.MONGODB_ID_FIELD)) {
includeFields.add(MongoDBRiver.MONGODB_ID_FIELD);
}
builder.includeFields(includeFields);
} else if (mongoOptionsSettings.containsKey(EXCLUDE_FIELDS_FIELD)) {
Set<String> excludeFields = new HashSet<String>();
Object excludeFieldsSettings = mongoOptionsSettings.get(EXCLUDE_FIELDS_FIELD);
logger.trace("excludeFieldsSettings: " + excludeFieldsSettings);
boolean array = XContentMapValues.isArray(excludeFieldsSettings);
if (array) {
ArrayList<String> fields = (ArrayList<String>) excludeFieldsSettings;
for (String field : fields) {
logger.trace("Field: " + field);
excludeFields.add(field);
}
}
builder.excludeFields(excludeFields);
}
if (mongoOptionsSettings.containsKey(INITIAL_TIMESTAMP_FIELD)) {
BSONTimestamp timeStamp = null;
try {
Map<String, Object> initalTimestampSettings =
(Map<String, Object>) mongoOptionsSettings.get(INITIAL_TIMESTAMP_FIELD);
String scriptType = "js";
if (initalTimestampSettings.containsKey(INITIAL_TIMESTAMP_SCRIPT_TYPE_FIELD)) {
scriptType =
initalTimestampSettings.get(INITIAL_TIMESTAMP_SCRIPT_TYPE_FIELD).toString();
}
if (initalTimestampSettings.containsKey(INITIAL_TIMESTAMP_SCRIPT_FIELD)) {
ExecutableScript scriptExecutable =
scriptService.executable(
scriptType,
initalTimestampSettings.get(INITIAL_TIMESTAMP_SCRIPT_FIELD).toString(),
ScriptService.ScriptType.INLINE,
Maps.newHashMap());
Object ctx = scriptExecutable.run();
logger.trace("initialTimestamp script returned: {}", ctx);
if (ctx != null) {
long timestamp = Long.parseLong(ctx.toString());
timeStamp = new BSONTimestamp((int) (new Date(timestamp).getTime() / 1000), 1);
}
}
} catch (Throwable t) {
logger.error("Could not set initial timestamp", t);
} finally {
builder.initialTimestamp(timeStamp);
}
}
}
builder.mongoClientOptions(mongoClientOptionsBuilder.build());
// Credentials
if (mongoSettings.containsKey(CREDENTIALS_FIELD)) {
String dbCredential;
String mau = "";
String map = "";
String maad = "";
String mlu = "";
String mlp = "";
String mlad = "";
// String mdu = "";
// String mdp = "";
Object mongoCredentialsSettings = mongoSettings.get(CREDENTIALS_FIELD);
boolean array = XContentMapValues.isArray(mongoCredentialsSettings);
if (array) {
ArrayList<Map<String, Object>> credentials =
(ArrayList<Map<String, Object>>) mongoCredentialsSettings;
for (Map<String, Object> credential : credentials) {
dbCredential = XContentMapValues.nodeStringValue(credential.get(DB_FIELD), null);
if (ADMIN_DB_FIELD.equals(dbCredential)) {
mau = XContentMapValues.nodeStringValue(credential.get(USER_FIELD), null);
map = XContentMapValues.nodeStringValue(credential.get(PASSWORD_FIELD), null);
maad = XContentMapValues.nodeStringValue(credential.get(AUTH_FIELD), null);
} else if (LOCAL_DB_FIELD.equals(dbCredential)) {
mlu = XContentMapValues.nodeStringValue(credential.get(USER_FIELD), null);
mlp = XContentMapValues.nodeStringValue(credential.get(PASSWORD_FIELD), null);
mlad = XContentMapValues.nodeStringValue(credential.get(AUTH_FIELD), null);
// } else {
// mdu = XContentMapValues.nodeStringValue(
// credential.get(USER_FIELD), null);
// mdp = XContentMapValues.nodeStringValue(
// credential.get(PASSWORD_FIELD), null);
}
}
}
builder.mongoAdminUser(mau);
builder.mongoAdminPassword(map);
builder.mongoAdminAuthDatabase(maad);
builder.mongoLocalUser(mlu);
builder.mongoLocalPassword(mlp);
builder.mongoLocalAuthDatabase(mlad);
// mongoDbUser = mdu;
// mongoDbPassword = mdp;
}
builder.mongoDb(XContentMapValues.nodeStringValue(mongoSettings.get(DB_FIELD), riverName));
builder.mongoCollection(
XContentMapValues.nodeStringValue(mongoSettings.get(COLLECTION_FIELD), riverName));
builder.mongoGridFS(
XContentMapValues.nodeBooleanValue(mongoSettings.get(GRIDFS_FIELD), false));
if (mongoSettings.containsKey(FILTER_FIELD)) {
String filter = XContentMapValues.nodeStringValue(mongoSettings.get(FILTER_FIELD), "");
filter = removePrefix("o.", filter);
builder.mongoCollectionFilter(convertToBasicDBObject(filter));
// DBObject bsonObject = (DBObject) JSON.parse(filter);
// builder.mongoOplogFilter(convertToBasicDBObject(addPrefix("o.",
// filter)));
builder.mongoOplogFilter(convertToBasicDBObject(removePrefix("o.", filter)));
// } else {
// builder.mongoOplogFilter("");
}
if (mongoSettings.containsKey(SCRIPT_FIELD)) {
String scriptType = "js";
builder.script(mongoSettings.get(SCRIPT_FIELD).toString());
if (mongoSettings.containsKey("scriptType")) {
scriptType = mongoSettings.get("scriptType").toString();
} else if (mongoSettings.containsKey(SCRIPT_TYPE_FIELD)) {
scriptType = mongoSettings.get(SCRIPT_TYPE_FIELD).toString();
}
builder.scriptType(scriptType);
}
} else {
mongoHost = DEFAULT_DB_HOST;
mongoPort = DEFAULT_DB_PORT;
try {
mongoServers.add(new ServerAddress(mongoHost, mongoPort));
builder.mongoServers(mongoServers);
} catch (UnknownHostException e) {
e.printStackTrace();
}
builder.mongoDb(riverName);
builder.mongoCollection(riverName);
}
if (settings.settings().containsKey(INDEX_OBJECT)) {
Map<String, Object> indexSettings =
(Map<String, Object>) settings.settings().get(INDEX_OBJECT);
builder.indexName(
XContentMapValues.nodeStringValue(indexSettings.get(NAME_FIELD), builder.mongoDb));
builder.typeName(
XContentMapValues.nodeStringValue(indexSettings.get(TYPE_FIELD), builder.mongoDb));
Bulk.Builder bulkBuilder = new Bulk.Builder();
if (indexSettings.containsKey(BULK_FIELD)) {
Map<String, Object> bulkSettings = (Map<String, Object>) indexSettings.get(BULK_FIELD);
int bulkActions =
XContentMapValues.nodeIntegerValue(
bulkSettings.get(ACTIONS_FIELD), DEFAULT_BULK_ACTIONS);
bulkBuilder.bulkActions(bulkActions);
String size =
XContentMapValues.nodeStringValue(
bulkSettings.get(SIZE_FIELD), DEFAULT_BULK_SIZE.toString());
bulkBuilder.bulkSize(ByteSizeValue.parseBytesSizeValue(size));
bulkBuilder.concurrentRequests(
XContentMapValues.nodeIntegerValue(
bulkSettings.get(CONCURRENT_REQUESTS_FIELD),
EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY)));
bulkBuilder.flushInterval(
XContentMapValues.nodeTimeValue(
bulkSettings.get(FLUSH_INTERVAL_FIELD), DEFAULT_FLUSH_INTERVAL));
builder.throttleSize(
XContentMapValues.nodeIntegerValue(
indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5));
} else {
int bulkActions =
XContentMapValues.nodeIntegerValue(
indexSettings.get(BULK_SIZE_FIELD), DEFAULT_BULK_ACTIONS);
bulkBuilder.bulkActions(bulkActions);
bulkBuilder.bulkSize(DEFAULT_BULK_SIZE);
bulkBuilder.flushInterval(
XContentMapValues.nodeTimeValue(
indexSettings.get(BULK_TIMEOUT_FIELD), DEFAULT_FLUSH_INTERVAL));
bulkBuilder.concurrentRequests(
XContentMapValues.nodeIntegerValue(
indexSettings.get(CONCURRENT_BULK_REQUESTS_FIELD),
EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY)));
builder.throttleSize(
XContentMapValues.nodeIntegerValue(
indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5));
}
builder.bulk(bulkBuilder.build());
} else {
builder.indexName(builder.mongoDb);
builder.typeName(builder.mongoDb);
builder.bulk(new Bulk.Builder().build());
}
return builder.build();
}
@Override
public Mapper.Builder parse(
String fieldName, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
StringFieldMapper.Builder builder = new StringFieldMapper.Builder(fieldName);
// hack for the fact that string can't just accept true/false for
// the index property and still accepts no/not_analyzed/analyzed
final Object index = node.remove("index");
if (index != null) {
final String normalizedIndex = Strings.toUnderscoreCase(index.toString());
switch (normalizedIndex) {
case "analyzed":
builder.tokenized(true);
node.put("index", true);
break;
case "not_analyzed":
builder.tokenized(false);
node.put("index", true);
break;
case "no":
node.put("index", false);
break;
default:
throw new IllegalArgumentException(
"Can't parse [index] value ["
+ index
+ "] for field ["
+ fieldName
+ "], expected [true], [false], [no], [not_analyzed] or [analyzed]");
}
}
parseTextField(builder, fieldName, node, parserContext);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
iterator.hasNext(); ) {
Map.Entry<String, Object> entry = iterator.next();
String propName = Strings.toUnderscoreCase(entry.getKey());
Object propNode = entry.getValue();
if (propName.equals("null_value")) {
if (propNode == null) {
throw new MapperParsingException("Property [null_value] cannot be null.");
}
builder.nullValue(propNode.toString());
iterator.remove();
} else if (propName.equals("position_increment_gap")) {
int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1);
if (newPositionIncrementGap < 0) {
throw new MapperParsingException("positions_increment_gap less than 0 aren't allowed.");
}
builder.positionIncrementGap(newPositionIncrementGap);
// we need to update to actual analyzers if they are not set in this case...
// so we can inject the position increment gap...
if (builder.fieldType().indexAnalyzer() == null) {
builder
.fieldType()
.setIndexAnalyzer(parserContext.analysisService().defaultIndexAnalyzer());
}
if (builder.fieldType().searchAnalyzer() == null) {
builder
.fieldType()
.setSearchAnalyzer(parserContext.analysisService().defaultSearchAnalyzer());
}
if (builder.fieldType().searchQuoteAnalyzer() == null) {
builder
.fieldType()
.setSearchQuoteAnalyzer(
parserContext.analysisService().defaultSearchQuoteAnalyzer());
}
iterator.remove();
} else if (propName.equals("ignore_above")) {
builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
iterator.remove();
} else if (parseMultiField(builder, fieldName, parserContext, propName, propNode)) {
iterator.remove();
}
}
return builder;
}
@Override
public Mapper.Builder<?, ?> parse(
String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
Builder builder = geoPointField(name);
final boolean indexCreatedBeforeV2_0 =
parserContext.indexVersionCreated().before(Version.V_2_0_0);
parseField(builder, name, node, parserContext);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
iterator.hasNext(); ) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (fieldName.equals("path")
&& parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) {
builder.multiFieldPathType(parsePathType(name, fieldNode.toString()));
iterator.remove();
} else if (fieldName.equals("lat_lon")) {
builder.enableLatLon(XContentMapValues.nodeBooleanValue(fieldNode));
iterator.remove();
} else if (fieldName.equals("geohash")) {
builder.enableGeoHash(XContentMapValues.nodeBooleanValue(fieldNode));
iterator.remove();
} else if (fieldName.equals("geohash_prefix")) {
builder.geohashPrefix(XContentMapValues.nodeBooleanValue(fieldNode));
if (XContentMapValues.nodeBooleanValue(fieldNode)) {
builder.enableGeoHash(true);
}
iterator.remove();
} else if (fieldName.equals("precision_step")) {
builder.precisionStep(XContentMapValues.nodeIntegerValue(fieldNode));
iterator.remove();
} else if (fieldName.equals("geohash_precision")) {
if (fieldNode instanceof Integer) {
builder.geoHashPrecision(XContentMapValues.nodeIntegerValue(fieldNode));
} else {
builder.geoHashPrecision(GeoUtils.geoHashLevelsForPrecision(fieldNode.toString()));
}
iterator.remove();
} else if (fieldName.equals(Names.IGNORE_MALFORMED)) {
if (builder.fieldType().coerce == false) {
builder.fieldType().ignoreMalformed = XContentMapValues.nodeBooleanValue(fieldNode);
}
iterator.remove();
} else if (indexCreatedBeforeV2_0 && fieldName.equals("validate")) {
if (builder.fieldType().ignoreMalformed == false) {
builder.fieldType().ignoreMalformed = !XContentMapValues.nodeBooleanValue(fieldNode);
}
iterator.remove();
} else if (indexCreatedBeforeV2_0 && fieldName.equals("validate_lon")) {
if (builder.fieldType().ignoreMalformed() == false) {
builder.fieldType().ignoreMalformed = !XContentMapValues.nodeBooleanValue(fieldNode);
}
iterator.remove();
} else if (indexCreatedBeforeV2_0 && fieldName.equals("validate_lat")) {
if (builder.fieldType().ignoreMalformed == false) {
builder.fieldType().ignoreMalformed = !XContentMapValues.nodeBooleanValue(fieldNode);
}
iterator.remove();
} else if (fieldName.equals(Names.COERCE)) {
builder.fieldType().coerce = XContentMapValues.nodeBooleanValue(fieldNode);
if (builder.fieldType().coerce == true) {
builder.fieldType().ignoreMalformed = true;
}
iterator.remove();
} else if (indexCreatedBeforeV2_0 && fieldName.equals("normalize")) {
builder.fieldType().coerce = XContentMapValues.nodeBooleanValue(fieldNode);
iterator.remove();
} else if (indexCreatedBeforeV2_0 && fieldName.equals("normalize_lat")) {
builder.fieldType().coerce = XContentMapValues.nodeBooleanValue(fieldNode);
iterator.remove();
} else if (indexCreatedBeforeV2_0 && fieldName.equals("normalize_lon")) {
if (builder.fieldType().coerce == false) {
builder.fieldType().coerce = XContentMapValues.nodeBooleanValue(fieldNode);
}
iterator.remove();
} else if (parseMultiField(builder, name, parserContext, fieldName, fieldNode)) {
iterator.remove();
}
}
return builder;
}
@SuppressWarnings({"unchecked"})
@Inject
public AmazonsqsRiver(RiverName riverName, RiverSettings settings, Client client) {
super(riverName, settings);
this.client = client;
if (settings.settings().containsKey("amazonsqs")) {
Map<String, Object> sqsSettings = (Map<String, Object>) settings.settings().get("amazonsqs");
REGION = XContentMapValues.nodeStringValue(sqsSettings.get("region"), "null");
ENDPOINT = XContentMapValues.nodeStringValue(sqsSettings.get("endpoint"), "null");
ACCESS_KEY = XContentMapValues.nodeStringValue(sqsSettings.get("access_key"), "null");
SECRET_KEY = XContentMapValues.nodeStringValue(sqsSettings.get("secret_key"), "null");
QUEUE_URL = XContentMapValues.nodeStringValue(sqsSettings.get("queue_url"), "null");
SLEEP = XContentMapValues.nodeIntegerValue(sqsSettings.get("sleep"), DEFAULT_SLEEP);
LONGPOLLING_INTERVAL =
XContentMapValues.nodeIntegerValue(
sqsSettings.get("longpolling_interval"), DEFAULT_LONGPOLLING_INTERVAL);
DEBUG = XContentMapValues.nodeBooleanValue(sqsSettings.get("debug"), DEFAULT_DEBUG);
} else {
REGION = settings.globalSettings().get("cloud.aws.region");
ENDPOINT = settings.globalSettings().get("cloud.aws.endpoint");
ACCESS_KEY = settings.globalSettings().get("cloud.aws.access_key");
SECRET_KEY = settings.globalSettings().get("cloud.aws.secret_key");
QUEUE_URL = settings.globalSettings().get("cloud.aws.sqs.queue_url");
SLEEP = settings.globalSettings().getAsInt("cloud.aws.sqs.sleep", DEFAULT_SLEEP);
LONGPOLLING_INTERVAL =
settings
.globalSettings()
.getAsInt("cloud.aws.sqs.longpolling_interval", DEFAULT_LONGPOLLING_INTERVAL);
DEBUG = settings.globalSettings().getAsBoolean("cloud.aws.sqs.debug", DEFAULT_DEBUG);
}
if (settings.settings().containsKey("index")) {
Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index");
INDEX = XContentMapValues.nodeStringValue(indexSettings.get("index"), DEFAULT_INDEX);
MAX_MESSAGES =
XContentMapValues.nodeIntegerValue(
indexSettings.get("max_messages"), DEFAULT_MAX_MESSAGES);
} else {
INDEX = DEFAULT_INDEX;
MAX_MESSAGES = DEFAULT_MAX_MESSAGES;
}
if (ACCESS_KEY == null || ACCESS_KEY.trim().isEmpty() || ACCESS_KEY.equals("null")) {
sqs = new AmazonSQSAsyncClient();
} else {
sqs = new AmazonSQSAsyncClient(new BasicAWSCredentials(ACCESS_KEY, SECRET_KEY));
}
String endpoint = ENDPOINT;
if (ENDPOINT == null || ENDPOINT.trim().isEmpty() || ENDPOINT.equals("null")) {
endpoint = "https://sqs.".concat(REGION).concat(".amazonaws.com");
}
sqs.setEndpoint(endpoint);
if (DEBUG) {
if (ACCESS_KEY == null
|| ACCESS_KEY.length() < 2
|| SECRET_KEY == null
|| SECRET_KEY.length() < 2) {
logger.warn(
"AWS Credentials are not correct! "
.concat("Access Key: ")
.concat(ACCESS_KEY + " ")
.concat("Secret Key: ")
.concat(SECRET_KEY + " ")
.concat("Endpoint: ")
.concat(endpoint));
} else {
logger.info(
"AWS Credentials: "
.concat("Access Key length: ")
.concat(ACCESS_KEY.length() + " ")
.concat("Secret Key length: ")
.concat(SECRET_KEY.length() + " ")
.concat("Endpoint: ")
.concat(endpoint));
}
}
mapper = new ObjectMapper();
}