@Test
public void testIpMultiField() throws Exception {
assertAcked(
client()
.admin()
.indices()
.prepareCreate("my-index")
.addMapping("my-type", createMappingSource("ip")));
GetMappingsResponse getMappingsResponse =
client().admin().indices().prepareGetMappings("my-index").get();
MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type");
assertThat(mappingMetaData, not(nullValue()));
Map<String, Object> mappingSource = mappingMetaData.sourceAsMap();
Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource));
assertThat(aField.size(), equalTo(2));
assertThat(aField.get("type").toString(), equalTo("ip"));
assertThat(aField.get("fields"), notNullValue());
Map bField = ((Map) XContentMapValues.extractValue("properties.a.fields.b", mappingSource));
assertThat(bField.size(), equalTo(2));
assertThat(bField.get("type").toString(), equalTo("string"));
assertThat(bField.get("index").toString(), equalTo("not_analyzed"));
client()
.prepareIndex("my-index", "my-type", "1")
.setSource("a", "127.0.0.1")
.setRefresh(true)
.get();
CountResponse countResponse =
client().prepareCount("my-index").setQuery(matchQuery("a.b", "127.0.0.1")).get();
assertThat(countResponse.getCount(), equalTo(1l));
}
@Override
public HashMap<String, String> loadKey(String username) {
logger.info("loading password for username {}", username);
HashMap<String, String> ret = new HashMap<>();
String riverIndexName = getRiverIndexName();
refreshSearchIndex(riverIndexName);
GetResponse resp =
client.prepareGet(riverIndexName, riverName().name(), "_pwd").execute().actionGet();
if (resp.isExists()) {
if (logger.isDebugEnabled()) {
logger.debug("Password document: {}", resp.getSourceAsString());
}
Map<String, Object> newset = resp.getSource();
Set<String> keys = newset.keySet();
for (String s : keys) {
logger.info(
"Added key {} with a value of {}",
s,
XContentMapValues.nodeStringValue(newset.get(s), null));
ret.put(s, XContentMapValues.nodeStringValue(newset.get(s), null));
}
}
if (ret.isEmpty()) {
return null;
}
return ret;
}
@SuppressWarnings("unchecked")
public void testNestedFiltering() {
Map<String, Object> map = new HashMap<>();
map.put("field", "value");
map.put(
"array",
Arrays.asList(
1,
new HashMap<String, Object>() {
{
put("nested", 2);
put("nested_2", 3);
}
}));
Map<String, Object> falteredMap =
XContentMapValues.filter(map, new String[] {"array.nested"}, Strings.EMPTY_ARRAY);
assertThat(falteredMap.size(), equalTo(1));
// Selecting members of objects within arrays (ex. [ 1, { nested: "value"} ]) always returns
// all values in the array (1 in the ex)
// this is expected behavior as this types of objects are not supported in ES
assertThat((Integer) ((List) falteredMap.get("array")).get(0), equalTo(1));
assertThat(((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).size(), equalTo(1));
assertThat(
(Integer) ((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).get("nested"),
equalTo(2));
falteredMap = XContentMapValues.filter(map, new String[] {"array.*"}, Strings.EMPTY_ARRAY);
assertThat(falteredMap.size(), equalTo(1));
assertThat((Integer) ((List) falteredMap.get("array")).get(0), equalTo(1));
assertThat(((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).size(), equalTo(2));
map.clear();
map.put("field", "value");
map.put(
"obj",
new HashMap<String, Object>() {
{
put("field", "value");
put("field2", "value2");
}
});
falteredMap = XContentMapValues.filter(map, new String[] {"obj.field"}, Strings.EMPTY_ARRAY);
assertThat(falteredMap.size(), equalTo(1));
assertThat(((Map<String, Object>) falteredMap.get("obj")).size(), equalTo(1));
assertThat(
(String) ((Map<String, Object>) falteredMap.get("obj")).get("field"), equalTo("value"));
falteredMap = XContentMapValues.filter(map, new String[] {"obj.*"}, Strings.EMPTY_ARRAY);
assertThat(falteredMap.size(), equalTo(1));
assertThat(((Map<String, Object>) falteredMap.get("obj")).size(), equalTo(2));
assertThat(
(String) ((Map<String, Object>) falteredMap.get("obj")).get("field"), equalTo("value"));
assertThat(
(String) ((Map<String, Object>) falteredMap.get("obj")).get("field2"), equalTo("value2"));
}
public void testExtractRawValue() throws Exception {
XContentBuilder builder =
XContentFactory.jsonBuilder().startObject().field("test", "value").endObject();
Map<String, Object> map;
try (XContentParser parser =
XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
map = parser.map();
}
assertThat(XContentMapValues.extractRawValues("test", map).get(0).toString(), equalTo("value"));
builder = XContentFactory.jsonBuilder().startObject().field("test.me", "value").endObject();
try (XContentParser parser =
XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
map = parser.map();
}
assertThat(
XContentMapValues.extractRawValues("test.me", map).get(0).toString(), equalTo("value"));
builder =
XContentFactory.jsonBuilder()
.startObject()
.startObject("path1")
.startObject("path2")
.field("test", "value")
.endObject()
.endObject()
.endObject();
try (XContentParser parser =
XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
map = parser.map();
}
assertThat(
XContentMapValues.extractRawValues("path1.path2.test", map).get(0).toString(),
equalTo("value"));
builder =
XContentFactory.jsonBuilder()
.startObject()
.startObject("path1.xxx")
.startObject("path2.yyy")
.field("test", "value")
.endObject()
.endObject()
.endObject();
try (XContentParser parser =
XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
map = parser.map();
}
assertThat(
XContentMapValues.extractRawValues("path1.xxx.path2.yyy.test", map).get(0).toString(),
equalTo("value"));
}
public void testSupplementaryCharactersInPaths() {
Map<String, Object> map = new HashMap<>();
map.put("搜索", 2);
map.put("指数", 3);
assertEquals(
Collections.singletonMap("搜索", 2),
XContentMapValues.filter(map, new String[] {"搜索"}, new String[0]));
assertEquals(
Collections.singletonMap("指数", 3),
XContentMapValues.filter(map, new String[0], new String[] {"搜索"}));
}
@SuppressWarnings("unchecked")
public void testCompleteObjectFiltering() {
Map<String, Object> map = new HashMap<>();
map.put("field", "value");
map.put(
"obj",
new HashMap<String, Object>() {
{
put("field", "value");
put("field2", "value2");
}
});
map.put(
"array",
Arrays.asList(
1,
new HashMap<String, Object>() {
{
put("field", "value");
put("field2", "value2");
}
}));
Map<String, Object> filteredMap =
XContentMapValues.filter(map, new String[] {"obj"}, Strings.EMPTY_ARRAY);
assertThat(filteredMap.size(), equalTo(1));
assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(2));
assertThat(
((Map<String, Object>) filteredMap.get("obj")).get("field").toString(), equalTo("value"));
assertThat(
((Map<String, Object>) filteredMap.get("obj")).get("field2").toString(), equalTo("value2"));
filteredMap = XContentMapValues.filter(map, new String[] {"obj"}, new String[] {"*.field2"});
assertThat(filteredMap.size(), equalTo(1));
assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(1));
assertThat(
((Map<String, Object>) filteredMap.get("obj")).get("field").toString(), equalTo("value"));
filteredMap = XContentMapValues.filter(map, new String[] {"array"}, new String[] {});
assertThat(filteredMap.size(), equalTo(1));
assertThat(((List) filteredMap.get("array")).size(), equalTo(2));
assertThat((Integer) ((List) filteredMap.get("array")).get(0), equalTo(1));
assertThat(((Map<String, Object>) ((List) filteredMap.get("array")).get(1)).size(), equalTo(2));
filteredMap = XContentMapValues.filter(map, new String[] {"array"}, new String[] {"*.field2"});
assertThat(filteredMap.size(), equalTo(1));
assertThat(((List) filteredMap.get("array")).size(), equalTo(2));
assertThat((Integer) ((List) filteredMap.get("array")).get(0), equalTo(1));
assertThat(((Map<String, Object>) ((List) filteredMap.get("array")).get(1)).size(), equalTo(1));
assertThat(
((Map<String, Object>) ((List) filteredMap.get("array")).get(1)).get("field").toString(),
equalTo("value"));
}
public void testSharedPrefixes() {
Map<String, Object> map = new HashMap<>();
map.put("foobar", 2);
map.put("foobaz", 3);
assertEquals(
Collections.singletonMap("foobar", 2),
XContentMapValues.filter(map, new String[] {"foobar"}, new String[0]));
assertEquals(
Collections.singletonMap("foobaz", 3),
XContentMapValues.filter(map, new String[0], new String[] {"foobar"}));
}
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
StringFieldMapper.Builder builder = stringField(name);
parseField(builder, name, node, parserContext);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
iterator.hasNext(); ) {
Map.Entry<String, Object> entry = iterator.next();
String propName = Strings.toUnderscoreCase(entry.getKey());
Object propNode = entry.getValue();
if (propName.equals("null_value")) {
if (propNode == null) {
throw new MapperParsingException("Property [null_value] cannot be null.");
}
builder.nullValue(propNode.toString());
iterator.remove();
} else if (propName.equals("search_quote_analyzer")) {
NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
if (analyzer == null) {
throw new MapperParsingException(
"Analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
}
builder.searchQuotedAnalyzer(analyzer);
iterator.remove();
} else if (propName.equals("position_offset_gap")) {
builder.positionOffsetGap(XContentMapValues.nodeIntegerValue(propNode, -1));
// we need to update to actual analyzers if they are not set in this case...
// so we can inject the position offset gap...
if (builder.indexAnalyzer == null) {
builder.indexAnalyzer = parserContext.analysisService().defaultIndexAnalyzer();
}
if (builder.searchAnalyzer == null) {
builder.searchAnalyzer = parserContext.analysisService().defaultSearchAnalyzer();
}
if (builder.searchQuotedAnalyzer == null) {
builder.searchQuotedAnalyzer =
parserContext.analysisService().defaultSearchQuoteAnalyzer();
}
iterator.remove();
} else if (propName.equals("ignore_above")) {
builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
iterator.remove();
} else if (parseMultiField(builder, name, parserContext, propName, propNode)) {
iterator.remove();
}
}
return builder;
}
/** The template source definition. */
public PutIndexTemplateRequest source(Map templateSource) {
Map<String, Object> source = templateSource;
if (source.containsKey("template")) {
template(source.get("template").toString());
}
if (source.containsKey("order")) {
order(XContentMapValues.nodeIntegerValue(source.get("order"), order()));
}
if (source.containsKey("settings")) {
if (!(source.get("settings") instanceof Map)) {
throw new ElasticSearchIllegalArgumentException(
"Malformed settings section, should include an inner object");
}
settings((Map<String, Object>) source.get("settings"));
}
if (source.containsKey("mappings")) {
Map<String, Object> mappings = (Map<String, Object>) source.get("mappings");
for (Map.Entry<String, Object> entry : mappings.entrySet()) {
if (!(entry.getValue() instanceof Map)) {
throw new ElasticSearchIllegalArgumentException(
"Malformed mappings section for type ["
+ entry.getKey()
+ "], should include an inner object describing the mapping");
}
mapping(entry.getKey(), (Map<String, Object>) entry.getValue());
}
}
return this;
}
public void testFilterWithEmptyIncludesExcludes() {
Map<String, Object> map = new HashMap<>();
map.put("field", "value");
Map<String, Object> filteredMap =
XContentMapValues.filter(map, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY);
assertThat(filteredMap.size(), equalTo(1));
assertThat(filteredMap.get("field").toString(), equalTo("value"));
}
@SuppressWarnings("unchecked")
public void testFilterIncludesUsingStarPrefix() {
Map<String, Object> map = new HashMap<>();
map.put("field", "value");
map.put(
"obj",
new HashMap<String, Object>() {
{
put("field", "value");
put("field2", "value2");
}
});
map.put(
"n_obj",
new HashMap<String, Object>() {
{
put("n_field", "value");
put("n_field2", "value2");
}
});
Map<String, Object> filteredMap =
XContentMapValues.filter(map, new String[] {"*.field2"}, Strings.EMPTY_ARRAY);
assertThat(filteredMap.size(), equalTo(1));
assertThat(filteredMap, hasKey("obj"));
assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(1));
assertThat(((Map<String, Object>) filteredMap.get("obj")), hasKey("field2"));
// only objects
filteredMap = XContentMapValues.filter(map, new String[] {"*.*"}, Strings.EMPTY_ARRAY);
assertThat(filteredMap.size(), equalTo(2));
assertThat(filteredMap, hasKey("obj"));
assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(2));
assertThat(filteredMap, hasKey("n_obj"));
assertThat(((Map<String, Object>) filteredMap.get("n_obj")).size(), equalTo(2));
filteredMap = XContentMapValues.filter(map, new String[] {"*"}, new String[] {"*.*2"});
assertThat(filteredMap.size(), equalTo(3));
assertThat(filteredMap, hasKey("field"));
assertThat(filteredMap, hasKey("obj"));
assertThat(((Map) filteredMap.get("obj")).size(), equalTo(1));
assertThat(((Map<String, Object>) filteredMap.get("obj")), hasKey("field"));
assertThat(filteredMap, hasKey("n_obj"));
assertThat(((Map<String, Object>) filteredMap.get("n_obj")).size(), equalTo(1));
assertThat(((Map<String, Object>) filteredMap.get("n_obj")), hasKey("n_field"));
}
@SuppressWarnings({"unchecked"})
@Inject
public SimpleRiver(
RiverName riverName, RiverSettings settings, Client client, ThreadPool threadPool) {
super(riverName, settings);
this.client = client;
if (settings.settings().containsKey("simple")) {
Map<String, Object> simpleSettings = (Map<String, Object>) settings.settings().get("simple");
simpleNumber = XContentMapValues.nodeIntegerValue(simpleSettings.get("number"), 100);
fieldName = XContentMapValues.nodeStringValue(simpleSettings.get("field"), "test");
poll =
XContentMapValues.nodeTimeValue(
simpleSettings.get("poll"), TimeValue.timeValueMinutes(60));
}
logger.info(
"creating simple stream river for [{} numbers] with field [{}]", simpleNumber, fieldName);
if (settings.settings().containsKey("index")) {
Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index");
indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name());
typeName = XContentMapValues.nodeStringValue(indexSettings.get("type"), "simple_type");
bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size"), 100);
bulkThreshold = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_threshold"), 10);
} else {
indexName = riverName.name();
typeName = "simple_type";
bulkSize = 100;
bulkThreshold = 10;
}
}
public void testPrefixedNamesFilteringTest() {
Map<String, Object> map = new HashMap<>();
map.put("obj", "value");
map.put("obj_name", "value_name");
Map<String, Object> filterdMap =
XContentMapValues.filter(map, new String[] {"obj_name"}, Strings.EMPTY_ARRAY);
assertThat(filterdMap.size(), equalTo(1));
assertThat((String) filterdMap.get("obj_name"), equalTo("value_name"));
}
public void testThatFilterIncludesEmptyObjectWhenUsingIncludes() throws Exception {
XContentBuilder builder =
XContentFactory.jsonBuilder().startObject().startObject("obj").endObject().endObject();
Tuple<XContentType, Map<String, Object>> mapTuple =
XContentHelper.convertToMap(builder.bytes(), true);
Map<String, Object> filteredSource =
XContentMapValues.filter(mapTuple.v2(), new String[] {"obj"}, Strings.EMPTY_ARRAY);
assertThat(mapTuple.v2(), equalTo(filteredSource));
}
public void testDotsInFieldNames() {
Map<String, Object> map = new HashMap<>();
map.put("foo.bar", 2);
Map<String, Object> sub = new HashMap<>();
sub.put("baz", 3);
map.put("foo", sub);
map.put("quux", 5);
// dots in field names in includes
Map<String, Object> filtered =
XContentMapValues.filter(map, new String[] {"foo"}, new String[0]);
Map<String, Object> expected = new HashMap<>(map);
expected.remove("quux");
assertEquals(expected, filtered);
// dots in field names in excludes
filtered = XContentMapValues.filter(map, new String[0], new String[] {"foo"});
expected = new HashMap<>(map);
expected.keySet().retainAll(Collections.singleton("quux"));
assertEquals(expected, filtered);
}
@SuppressWarnings({"unchecked"})
public void testNotOmittingObjectWithNestedExcludedObject() throws Exception {
XContentBuilder builder =
XContentFactory.jsonBuilder()
.startObject()
.startObject("obj1")
.startObject("obj2")
.startObject("obj3")
.endObject()
.endObject()
.endObject()
.endObject();
// implicit include
Tuple<XContentType, Map<String, Object>> mapTuple =
XContentHelper.convertToMap(builder.bytes(), true);
Map<String, Object> filteredSource =
XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[] {"*.obj2"});
assertThat(filteredSource.size(), equalTo(1));
assertThat(filteredSource, hasKey("obj1"));
assertThat(((Map) filteredSource.get("obj1")).size(), Matchers.equalTo(0));
// explicit include
filteredSource =
XContentMapValues.filter(mapTuple.v2(), new String[] {"obj1"}, new String[] {"*.obj2"});
assertThat(filteredSource.size(), equalTo(1));
assertThat(filteredSource, hasKey("obj1"));
assertThat(((Map) filteredSource.get("obj1")).size(), Matchers.equalTo(0));
// wild card include
filteredSource =
XContentMapValues.filter(mapTuple.v2(), new String[] {"*.obj2"}, new String[] {"*.obj3"});
assertThat(filteredSource.size(), equalTo(1));
assertThat(filteredSource, hasKey("obj1"));
assertThat(((Map<String, Object>) filteredSource.get("obj1")), hasKey("obj2"));
assertThat(((Map) ((Map) filteredSource.get("obj1")).get("obj2")).size(), Matchers.equalTo(0));
}
@Nullable
@Override
public Object referenceValue(Reference reference) {
if (updatedColumnValues == null) {
return super.referenceValue(reference);
}
Object value = updatedColumnValues.get(reference.ident().columnIdent().fqn());
if (value == null && !reference.ident().isColumn()) {
value =
XContentMapValues.extractValue(
reference.ident().columnIdent().fqn(), updatedColumnValues);
}
return reference.valueType().value(value);
}
@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
Builder builder = geoShapeField(name);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
iterator.hasNext(); ) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Object fieldNode = entry.getValue();
if (Names.TREE.equals(fieldName)) {
builder.fieldType().setTree(fieldNode.toString());
iterator.remove();
} else if (Names.TREE_LEVELS.equals(fieldName)) {
builder.fieldType().setTreeLevels(Integer.parseInt(fieldNode.toString()));
iterator.remove();
} else if (Names.TREE_PRESISION.equals(fieldName)) {
builder
.fieldType()
.setPrecisionInMeters(
DistanceUnit.parse(
fieldNode.toString(), DistanceUnit.DEFAULT, DistanceUnit.DEFAULT));
iterator.remove();
} else if (Names.DISTANCE_ERROR_PCT.equals(fieldName)) {
builder.fieldType().setDistanceErrorPct(Double.parseDouble(fieldNode.toString()));
iterator.remove();
} else if (Names.ORIENTATION.equals(fieldName)) {
builder
.fieldType()
.setOrientation(ShapeBuilder.orientationFromString(fieldNode.toString()));
iterator.remove();
} else if (Names.STRATEGY.equals(fieldName)) {
builder.fieldType().setStrategyName(fieldNode.toString());
iterator.remove();
} else if (Names.COERCE.equals(fieldName)) {
builder.coerce(nodeBooleanValue(fieldNode));
iterator.remove();
} else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName)
&& builder.fieldType().strategyName.equals(SpatialStrategy.TERM.getStrategyName())
== false) {
builder.fieldType().setPointsOnly(XContentMapValues.nodeBooleanValue(fieldNode));
iterator.remove();
}
}
return builder;
}
public void testNotOmittingObjectsWithExcludedProperties() throws Exception {
XContentBuilder builder =
XContentFactory.jsonBuilder()
.startObject()
.startObject("obj")
.field("f1", "v1")
.endObject()
.endObject();
Tuple<XContentType, Map<String, Object>> mapTuple =
XContentHelper.convertToMap(builder.bytes(), true);
Map<String, Object> filteredSource =
XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[] {"obj.f1"});
assertThat(filteredSource.size(), equalTo(1));
assertThat(filteredSource, hasKey("obj"));
assertThat(((Map) filteredSource.get("obj")).size(), equalTo(0));
}
/** The template source definition. */
public PutIndexTemplateRequest source(Map templateSource) {
Map<String, Object> source = templateSource;
for (Map.Entry<String, Object> entry : source.entrySet()) {
String name = entry.getKey();
if (name.equals("template")) {
template(entry.getValue().toString());
} else if (name.equals("order")) {
order(XContentMapValues.nodeIntegerValue(entry.getValue(), order()));
} else if (name.equals("settings")) {
if (!(entry.getValue() instanceof Map)) {
throw new ElasticSearchIllegalArgumentException(
"Malformed settings section, should include an inner object");
}
settings((Map<String, Object>) entry.getValue());
} else if (name.equals("mappings")) {
Map<String, Object> mappings = (Map<String, Object>) entry.getValue();
for (Map.Entry<String, Object> entry1 : mappings.entrySet()) {
if (!(entry1.getValue() instanceof Map)) {
throw new ElasticSearchIllegalArgumentException(
"Malformed mappings section for type ["
+ entry1.getKey()
+ "], should include an inner object describing the mapping");
}
mapping(entry1.getKey(), (Map<String, Object>) entry1.getValue());
}
} else {
// maybe custom?
IndexMetaData.Custom.Factory factory = IndexMetaData.lookupFactory(name);
if (factory != null) {
try {
customs.put(name, factory.fromMap((Map<String, Object>) entry.getValue()));
} catch (IOException e) {
throw new ElasticSearchParseException(
"failed to parse custom metadata for [" + name + "]");
}
}
}
}
return this;
}
@Override
public Mapper.Builder parse(
String fieldName, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
TextFieldMapper.Builder builder = new TextFieldMapper.Builder(fieldName);
builder.fieldType().setIndexAnalyzer(parserContext.analysisService().defaultIndexAnalyzer());
builder
.fieldType()
.setSearchAnalyzer(parserContext.analysisService().defaultSearchAnalyzer());
builder
.fieldType()
.setSearchQuoteAnalyzer(parserContext.analysisService().defaultSearchQuoteAnalyzer());
parseTextField(builder, fieldName, node, parserContext);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
iterator.hasNext(); ) {
Map.Entry<String, Object> entry = iterator.next();
String propName = entry.getKey();
Object propNode = entry.getValue();
if (propName.equals("position_increment_gap")) {
int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1);
builder.positionIncrementGap(newPositionIncrementGap);
iterator.remove();
} else if (propName.equals("fielddata")) {
builder.fielddata(XContentMapValues.nodeBooleanValue(propNode));
iterator.remove();
} else if (propName.equals("eager_global_ordinals")) {
builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode));
iterator.remove();
} else if (propName.equals("fielddata_frequency_filter")) {
Map<?, ?> frequencyFilter = (Map<?, ?>) propNode;
double minFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("min"), 0);
double maxFrequency =
XContentMapValues.nodeDoubleValue(frequencyFilter.remove("max"), Integer.MAX_VALUE);
int minSegmentSize =
XContentMapValues.nodeIntegerValue(frequencyFilter.remove("min_segment_size"), 0);
builder.fielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize);
DocumentMapperParser.checkNoRemainingFields(
propName, frequencyFilter, parserContext.indexVersionCreated());
iterator.remove();
}
}
return builder;
}
/**
* Configure the river.
*
* @param settings used for configuration.
*/
@SuppressWarnings({"unchecked"})
protected void configure(Map<String, Object> settings) {
if (!closed) throw new IllegalStateException("Remote River must be stopped to configure it!");
if (settings.containsKey("remote")) {
Map<String, Object> remoteSettings = (Map<String, Object>) settings.get("remote");
maxIndexingThreads =
XContentMapValues.nodeIntegerValue(remoteSettings.get("maxIndexingThreads"), 1);
SpaceIndexingMode sim =
SpaceIndexingMode.parseConfiguration((String) remoteSettings.get("listDocumentsMode"));
if (sim != null) spaceIndexingMode = sim;
else if (XContentMapValues.nodeBooleanValue(remoteSettings.get("simpleGetDocuments"), false))
spaceIndexingMode = SpaceIndexingMode.SIMPLE;
indexFullUpdatePeriod =
Utils.parseTimeValue(remoteSettings, "indexFullUpdatePeriod", 12, TimeUnit.HOURS);
String ifuce = Utils.trimToNull((String) remoteSettings.get("indexFullUpdateCronExpression"));
if (ifuce != null) {
try {
this.indexFullUpdateCronExpression = new CronExpression(ifuce);
} catch (ParseException e) {
throw new SettingsException(
"Cron expression in indexFullUpdateCronExpression is invalid: " + e.getMessage());
}
}
if (spaceIndexingMode.isIncrementalUpdateSupported()
|| (indexFullUpdatePeriod < 1 && indexFullUpdateCronExpression == null))
indexUpdatePeriod =
Utils.parseTimeValue(remoteSettings, "indexUpdatePeriod", 5, TimeUnit.MINUTES);
else indexUpdatePeriod = 0;
if (remoteSettings.containsKey("spacesIndexed")) {
allIndexedSpacesKeys =
Utils.parseCsvString(
XContentMapValues.nodeStringValue(remoteSettings.get("spacesIndexed"), null));
if (allIndexedSpacesKeys != null) {
// stop spaces loading from remote system
allIndexedSpacesKeysNextRefresh = Long.MAX_VALUE;
}
}
if (remoteSettings.containsKey("spaceKeysExcluded")) {
spaceKeysExcluded =
Utils.parseCsvString(
XContentMapValues.nodeStringValue(remoteSettings.get("spaceKeysExcluded"), null));
}
String remoteClientClass =
Utils.trimToNull(
XContentMapValues.nodeStringValue(remoteSettings.get("remoteClientClass"), null));
if (remoteClientClass != null) {
try {
remoteSystemClient = (IRemoteSystemClient) Class.forName(remoteClientClass).newInstance();
} catch (Exception e) {
throw new SettingsException(
"Unable to instantiate class defined by 'remote/remoteClientClass': "
+ e.getMessage());
}
} else {
remoteSystemClient = new GetJSONClient();
}
remoteSystemClient.init(
this, remoteSettings, allIndexedSpacesKeysNextRefresh != Long.MAX_VALUE, this);
} else {
throw new SettingsException("'remote' element of river configuration structure not found");
}
Map<String, Object> indexSettings = null;
if (settings.containsKey("index")) {
indexSettings = (Map<String, Object>) settings.get("index");
indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name());
typeName =
XContentMapValues.nodeStringValue(
indexSettings.get("type"), INDEX_DOCUMENT_TYPE_NAME_DEFAULT);
} else {
indexName = riverName.name();
typeName = INDEX_DOCUMENT_TYPE_NAME_DEFAULT;
}
Map<String, Object> activityLogSettings = null;
if (settings.containsKey("activity_log")) {
activityLogSettings = (Map<String, Object>) settings.get("activity_log");
activityLogIndexName =
Utils.trimToNull(
XContentMapValues.nodeStringValue(activityLogSettings.get("index"), null));
if (activityLogIndexName == null) {
throw new SettingsException(
"'activity_log/index' element of river configuration structure must be defined with some string");
}
activityLogTypeName =
Utils.trimToNull(
XContentMapValues.nodeStringValue(
activityLogSettings.get("type"), INDEX_ACTIVITY_TYPE_NAME_DEFAULT));
}
documentIndexStructureBuilder =
new DocumentWithCommentsIndexStructureBuilder(
this, indexName, typeName, indexSettings, spaceIndexingMode.isUpdateDateMandatory());
preparePreprocessors(indexSettings, documentIndexStructureBuilder);
remoteSystemClient.setIndexStructureBuilder(documentIndexStructureBuilder);
logger.info(
"Configured Remote River '{}'. Search index name '{}', document type for issues '{}'. Indexing mode '{}'.",
riverName.getName(),
indexName,
typeName,
spaceIndexingMode.getConfigValue());
if (activityLogIndexName != null) {
logger.info(
"Activity log for Remote River '{}' is enabled. Search index name '{}', document type for index updates '{}'.",
riverName.getName(),
activityLogIndexName,
activityLogTypeName);
}
}
public Object extractValue(String path) {
return XContentMapValues.extractValue(path, loadSourceIfNeeded());
}
public Object filter(String[] includes, String[] excludes) {
return XContentMapValues.filter(loadSourceIfNeeded(), includes, excludes);
}
/**
* Returns the values associated with the path. Those are "low" level values, and it can handle
* path expression where an array/list is navigated within.
*/
public List<Object> extractRawValues(String path) {
return XContentMapValues.extractRawValues(path, loadSourceIfNeeded());
}
private InternalSearchHit createNestedSearchHit(
SearchContext context,
int nestedTopDocId,
int nestedSubDocId,
int rootSubDocId,
List<String> extractFieldNames,
boolean loadAllStored,
Set<String> fieldNames,
LeafReaderContext subReaderContext)
throws IOException {
// Also if highlighting is requested on nested documents we need to fetch the _source from the
// root document,
// otherwise highlighting will attempt to fetch the _source from the nested doc, which will
// fail,
// because the entire _source is only stored with the root document.
final FieldsVisitor rootFieldsVisitor =
new FieldsVisitor(
context.sourceRequested() || extractFieldNames != null || context.highlight() != null);
loadStoredFields(context, subReaderContext, rootFieldsVisitor, rootSubDocId);
rootFieldsVisitor.postProcess(context.mapperService());
Map<String, SearchHitField> searchFields =
getSearchFields(context, nestedSubDocId, loadAllStored, fieldNames, subReaderContext);
DocumentMapper documentMapper =
context.mapperService().documentMapper(rootFieldsVisitor.uid().type());
SourceLookup sourceLookup = context.lookup().source();
sourceLookup.setSegmentAndDocument(subReaderContext, nestedSubDocId);
ObjectMapper nestedObjectMapper =
documentMapper.findNestedObjectMapper(nestedSubDocId, context, subReaderContext);
assert nestedObjectMapper != null;
InternalSearchHit.InternalNestedIdentity nestedIdentity =
getInternalNestedIdentity(
context, nestedSubDocId, subReaderContext, documentMapper, nestedObjectMapper);
BytesReference source = rootFieldsVisitor.source();
if (source != null) {
Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(source, true);
Map<String, Object> sourceAsMap = tuple.v2();
List<Map<String, Object>> nestedParsedSource;
SearchHit.NestedIdentity nested = nestedIdentity;
do {
Object extractedValue =
XContentMapValues.extractValue(nested.getField().string(), sourceAsMap);
if (extractedValue == null) {
// The nested objects may not exist in the _source, because it was filtered because of
// _source filtering
break;
} else if (extractedValue instanceof List) {
// nested field has an array value in the _source
nestedParsedSource = (List<Map<String, Object>>) extractedValue;
} else if (extractedValue instanceof Map) {
// nested field has an object value in the _source. This just means the nested field has
// just one inner object, which is valid, but uncommon.
nestedParsedSource = ImmutableList.of((Map<String, Object>) extractedValue);
} else {
throw new IllegalStateException("extracted source isn't an object or an array");
}
sourceAsMap = nestedParsedSource.get(nested.getOffset());
nested = nested.getChild();
} while (nested != null);
context.lookup().source().setSource(sourceAsMap);
XContentType contentType = tuple.v1();
BytesReference nestedSource = contentBuilder(contentType).map(sourceAsMap).bytes();
context.lookup().source().setSource(nestedSource);
context.lookup().source().setSourceContentType(contentType);
}
InternalSearchHit searchHit =
new InternalSearchHit(
nestedTopDocId,
rootFieldsVisitor.uid().id(),
documentMapper.typeText(),
nestedIdentity,
searchFields);
if (extractFieldNames != null) {
for (String extractFieldName : extractFieldNames) {
List<Object> values = context.lookup().source().extractRawValues(extractFieldName);
if (!values.isEmpty()) {
if (searchHit.fieldsOrNull() == null) {
searchHit.fields(new HashMap<String, SearchHitField>(2));
}
SearchHitField hitField = searchHit.fields().get(extractFieldName);
if (hitField == null) {
hitField = new InternalSearchHitField(extractFieldName, new ArrayList<>(2));
searchHit.fields().put(extractFieldName, hitField);
}
for (Object value : values) {
hitField.values().add(value);
}
}
}
}
return searchHit;
}
@SuppressWarnings({"unchecked"})
@Inject
public RabbitmqRiver(
RiverName riverName, RiverSettings settings, Client client, ScriptService scriptService) {
super(riverName, settings);
this.client = client;
if (settings.settings().containsKey("rabbitmq")) {
Map<String, Object> rabbitSettings =
(Map<String, Object>) settings.settings().get("rabbitmq");
if (rabbitSettings.containsKey("addresses")) {
List<Address> addresses = new ArrayList<Address>();
for (Map<String, Object> address :
(List<Map<String, Object>>) rabbitSettings.get("addresses")) {
addresses.add(
new Address(
XContentMapValues.nodeStringValue(address.get("host"), "localhost"),
XContentMapValues.nodeIntegerValue(address.get("port"), AMQP.PROTOCOL.PORT)));
}
rabbitAddresses = addresses.toArray(new Address[addresses.size()]);
} else {
String rabbitHost =
XContentMapValues.nodeStringValue(rabbitSettings.get("host"), "localhost");
int rabbitPort =
XContentMapValues.nodeIntegerValue(rabbitSettings.get("port"), AMQP.PROTOCOL.PORT);
rabbitAddresses = new Address[] {new Address(rabbitHost, rabbitPort)};
}
rabbitUser = XContentMapValues.nodeStringValue(rabbitSettings.get("user"), "guest");
rabbitPassword = XContentMapValues.nodeStringValue(rabbitSettings.get("pass"), "guest");
rabbitVhost = XContentMapValues.nodeStringValue(rabbitSettings.get("vhost"), "/");
rabbitQueue = XContentMapValues.nodeStringValue(rabbitSettings.get("queue"), "elasticsearch");
rabbitExchange =
XContentMapValues.nodeStringValue(rabbitSettings.get("exchange"), "elasticsearch");
rabbitRoutingKey =
XContentMapValues.nodeStringValue(rabbitSettings.get("routing_key"), "elasticsearch");
rabbitExchangeDeclare =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("exchange_declare"), true);
if (rabbitExchangeDeclare) {
rabbitExchangeType =
XContentMapValues.nodeStringValue(rabbitSettings.get("exchange_type"), "direct");
rabbitExchangeDurable =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("exchange_durable"), true);
} else {
rabbitExchangeType = "direct";
rabbitExchangeDurable = true;
}
rabbitQueueDeclare =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_declare"), true);
if (rabbitQueueDeclare) {
rabbitQueueDurable =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_durable"), true);
rabbitQueueAutoDelete =
XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_auto_delete"), false);
if (rabbitSettings.containsKey("args")) {
rabbitQueueArgs = (Map<String, Object>) rabbitSettings.get("args");
}
} else {
rabbitQueueDurable = true;
rabbitQueueAutoDelete = false;
}
rabbitQueueBind = XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_bind"), true);
rabbitQosPrefetchSize =
XContentMapValues.nodeIntegerValue(rabbitSettings.get("qos_prefetch_size"), 0);
rabbitQosPrefetchCount =
XContentMapValues.nodeIntegerValue(rabbitSettings.get("qos_prefetch_count"), 10);
rabbitHeartbeat =
TimeValue.parseTimeValue(
XContentMapValues.nodeStringValue(rabbitSettings.get("heartbeat"), "30m"),
TimeValue.timeValueMinutes(30));
} else {
rabbitAddresses = new Address[] {new Address("localhost", AMQP.PROTOCOL.PORT)};
rabbitUser = "******";
rabbitPassword = "******";
rabbitVhost = "/";
rabbitQueue = "elasticsearch";
rabbitQueueAutoDelete = false;
rabbitQueueDurable = true;
rabbitExchange = "elasticsearch";
rabbitExchangeType = "direct";
rabbitExchangeDurable = true;
rabbitRoutingKey = "elasticsearch";
rabbitExchangeDeclare = true;
rabbitQueueDeclare = true;
rabbitQueueBind = true;
rabbitQosPrefetchSize = 0;
rabbitQosPrefetchCount = 10;
rabbitHeartbeat = TimeValue.timeValueMinutes(30);
}
if (settings.settings().containsKey("index")) {
Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index");
bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size"), 100);
if (indexSettings.containsKey("bulk_timeout")) {
bulkTimeout =
TimeValue.parseTimeValue(
XContentMapValues.nodeStringValue(indexSettings.get("bulk_timeout"), "10ms"),
TimeValue.timeValueMillis(10));
} else {
bulkTimeout = TimeValue.timeValueMillis(10);
}
ordered = XContentMapValues.nodeBooleanValue(indexSettings.get("ordered"), false);
} else {
bulkSize = 100;
bulkTimeout = TimeValue.timeValueMillis(10);
ordered = false;
}
if (settings.settings().containsKey("bulk_script_filter")) {
Map<String, Object> scriptSettings =
(Map<String, Object>) settings.settings().get("bulk_script_filter");
if (scriptSettings.containsKey("script")) {
String scriptLang = "native";
if (scriptSettings.containsKey("script_lang")) {
scriptLang = scriptSettings.get("script_lang").toString();
}
Map<String, Object> scriptParams = null;
if (scriptSettings.containsKey("script_params")) {
scriptParams = (Map<String, Object>) scriptSettings.get("script_params");
} else {
scriptParams = Maps.newHashMap();
}
bulkScript =
scriptService.executable(
scriptLang, scriptSettings.get("script").toString(), scriptParams);
} else {
bulkScript = null;
}
} else {
bulkScript = null;
}
if (settings.settings().containsKey("script_filter")) {
Map<String, Object> scriptSettings =
(Map<String, Object>) settings.settings().get("script_filter");
if (scriptSettings.containsKey("script")) {
String scriptLang = "mvel";
if (scriptSettings.containsKey("script_lang")) {
scriptLang = scriptSettings.get("script_lang").toString();
}
Map<String, Object> scriptParams = null;
if (scriptSettings.containsKey("script_params")) {
scriptParams = (Map<String, Object>) scriptSettings.get("script_params");
} else {
scriptParams = Maps.newHashMap();
}
script =
scriptService.executable(
scriptLang, scriptSettings.get("script").toString(), scriptParams);
} else {
script = null;
}
} else {
script = null;
}
}
@SuppressWarnings("unchecked")
public static synchronized MongoDBRiverDefinition parseSettings(
String riverName,
String riverIndexName,
RiverSettings settings,
ScriptService scriptService) {
logger.trace("Parse river settings for {}", riverName);
Preconditions.checkNotNull(riverName, "No riverName specified");
Preconditions.checkNotNull(riverIndexName, "No riverIndexName specified");
Preconditions.checkNotNull(settings, "No settings specified");
Builder builder = new Builder();
builder.riverName(riverName);
builder.riverIndexName(riverIndexName);
List<ServerAddress> mongoServers = new ArrayList<ServerAddress>();
String mongoHost;
int mongoPort;
if (settings.settings().containsKey(MongoDBRiver.TYPE)) {
Map<String, Object> mongoSettings =
(Map<String, Object>) settings.settings().get(MongoDBRiver.TYPE);
if (mongoSettings.containsKey(SERVERS_FIELD)) {
Object mongoServersSettings = mongoSettings.get(SERVERS_FIELD);
logger.trace("mongoServersSettings: " + mongoServersSettings);
boolean array = XContentMapValues.isArray(mongoServersSettings);
if (array) {
ArrayList<Map<String, Object>> feeds =
(ArrayList<Map<String, Object>>) mongoServersSettings;
for (Map<String, Object> feed : feeds) {
mongoHost = XContentMapValues.nodeStringValue(feed.get(HOST_FIELD), null);
mongoPort = XContentMapValues.nodeIntegerValue(feed.get(PORT_FIELD), DEFAULT_DB_PORT);
logger.trace("Server: " + mongoHost + " - " + mongoPort);
try {
mongoServers.add(new ServerAddress(mongoHost, mongoPort));
} catch (UnknownHostException uhEx) {
logger.warn("Cannot add mongo server {}:{}", uhEx, mongoHost, mongoPort);
}
}
}
} else {
mongoHost =
XContentMapValues.nodeStringValue(mongoSettings.get(HOST_FIELD), DEFAULT_DB_HOST);
mongoPort =
XContentMapValues.nodeIntegerValue(mongoSettings.get(PORT_FIELD), DEFAULT_DB_PORT);
try {
mongoServers.add(new ServerAddress(mongoHost, mongoPort));
} catch (UnknownHostException uhEx) {
logger.warn("Cannot add mongo server {}:{}", uhEx, mongoHost, mongoPort);
}
}
builder.mongoServers(mongoServers);
MongoClientOptions.Builder mongoClientOptionsBuilder =
MongoClientOptions.builder().socketKeepAlive(true);
// MongoDB options
if (mongoSettings.containsKey(OPTIONS_FIELD)) {
Map<String, Object> mongoOptionsSettings =
(Map<String, Object>) mongoSettings.get(OPTIONS_FIELD);
logger.trace("mongoOptionsSettings: " + mongoOptionsSettings);
builder.mongoSecondaryReadPreference(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SECONDARY_READ_PREFERENCE_FIELD), false));
builder.connectTimeout(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(CONNECT_TIMEOUT), DEFAULT_CONNECT_TIMEOUT));
builder.socketTimeout(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(SOCKET_TIMEOUT), DEFAULT_SOCKET_TIMEOUT));
builder.dropCollection(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(DROP_COLLECTION_FIELD), false));
String isMongos =
XContentMapValues.nodeStringValue(mongoOptionsSettings.get(IS_MONGOS_FIELD), null);
if (isMongos != null) {
builder.isMongos(Boolean.valueOf(isMongos));
}
builder.mongoUseSSL(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SSL_CONNECTION_FIELD), false));
builder.mongoSSLVerifyCertificate(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SSL_VERIFY_CERT_FIELD), true));
builder.advancedTransformation(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(ADVANCED_TRANSFORMATION_FIELD), false));
builder.skipInitialImport(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(SKIP_INITIAL_IMPORT_FIELD), false));
builder.connectionsPerHost(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(CONNECTIONS_PER_HOST), DEFAULT_CONNECTIONS_PER_HOST));
builder.threadsAllowedToBlockForConnectionMultiplier(
XContentMapValues.nodeIntegerValue(
mongoOptionsSettings.get(THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER),
DEFAULT_THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER));
mongoClientOptionsBuilder
.connectTimeout(builder.connectTimeout)
.socketTimeout(builder.socketTimeout)
.connectionsPerHost(builder.connectionsPerHost)
.threadsAllowedToBlockForConnectionMultiplier(
builder.threadsAllowedToBlockForConnectionMultiplier);
if (builder.mongoSecondaryReadPreference) {
mongoClientOptionsBuilder.readPreference(ReadPreference.secondaryPreferred());
}
if (builder.mongoUseSSL) {
mongoClientOptionsBuilder.socketFactory(getSSLSocketFactory());
}
if (mongoOptionsSettings.containsKey(PARENT_TYPES_FIELD)) {
Set<String> parentTypes = new HashSet<String>();
Object parentTypesSettings = mongoOptionsSettings.get(PARENT_TYPES_FIELD);
logger.trace("parentTypesSettings: " + parentTypesSettings);
boolean array = XContentMapValues.isArray(parentTypesSettings);
if (array) {
ArrayList<String> fields = (ArrayList<String>) parentTypesSettings;
for (String field : fields) {
logger.trace("Field: " + field);
parentTypes.add(field);
}
}
builder.parentTypes(parentTypes);
}
if (mongoOptionsSettings.containsKey(STORE_STATISTICS_FIELD)) {
Object storeStatistics = mongoOptionsSettings.get(STORE_STATISTICS_FIELD);
boolean object = XContentMapValues.isObject(storeStatistics);
if (object) {
Map<String, Object> storeStatisticsSettings = (Map<String, Object>) storeStatistics;
builder.storeStatistics(true);
builder.statisticsIndexName(
XContentMapValues.nodeStringValue(
storeStatisticsSettings.get(INDEX_OBJECT), riverName + "-stats"));
builder.statisticsTypeName(
XContentMapValues.nodeStringValue(
storeStatisticsSettings.get(TYPE_FIELD), "stats"));
} else {
builder.storeStatistics(XContentMapValues.nodeBooleanValue(storeStatistics, false));
if (builder.storeStatistics) {
builder.statisticsIndexName(riverName + "-stats");
builder.statisticsTypeName("stats");
}
}
}
// builder.storeStatistics(XContentMapValues.nodeBooleanValue(mongoOptionsSettings.get(STORE_STATISTICS_FIELD),
// false));
builder.importAllCollections(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(IMPORT_ALL_COLLECTIONS_FIELD), false));
builder.disableIndexRefresh(
XContentMapValues.nodeBooleanValue(
mongoOptionsSettings.get(DISABLE_INDEX_REFRESH_FIELD), false));
builder.includeCollection(
XContentMapValues.nodeStringValue(
mongoOptionsSettings.get(INCLUDE_COLLECTION_FIELD), ""));
if (mongoOptionsSettings.containsKey(INCLUDE_FIELDS_FIELD)) {
Set<String> includeFields = new HashSet<String>();
Object includeFieldsSettings = mongoOptionsSettings.get(INCLUDE_FIELDS_FIELD);
logger.trace("includeFieldsSettings: " + includeFieldsSettings);
boolean array = XContentMapValues.isArray(includeFieldsSettings);
if (array) {
ArrayList<String> fields = (ArrayList<String>) includeFieldsSettings;
for (String field : fields) {
logger.trace("Field: " + field);
includeFields.add(field);
}
}
if (!includeFields.contains(MongoDBRiver.MONGODB_ID_FIELD)) {
includeFields.add(MongoDBRiver.MONGODB_ID_FIELD);
}
builder.includeFields(includeFields);
} else if (mongoOptionsSettings.containsKey(EXCLUDE_FIELDS_FIELD)) {
Set<String> excludeFields = new HashSet<String>();
Object excludeFieldsSettings = mongoOptionsSettings.get(EXCLUDE_FIELDS_FIELD);
logger.trace("excludeFieldsSettings: " + excludeFieldsSettings);
boolean array = XContentMapValues.isArray(excludeFieldsSettings);
if (array) {
ArrayList<String> fields = (ArrayList<String>) excludeFieldsSettings;
for (String field : fields) {
logger.trace("Field: " + field);
excludeFields.add(field);
}
}
builder.excludeFields(excludeFields);
}
if (mongoOptionsSettings.containsKey(INITIAL_TIMESTAMP_FIELD)) {
BSONTimestamp timeStamp = null;
try {
Map<String, Object> initalTimestampSettings =
(Map<String, Object>) mongoOptionsSettings.get(INITIAL_TIMESTAMP_FIELD);
String scriptType = "js";
if (initalTimestampSettings.containsKey(INITIAL_TIMESTAMP_SCRIPT_TYPE_FIELD)) {
scriptType =
initalTimestampSettings.get(INITIAL_TIMESTAMP_SCRIPT_TYPE_FIELD).toString();
}
if (initalTimestampSettings.containsKey(INITIAL_TIMESTAMP_SCRIPT_FIELD)) {
ExecutableScript scriptExecutable =
scriptService.executable(
scriptType,
initalTimestampSettings.get(INITIAL_TIMESTAMP_SCRIPT_FIELD).toString(),
ScriptService.ScriptType.INLINE,
Maps.newHashMap());
Object ctx = scriptExecutable.run();
logger.trace("initialTimestamp script returned: {}", ctx);
if (ctx != null) {
long timestamp = Long.parseLong(ctx.toString());
timeStamp = new BSONTimestamp((int) (new Date(timestamp).getTime() / 1000), 1);
}
}
} catch (Throwable t) {
logger.error("Could not set initial timestamp", t);
} finally {
builder.initialTimestamp(timeStamp);
}
}
}
builder.mongoClientOptions(mongoClientOptionsBuilder.build());
// Credentials
if (mongoSettings.containsKey(CREDENTIALS_FIELD)) {
String dbCredential;
String mau = "";
String map = "";
String maad = "";
String mlu = "";
String mlp = "";
String mlad = "";
// String mdu = "";
// String mdp = "";
Object mongoCredentialsSettings = mongoSettings.get(CREDENTIALS_FIELD);
boolean array = XContentMapValues.isArray(mongoCredentialsSettings);
if (array) {
ArrayList<Map<String, Object>> credentials =
(ArrayList<Map<String, Object>>) mongoCredentialsSettings;
for (Map<String, Object> credential : credentials) {
dbCredential = XContentMapValues.nodeStringValue(credential.get(DB_FIELD), null);
if (ADMIN_DB_FIELD.equals(dbCredential)) {
mau = XContentMapValues.nodeStringValue(credential.get(USER_FIELD), null);
map = XContentMapValues.nodeStringValue(credential.get(PASSWORD_FIELD), null);
maad = XContentMapValues.nodeStringValue(credential.get(AUTH_FIELD), null);
} else if (LOCAL_DB_FIELD.equals(dbCredential)) {
mlu = XContentMapValues.nodeStringValue(credential.get(USER_FIELD), null);
mlp = XContentMapValues.nodeStringValue(credential.get(PASSWORD_FIELD), null);
mlad = XContentMapValues.nodeStringValue(credential.get(AUTH_FIELD), null);
// } else {
// mdu = XContentMapValues.nodeStringValue(
// credential.get(USER_FIELD), null);
// mdp = XContentMapValues.nodeStringValue(
// credential.get(PASSWORD_FIELD), null);
}
}
}
builder.mongoAdminUser(mau);
builder.mongoAdminPassword(map);
builder.mongoAdminAuthDatabase(maad);
builder.mongoLocalUser(mlu);
builder.mongoLocalPassword(mlp);
builder.mongoLocalAuthDatabase(mlad);
// mongoDbUser = mdu;
// mongoDbPassword = mdp;
}
builder.mongoDb(XContentMapValues.nodeStringValue(mongoSettings.get(DB_FIELD), riverName));
builder.mongoCollection(
XContentMapValues.nodeStringValue(mongoSettings.get(COLLECTION_FIELD), riverName));
builder.mongoGridFS(
XContentMapValues.nodeBooleanValue(mongoSettings.get(GRIDFS_FIELD), false));
if (mongoSettings.containsKey(FILTER_FIELD)) {
String filter = XContentMapValues.nodeStringValue(mongoSettings.get(FILTER_FIELD), "");
filter = removePrefix("o.", filter);
builder.mongoCollectionFilter(convertToBasicDBObject(filter));
// DBObject bsonObject = (DBObject) JSON.parse(filter);
// builder.mongoOplogFilter(convertToBasicDBObject(addPrefix("o.",
// filter)));
builder.mongoOplogFilter(convertToBasicDBObject(removePrefix("o.", filter)));
// } else {
// builder.mongoOplogFilter("");
}
if (mongoSettings.containsKey(SCRIPT_FIELD)) {
String scriptType = "js";
builder.script(mongoSettings.get(SCRIPT_FIELD).toString());
if (mongoSettings.containsKey("scriptType")) {
scriptType = mongoSettings.get("scriptType").toString();
} else if (mongoSettings.containsKey(SCRIPT_TYPE_FIELD)) {
scriptType = mongoSettings.get(SCRIPT_TYPE_FIELD).toString();
}
builder.scriptType(scriptType);
}
} else {
mongoHost = DEFAULT_DB_HOST;
mongoPort = DEFAULT_DB_PORT;
try {
mongoServers.add(new ServerAddress(mongoHost, mongoPort));
builder.mongoServers(mongoServers);
} catch (UnknownHostException e) {
e.printStackTrace();
}
builder.mongoDb(riverName);
builder.mongoCollection(riverName);
}
if (settings.settings().containsKey(INDEX_OBJECT)) {
Map<String, Object> indexSettings =
(Map<String, Object>) settings.settings().get(INDEX_OBJECT);
builder.indexName(
XContentMapValues.nodeStringValue(indexSettings.get(NAME_FIELD), builder.mongoDb));
builder.typeName(
XContentMapValues.nodeStringValue(indexSettings.get(TYPE_FIELD), builder.mongoDb));
Bulk.Builder bulkBuilder = new Bulk.Builder();
if (indexSettings.containsKey(BULK_FIELD)) {
Map<String, Object> bulkSettings = (Map<String, Object>) indexSettings.get(BULK_FIELD);
int bulkActions =
XContentMapValues.nodeIntegerValue(
bulkSettings.get(ACTIONS_FIELD), DEFAULT_BULK_ACTIONS);
bulkBuilder.bulkActions(bulkActions);
String size =
XContentMapValues.nodeStringValue(
bulkSettings.get(SIZE_FIELD), DEFAULT_BULK_SIZE.toString());
bulkBuilder.bulkSize(ByteSizeValue.parseBytesSizeValue(size));
bulkBuilder.concurrentRequests(
XContentMapValues.nodeIntegerValue(
bulkSettings.get(CONCURRENT_REQUESTS_FIELD),
EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY)));
bulkBuilder.flushInterval(
XContentMapValues.nodeTimeValue(
bulkSettings.get(FLUSH_INTERVAL_FIELD), DEFAULT_FLUSH_INTERVAL));
builder.throttleSize(
XContentMapValues.nodeIntegerValue(
indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5));
} else {
int bulkActions =
XContentMapValues.nodeIntegerValue(
indexSettings.get(BULK_SIZE_FIELD), DEFAULT_BULK_ACTIONS);
bulkBuilder.bulkActions(bulkActions);
bulkBuilder.bulkSize(DEFAULT_BULK_SIZE);
bulkBuilder.flushInterval(
XContentMapValues.nodeTimeValue(
indexSettings.get(BULK_TIMEOUT_FIELD), DEFAULT_FLUSH_INTERVAL));
bulkBuilder.concurrentRequests(
XContentMapValues.nodeIntegerValue(
indexSettings.get(CONCURRENT_BULK_REQUESTS_FIELD),
EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY)));
builder.throttleSize(
XContentMapValues.nodeIntegerValue(
indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5));
}
builder.bulk(bulkBuilder.build());
} else {
builder.indexName(builder.mongoDb);
builder.typeName(builder.mongoDb);
builder.bulk(new Bulk.Builder().build());
}
return builder.build();
}
@Override
public Mapper.Builder parse(
String fieldName, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
StringFieldMapper.Builder builder = new StringFieldMapper.Builder(fieldName);
// hack for the fact that string can't just accept true/false for
// the index property and still accepts no/not_analyzed/analyzed
final Object index = node.remove("index");
if (index != null) {
final String normalizedIndex = Strings.toUnderscoreCase(index.toString());
switch (normalizedIndex) {
case "analyzed":
builder.tokenized(true);
node.put("index", true);
break;
case "not_analyzed":
builder.tokenized(false);
node.put("index", true);
break;
case "no":
node.put("index", false);
break;
default:
throw new IllegalArgumentException(
"Can't parse [index] value ["
+ index
+ "] for field ["
+ fieldName
+ "], expected [true], [false], [no], [not_analyzed] or [analyzed]");
}
}
parseTextField(builder, fieldName, node, parserContext);
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
iterator.hasNext(); ) {
Map.Entry<String, Object> entry = iterator.next();
String propName = Strings.toUnderscoreCase(entry.getKey());
Object propNode = entry.getValue();
if (propName.equals("null_value")) {
if (propNode == null) {
throw new MapperParsingException("Property [null_value] cannot be null.");
}
builder.nullValue(propNode.toString());
iterator.remove();
} else if (propName.equals("position_increment_gap")) {
int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1);
if (newPositionIncrementGap < 0) {
throw new MapperParsingException("positions_increment_gap less than 0 aren't allowed.");
}
builder.positionIncrementGap(newPositionIncrementGap);
// we need to update to actual analyzers if they are not set in this case...
// so we can inject the position increment gap...
if (builder.fieldType().indexAnalyzer() == null) {
builder
.fieldType()
.setIndexAnalyzer(parserContext.analysisService().defaultIndexAnalyzer());
}
if (builder.fieldType().searchAnalyzer() == null) {
builder
.fieldType()
.setSearchAnalyzer(parserContext.analysisService().defaultSearchAnalyzer());
}
if (builder.fieldType().searchQuoteAnalyzer() == null) {
builder
.fieldType()
.setSearchQuoteAnalyzer(
parserContext.analysisService().defaultSearchQuoteAnalyzer());
}
iterator.remove();
} else if (propName.equals("ignore_above")) {
builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
iterator.remove();
} else if (parseMultiField(builder, fieldName, parserContext, propName, propNode)) {
iterator.remove();
}
}
return builder;
}
public void testFilter() throws Exception {
XContentBuilder builder =
XContentFactory.jsonBuilder()
.startObject()
.field("test1", "value1")
.field("test2", "value2")
.field("something_else", "value3")
.endObject();
Map<String, Object> source;
try (XContentParser parser =
XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
source = parser.map();
}
Map<String, Object> filter =
XContentMapValues.filter(source, new String[] {"test1"}, Strings.EMPTY_ARRAY);
assertThat(filter.size(), equalTo(1));
assertThat(filter.get("test1").toString(), equalTo("value1"));
filter = XContentMapValues.filter(source, new String[] {"test*"}, Strings.EMPTY_ARRAY);
assertThat(filter.size(), equalTo(2));
assertThat(filter.get("test1").toString(), equalTo("value1"));
assertThat(filter.get("test2").toString(), equalTo("value2"));
filter = XContentMapValues.filter(source, Strings.EMPTY_ARRAY, new String[] {"test1"});
assertThat(filter.size(), equalTo(2));
assertThat(filter.get("test2").toString(), equalTo("value2"));
assertThat(filter.get("something_else").toString(), equalTo("value3"));
// more complex object...
builder =
XContentFactory.jsonBuilder()
.startObject()
.startObject("path1")
.startArray("path2")
.startObject()
.field("test", "value1")
.endObject()
.startObject()
.field("test", "value2")
.endObject()
.endArray()
.endObject()
.field("test1", "value1")
.endObject();
try (XContentParser parser =
XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
source = parser.map();
}
filter = XContentMapValues.filter(source, new String[] {"path1"}, Strings.EMPTY_ARRAY);
assertThat(filter.size(), equalTo(1));
filter = XContentMapValues.filter(source, new String[] {"path1*"}, Strings.EMPTY_ARRAY);
assertThat(filter.get("path1"), equalTo(source.get("path1")));
assertThat(filter.containsKey("test1"), equalTo(false));
filter = XContentMapValues.filter(source, new String[] {"test1*"}, Strings.EMPTY_ARRAY);
assertThat(filter.get("test1"), equalTo(source.get("test1")));
assertThat(filter.containsKey("path1"), equalTo(false));
filter = XContentMapValues.filter(source, new String[] {"path1.path2.*"}, Strings.EMPTY_ARRAY);
assertThat(filter.get("path1"), equalTo(source.get("path1")));
assertThat(filter.containsKey("test1"), equalTo(false));
}
