@Test public void testIpMultiField() throws Exception { assertAcked( client() .admin() .indices() .prepareCreate("my-index") .addMapping("my-type", createMappingSource("ip"))); GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("my-index").get(); MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type"); assertThat(mappingMetaData, not(nullValue())); Map<String, Object> mappingSource = mappingMetaData.sourceAsMap(); Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource)); assertThat(aField.size(), equalTo(2)); assertThat(aField.get("type").toString(), equalTo("ip")); assertThat(aField.get("fields"), notNullValue()); Map bField = ((Map) XContentMapValues.extractValue("properties.a.fields.b", mappingSource)); assertThat(bField.size(), equalTo(2)); assertThat(bField.get("type").toString(), equalTo("string")); assertThat(bField.get("index").toString(), equalTo("not_analyzed")); client() .prepareIndex("my-index", "my-type", "1") .setSource("a", "127.0.0.1") .setRefresh(true) .get(); CountResponse countResponse = client().prepareCount("my-index").setQuery(matchQuery("a.b", "127.0.0.1")).get(); assertThat(countResponse.getCount(), equalTo(1l)); }
@Override public HashMap<String, String> loadKey(String username) { logger.info("loading password for username {}", username); HashMap<String, String> ret = new HashMap<>(); String riverIndexName = getRiverIndexName(); refreshSearchIndex(riverIndexName); GetResponse resp = client.prepareGet(riverIndexName, riverName().name(), "_pwd").execute().actionGet(); if (resp.isExists()) { if (logger.isDebugEnabled()) { logger.debug("Password document: {}", resp.getSourceAsString()); } Map<String, Object> newset = resp.getSource(); Set<String> keys = newset.keySet(); for (String s : keys) { logger.info( "Added key {} with a value of {}", s, XContentMapValues.nodeStringValue(newset.get(s), null)); ret.put(s, XContentMapValues.nodeStringValue(newset.get(s), null)); } } if (ret.isEmpty()) { return null; } return ret; }
@SuppressWarnings("unchecked") public void testNestedFiltering() { Map<String, Object> map = new HashMap<>(); map.put("field", "value"); map.put( "array", Arrays.asList( 1, new HashMap<String, Object>() { { put("nested", 2); put("nested_2", 3); } })); Map<String, Object> falteredMap = XContentMapValues.filter(map, new String[] {"array.nested"}, Strings.EMPTY_ARRAY); assertThat(falteredMap.size(), equalTo(1)); // Selecting members of objects within arrays (ex. [ 1, { nested: "value"} ]) always returns // all values in the array (1 in the ex) // this is expected behavior as this types of objects are not supported in ES assertThat((Integer) ((List) falteredMap.get("array")).get(0), equalTo(1)); assertThat(((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).size(), equalTo(1)); assertThat( (Integer) ((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).get("nested"), equalTo(2)); falteredMap = XContentMapValues.filter(map, new String[] {"array.*"}, Strings.EMPTY_ARRAY); assertThat(falteredMap.size(), equalTo(1)); assertThat((Integer) ((List) falteredMap.get("array")).get(0), equalTo(1)); assertThat(((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).size(), equalTo(2)); map.clear(); map.put("field", "value"); map.put( "obj", new HashMap<String, Object>() { { put("field", "value"); put("field2", "value2"); } }); falteredMap = XContentMapValues.filter(map, new String[] {"obj.field"}, Strings.EMPTY_ARRAY); assertThat(falteredMap.size(), equalTo(1)); assertThat(((Map<String, Object>) falteredMap.get("obj")).size(), equalTo(1)); assertThat( (String) ((Map<String, Object>) falteredMap.get("obj")).get("field"), equalTo("value")); falteredMap = XContentMapValues.filter(map, new String[] {"obj.*"}, Strings.EMPTY_ARRAY); assertThat(falteredMap.size(), equalTo(1)); assertThat(((Map<String, Object>) falteredMap.get("obj")).size(), equalTo(2)); assertThat( (String) ((Map<String, Object>) falteredMap.get("obj")).get("field"), equalTo("value")); assertThat( (String) ((Map<String, Object>) falteredMap.get("obj")).get("field2"), equalTo("value2")); }
public void testExtractRawValue() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("test", "value").endObject(); Map<String, Object> map; try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) { map = parser.map(); } assertThat(XContentMapValues.extractRawValues("test", map).get(0).toString(), equalTo("value")); builder = XContentFactory.jsonBuilder().startObject().field("test.me", "value").endObject(); try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) { map = parser.map(); } assertThat( XContentMapValues.extractRawValues("test.me", map).get(0).toString(), equalTo("value")); builder = XContentFactory.jsonBuilder() .startObject() .startObject("path1") .startObject("path2") .field("test", "value") .endObject() .endObject() .endObject(); try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) { map = parser.map(); } assertThat( XContentMapValues.extractRawValues("path1.path2.test", map).get(0).toString(), equalTo("value")); builder = XContentFactory.jsonBuilder() .startObject() .startObject("path1.xxx") .startObject("path2.yyy") .field("test", "value") .endObject() .endObject() .endObject(); try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) { map = parser.map(); } assertThat( XContentMapValues.extractRawValues("path1.xxx.path2.yyy.test", map).get(0).toString(), equalTo("value")); }
public void testSupplementaryCharactersInPaths() { Map<String, Object> map = new HashMap<>(); map.put("搜索", 2); map.put("指数", 3); assertEquals( Collections.singletonMap("搜索", 2), XContentMapValues.filter(map, new String[] {"搜索"}, new String[0])); assertEquals( Collections.singletonMap("指数", 3), XContentMapValues.filter(map, new String[0], new String[] {"搜索"})); }
@SuppressWarnings("unchecked") public void testCompleteObjectFiltering() { Map<String, Object> map = new HashMap<>(); map.put("field", "value"); map.put( "obj", new HashMap<String, Object>() { { put("field", "value"); put("field2", "value2"); } }); map.put( "array", Arrays.asList( 1, new HashMap<String, Object>() { { put("field", "value"); put("field2", "value2"); } })); Map<String, Object> filteredMap = XContentMapValues.filter(map, new String[] {"obj"}, Strings.EMPTY_ARRAY); assertThat(filteredMap.size(), equalTo(1)); assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(2)); assertThat( ((Map<String, Object>) filteredMap.get("obj")).get("field").toString(), equalTo("value")); assertThat( ((Map<String, Object>) filteredMap.get("obj")).get("field2").toString(), equalTo("value2")); filteredMap = XContentMapValues.filter(map, new String[] {"obj"}, new String[] {"*.field2"}); assertThat(filteredMap.size(), equalTo(1)); assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(1)); assertThat( ((Map<String, Object>) filteredMap.get("obj")).get("field").toString(), equalTo("value")); filteredMap = XContentMapValues.filter(map, new String[] {"array"}, new String[] {}); assertThat(filteredMap.size(), equalTo(1)); assertThat(((List) filteredMap.get("array")).size(), equalTo(2)); assertThat((Integer) ((List) filteredMap.get("array")).get(0), equalTo(1)); assertThat(((Map<String, Object>) ((List) filteredMap.get("array")).get(1)).size(), equalTo(2)); filteredMap = XContentMapValues.filter(map, new String[] {"array"}, new String[] {"*.field2"}); assertThat(filteredMap.size(), equalTo(1)); assertThat(((List) filteredMap.get("array")).size(), equalTo(2)); assertThat((Integer) ((List) filteredMap.get("array")).get(0), equalTo(1)); assertThat(((Map<String, Object>) ((List) filteredMap.get("array")).get(1)).size(), equalTo(1)); assertThat( ((Map<String, Object>) ((List) filteredMap.get("array")).get(1)).get("field").toString(), equalTo("value")); }
public void testSharedPrefixes() { Map<String, Object> map = new HashMap<>(); map.put("foobar", 2); map.put("foobaz", 3); assertEquals( Collections.singletonMap("foobar", 2), XContentMapValues.filter(map, new String[] {"foobar"}, new String[0])); assertEquals( Collections.singletonMap("foobaz", 3), XContentMapValues.filter(map, new String[0], new String[] {"foobar"})); }
@Override public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException { StringFieldMapper.Builder builder = stringField(name); parseField(builder, name, node, parserContext); for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext(); ) { Map.Entry<String, Object> entry = iterator.next(); String propName = Strings.toUnderscoreCase(entry.getKey()); Object propNode = entry.getValue(); if (propName.equals("null_value")) { if (propNode == null) { throw new MapperParsingException("Property [null_value] cannot be null."); } builder.nullValue(propNode.toString()); iterator.remove(); } else if (propName.equals("search_quote_analyzer")) { NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString()); if (analyzer == null) { throw new MapperParsingException( "Analyzer [" + propNode.toString() + "] not found for field [" + name + "]"); } builder.searchQuotedAnalyzer(analyzer); iterator.remove(); } else if (propName.equals("position_offset_gap")) { builder.positionOffsetGap(XContentMapValues.nodeIntegerValue(propNode, -1)); // we need to update to actual analyzers if they are not set in this case... // so we can inject the position offset gap... if (builder.indexAnalyzer == null) { builder.indexAnalyzer = parserContext.analysisService().defaultIndexAnalyzer(); } if (builder.searchAnalyzer == null) { builder.searchAnalyzer = parserContext.analysisService().defaultSearchAnalyzer(); } if (builder.searchQuotedAnalyzer == null) { builder.searchQuotedAnalyzer = parserContext.analysisService().defaultSearchQuoteAnalyzer(); } iterator.remove(); } else if (propName.equals("ignore_above")) { builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1)); iterator.remove(); } else if (parseMultiField(builder, name, parserContext, propName, propNode)) { iterator.remove(); } } return builder; }
/** The template source definition. */ public PutIndexTemplateRequest source(Map templateSource) { Map<String, Object> source = templateSource; if (source.containsKey("template")) { template(source.get("template").toString()); } if (source.containsKey("order")) { order(XContentMapValues.nodeIntegerValue(source.get("order"), order())); } if (source.containsKey("settings")) { if (!(source.get("settings") instanceof Map)) { throw new ElasticSearchIllegalArgumentException( "Malformed settings section, should include an inner object"); } settings((Map<String, Object>) source.get("settings")); } if (source.containsKey("mappings")) { Map<String, Object> mappings = (Map<String, Object>) source.get("mappings"); for (Map.Entry<String, Object> entry : mappings.entrySet()) { if (!(entry.getValue() instanceof Map)) { throw new ElasticSearchIllegalArgumentException( "Malformed mappings section for type [" + entry.getKey() + "], should include an inner object describing the mapping"); } mapping(entry.getKey(), (Map<String, Object>) entry.getValue()); } } return this; }
public void testFilterWithEmptyIncludesExcludes() { Map<String, Object> map = new HashMap<>(); map.put("field", "value"); Map<String, Object> filteredMap = XContentMapValues.filter(map, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY); assertThat(filteredMap.size(), equalTo(1)); assertThat(filteredMap.get("field").toString(), equalTo("value")); }
@SuppressWarnings("unchecked") public void testFilterIncludesUsingStarPrefix() { Map<String, Object> map = new HashMap<>(); map.put("field", "value"); map.put( "obj", new HashMap<String, Object>() { { put("field", "value"); put("field2", "value2"); } }); map.put( "n_obj", new HashMap<String, Object>() { { put("n_field", "value"); put("n_field2", "value2"); } }); Map<String, Object> filteredMap = XContentMapValues.filter(map, new String[] {"*.field2"}, Strings.EMPTY_ARRAY); assertThat(filteredMap.size(), equalTo(1)); assertThat(filteredMap, hasKey("obj")); assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(1)); assertThat(((Map<String, Object>) filteredMap.get("obj")), hasKey("field2")); // only objects filteredMap = XContentMapValues.filter(map, new String[] {"*.*"}, Strings.EMPTY_ARRAY); assertThat(filteredMap.size(), equalTo(2)); assertThat(filteredMap, hasKey("obj")); assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(2)); assertThat(filteredMap, hasKey("n_obj")); assertThat(((Map<String, Object>) filteredMap.get("n_obj")).size(), equalTo(2)); filteredMap = XContentMapValues.filter(map, new String[] {"*"}, new String[] {"*.*2"}); assertThat(filteredMap.size(), equalTo(3)); assertThat(filteredMap, hasKey("field")); assertThat(filteredMap, hasKey("obj")); assertThat(((Map) filteredMap.get("obj")).size(), equalTo(1)); assertThat(((Map<String, Object>) filteredMap.get("obj")), hasKey("field")); assertThat(filteredMap, hasKey("n_obj")); assertThat(((Map<String, Object>) filteredMap.get("n_obj")).size(), equalTo(1)); assertThat(((Map<String, Object>) filteredMap.get("n_obj")), hasKey("n_field")); }
@SuppressWarnings({"unchecked"}) @Inject public SimpleRiver( RiverName riverName, RiverSettings settings, Client client, ThreadPool threadPool) { super(riverName, settings); this.client = client; if (settings.settings().containsKey("simple")) { Map<String, Object> simpleSettings = (Map<String, Object>) settings.settings().get("simple"); simpleNumber = XContentMapValues.nodeIntegerValue(simpleSettings.get("number"), 100); fieldName = XContentMapValues.nodeStringValue(simpleSettings.get("field"), "test"); poll = XContentMapValues.nodeTimeValue( simpleSettings.get("poll"), TimeValue.timeValueMinutes(60)); } logger.info( "creating simple stream river for [{} numbers] with field [{}]", simpleNumber, fieldName); if (settings.settings().containsKey("index")) { Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index"); indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name()); typeName = XContentMapValues.nodeStringValue(indexSettings.get("type"), "simple_type"); bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size"), 100); bulkThreshold = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_threshold"), 10); } else { indexName = riverName.name(); typeName = "simple_type"; bulkSize = 100; bulkThreshold = 10; } }
public void testPrefixedNamesFilteringTest() { Map<String, Object> map = new HashMap<>(); map.put("obj", "value"); map.put("obj_name", "value_name"); Map<String, Object> filterdMap = XContentMapValues.filter(map, new String[] {"obj_name"}, Strings.EMPTY_ARRAY); assertThat(filterdMap.size(), equalTo(1)); assertThat((String) filterdMap.get("obj_name"), equalTo("value_name")); }
public void testThatFilterIncludesEmptyObjectWhenUsingIncludes() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("obj").endObject().endObject(); Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true); Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[] {"obj"}, Strings.EMPTY_ARRAY); assertThat(mapTuple.v2(), equalTo(filteredSource)); }
public void testDotsInFieldNames() { Map<String, Object> map = new HashMap<>(); map.put("foo.bar", 2); Map<String, Object> sub = new HashMap<>(); sub.put("baz", 3); map.put("foo", sub); map.put("quux", 5); // dots in field names in includes Map<String, Object> filtered = XContentMapValues.filter(map, new String[] {"foo"}, new String[0]); Map<String, Object> expected = new HashMap<>(map); expected.remove("quux"); assertEquals(expected, filtered); // dots in field names in excludes filtered = XContentMapValues.filter(map, new String[0], new String[] {"foo"}); expected = new HashMap<>(map); expected.keySet().retainAll(Collections.singleton("quux")); assertEquals(expected, filtered); }
@SuppressWarnings({"unchecked"}) public void testNotOmittingObjectWithNestedExcludedObject() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() .startObject("obj1") .startObject("obj2") .startObject("obj3") .endObject() .endObject() .endObject() .endObject(); // implicit include Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true); Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[] {"*.obj2"}); assertThat(filteredSource.size(), equalTo(1)); assertThat(filteredSource, hasKey("obj1")); assertThat(((Map) filteredSource.get("obj1")).size(), Matchers.equalTo(0)); // explicit include filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[] {"obj1"}, new String[] {"*.obj2"}); assertThat(filteredSource.size(), equalTo(1)); assertThat(filteredSource, hasKey("obj1")); assertThat(((Map) filteredSource.get("obj1")).size(), Matchers.equalTo(0)); // wild card include filteredSource = XContentMapValues.filter(mapTuple.v2(), new String[] {"*.obj2"}, new String[] {"*.obj3"}); assertThat(filteredSource.size(), equalTo(1)); assertThat(filteredSource, hasKey("obj1")); assertThat(((Map<String, Object>) filteredSource.get("obj1")), hasKey("obj2")); assertThat(((Map) ((Map) filteredSource.get("obj1")).get("obj2")).size(), Matchers.equalTo(0)); }
@Nullable @Override public Object referenceValue(Reference reference) { if (updatedColumnValues == null) { return super.referenceValue(reference); } Object value = updatedColumnValues.get(reference.ident().columnIdent().fqn()); if (value == null && !reference.ident().isColumn()) { value = XContentMapValues.extractValue( reference.ident().columnIdent().fqn(), updatedColumnValues); } return reference.valueType().value(value); }
@Override public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException { Builder builder = geoShapeField(name); for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext(); ) { Map.Entry<String, Object> entry = iterator.next(); String fieldName = Strings.toUnderscoreCase(entry.getKey()); Object fieldNode = entry.getValue(); if (Names.TREE.equals(fieldName)) { builder.fieldType().setTree(fieldNode.toString()); iterator.remove(); } else if (Names.TREE_LEVELS.equals(fieldName)) { builder.fieldType().setTreeLevels(Integer.parseInt(fieldNode.toString())); iterator.remove(); } else if (Names.TREE_PRESISION.equals(fieldName)) { builder .fieldType() .setPrecisionInMeters( DistanceUnit.parse( fieldNode.toString(), DistanceUnit.DEFAULT, DistanceUnit.DEFAULT)); iterator.remove(); } else if (Names.DISTANCE_ERROR_PCT.equals(fieldName)) { builder.fieldType().setDistanceErrorPct(Double.parseDouble(fieldNode.toString())); iterator.remove(); } else if (Names.ORIENTATION.equals(fieldName)) { builder .fieldType() .setOrientation(ShapeBuilder.orientationFromString(fieldNode.toString())); iterator.remove(); } else if (Names.STRATEGY.equals(fieldName)) { builder.fieldType().setStrategyName(fieldNode.toString()); iterator.remove(); } else if (Names.COERCE.equals(fieldName)) { builder.coerce(nodeBooleanValue(fieldNode)); iterator.remove(); } else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName) && builder.fieldType().strategyName.equals(SpatialStrategy.TERM.getStrategyName()) == false) { builder.fieldType().setPointsOnly(XContentMapValues.nodeBooleanValue(fieldNode)); iterator.remove(); } } return builder; }
public void testNotOmittingObjectsWithExcludedProperties() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() .startObject("obj") .field("f1", "v1") .endObject() .endObject(); Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(builder.bytes(), true); Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[] {"obj.f1"}); assertThat(filteredSource.size(), equalTo(1)); assertThat(filteredSource, hasKey("obj")); assertThat(((Map) filteredSource.get("obj")).size(), equalTo(0)); }
/** The template source definition. */ public PutIndexTemplateRequest source(Map templateSource) { Map<String, Object> source = templateSource; for (Map.Entry<String, Object> entry : source.entrySet()) { String name = entry.getKey(); if (name.equals("template")) { template(entry.getValue().toString()); } else if (name.equals("order")) { order(XContentMapValues.nodeIntegerValue(entry.getValue(), order())); } else if (name.equals("settings")) { if (!(entry.getValue() instanceof Map)) { throw new ElasticSearchIllegalArgumentException( "Malformed settings section, should include an inner object"); } settings((Map<String, Object>) entry.getValue()); } else if (name.equals("mappings")) { Map<String, Object> mappings = (Map<String, Object>) entry.getValue(); for (Map.Entry<String, Object> entry1 : mappings.entrySet()) { if (!(entry1.getValue() instanceof Map)) { throw new ElasticSearchIllegalArgumentException( "Malformed mappings section for type [" + entry1.getKey() + "], should include an inner object describing the mapping"); } mapping(entry1.getKey(), (Map<String, Object>) entry1.getValue()); } } else { // maybe custom? IndexMetaData.Custom.Factory factory = IndexMetaData.lookupFactory(name); if (factory != null) { try { customs.put(name, factory.fromMap((Map<String, Object>) entry.getValue())); } catch (IOException e) { throw new ElasticSearchParseException( "failed to parse custom metadata for [" + name + "]"); } } } } return this; }
@Override public Mapper.Builder parse( String fieldName, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException { TextFieldMapper.Builder builder = new TextFieldMapper.Builder(fieldName); builder.fieldType().setIndexAnalyzer(parserContext.analysisService().defaultIndexAnalyzer()); builder .fieldType() .setSearchAnalyzer(parserContext.analysisService().defaultSearchAnalyzer()); builder .fieldType() .setSearchQuoteAnalyzer(parserContext.analysisService().defaultSearchQuoteAnalyzer()); parseTextField(builder, fieldName, node, parserContext); for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext(); ) { Map.Entry<String, Object> entry = iterator.next(); String propName = entry.getKey(); Object propNode = entry.getValue(); if (propName.equals("position_increment_gap")) { int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1); builder.positionIncrementGap(newPositionIncrementGap); iterator.remove(); } else if (propName.equals("fielddata")) { builder.fielddata(XContentMapValues.nodeBooleanValue(propNode)); iterator.remove(); } else if (propName.equals("eager_global_ordinals")) { builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode)); iterator.remove(); } else if (propName.equals("fielddata_frequency_filter")) { Map<?, ?> frequencyFilter = (Map<?, ?>) propNode; double minFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("min"), 0); double maxFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("max"), Integer.MAX_VALUE); int minSegmentSize = XContentMapValues.nodeIntegerValue(frequencyFilter.remove("min_segment_size"), 0); builder.fielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize); DocumentMapperParser.checkNoRemainingFields( propName, frequencyFilter, parserContext.indexVersionCreated()); iterator.remove(); } } return builder; }
/** * Configure the river. * * @param settings used for configuration. */ @SuppressWarnings({"unchecked"}) protected void configure(Map<String, Object> settings) { if (!closed) throw new IllegalStateException("Remote River must be stopped to configure it!"); if (settings.containsKey("remote")) { Map<String, Object> remoteSettings = (Map<String, Object>) settings.get("remote"); maxIndexingThreads = XContentMapValues.nodeIntegerValue(remoteSettings.get("maxIndexingThreads"), 1); SpaceIndexingMode sim = SpaceIndexingMode.parseConfiguration((String) remoteSettings.get("listDocumentsMode")); if (sim != null) spaceIndexingMode = sim; else if (XContentMapValues.nodeBooleanValue(remoteSettings.get("simpleGetDocuments"), false)) spaceIndexingMode = SpaceIndexingMode.SIMPLE; indexFullUpdatePeriod = Utils.parseTimeValue(remoteSettings, "indexFullUpdatePeriod", 12, TimeUnit.HOURS); String ifuce = Utils.trimToNull((String) remoteSettings.get("indexFullUpdateCronExpression")); if (ifuce != null) { try { this.indexFullUpdateCronExpression = new CronExpression(ifuce); } catch (ParseException e) { throw new SettingsException( "Cron expression in indexFullUpdateCronExpression is invalid: " + e.getMessage()); } } if (spaceIndexingMode.isIncrementalUpdateSupported() || (indexFullUpdatePeriod < 1 && indexFullUpdateCronExpression == null)) indexUpdatePeriod = Utils.parseTimeValue(remoteSettings, "indexUpdatePeriod", 5, TimeUnit.MINUTES); else indexUpdatePeriod = 0; if (remoteSettings.containsKey("spacesIndexed")) { allIndexedSpacesKeys = Utils.parseCsvString( XContentMapValues.nodeStringValue(remoteSettings.get("spacesIndexed"), null)); if (allIndexedSpacesKeys != null) { // stop spaces loading from remote system allIndexedSpacesKeysNextRefresh = Long.MAX_VALUE; } } if (remoteSettings.containsKey("spaceKeysExcluded")) { spaceKeysExcluded = Utils.parseCsvString( XContentMapValues.nodeStringValue(remoteSettings.get("spaceKeysExcluded"), null)); } String remoteClientClass = Utils.trimToNull( XContentMapValues.nodeStringValue(remoteSettings.get("remoteClientClass"), null)); if (remoteClientClass != null) { try { remoteSystemClient = (IRemoteSystemClient) Class.forName(remoteClientClass).newInstance(); } catch (Exception e) { throw new SettingsException( "Unable to instantiate class defined by 'remote/remoteClientClass': " + e.getMessage()); } } else { remoteSystemClient = new GetJSONClient(); } remoteSystemClient.init( this, remoteSettings, allIndexedSpacesKeysNextRefresh != Long.MAX_VALUE, this); } else { throw new SettingsException("'remote' element of river configuration structure not found"); } Map<String, Object> indexSettings = null; if (settings.containsKey("index")) { indexSettings = (Map<String, Object>) settings.get("index"); indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name()); typeName = XContentMapValues.nodeStringValue( indexSettings.get("type"), INDEX_DOCUMENT_TYPE_NAME_DEFAULT); } else { indexName = riverName.name(); typeName = INDEX_DOCUMENT_TYPE_NAME_DEFAULT; } Map<String, Object> activityLogSettings = null; if (settings.containsKey("activity_log")) { activityLogSettings = (Map<String, Object>) settings.get("activity_log"); activityLogIndexName = Utils.trimToNull( XContentMapValues.nodeStringValue(activityLogSettings.get("index"), null)); if (activityLogIndexName == null) { throw new SettingsException( "'activity_log/index' element of river configuration structure must be defined with some string"); } activityLogTypeName = Utils.trimToNull( XContentMapValues.nodeStringValue( activityLogSettings.get("type"), INDEX_ACTIVITY_TYPE_NAME_DEFAULT)); } documentIndexStructureBuilder = new DocumentWithCommentsIndexStructureBuilder( this, indexName, typeName, indexSettings, spaceIndexingMode.isUpdateDateMandatory()); preparePreprocessors(indexSettings, documentIndexStructureBuilder); remoteSystemClient.setIndexStructureBuilder(documentIndexStructureBuilder); logger.info( "Configured Remote River '{}'. Search index name '{}', document type for issues '{}'. Indexing mode '{}'.", riverName.getName(), indexName, typeName, spaceIndexingMode.getConfigValue()); if (activityLogIndexName != null) { logger.info( "Activity log for Remote River '{}' is enabled. Search index name '{}', document type for index updates '{}'.", riverName.getName(), activityLogIndexName, activityLogTypeName); } }
public Object extractValue(String path) { return XContentMapValues.extractValue(path, loadSourceIfNeeded()); }
public Object filter(String[] includes, String[] excludes) { return XContentMapValues.filter(loadSourceIfNeeded(), includes, excludes); }
/** * Returns the values associated with the path. Those are "low" level values, and it can handle * path expression where an array/list is navigated within. */ public List<Object> extractRawValues(String path) { return XContentMapValues.extractRawValues(path, loadSourceIfNeeded()); }
private InternalSearchHit createNestedSearchHit( SearchContext context, int nestedTopDocId, int nestedSubDocId, int rootSubDocId, List<String> extractFieldNames, boolean loadAllStored, Set<String> fieldNames, LeafReaderContext subReaderContext) throws IOException { // Also if highlighting is requested on nested documents we need to fetch the _source from the // root document, // otherwise highlighting will attempt to fetch the _source from the nested doc, which will // fail, // because the entire _source is only stored with the root document. final FieldsVisitor rootFieldsVisitor = new FieldsVisitor( context.sourceRequested() || extractFieldNames != null || context.highlight() != null); loadStoredFields(context, subReaderContext, rootFieldsVisitor, rootSubDocId); rootFieldsVisitor.postProcess(context.mapperService()); Map<String, SearchHitField> searchFields = getSearchFields(context, nestedSubDocId, loadAllStored, fieldNames, subReaderContext); DocumentMapper documentMapper = context.mapperService().documentMapper(rootFieldsVisitor.uid().type()); SourceLookup sourceLookup = context.lookup().source(); sourceLookup.setSegmentAndDocument(subReaderContext, nestedSubDocId); ObjectMapper nestedObjectMapper = documentMapper.findNestedObjectMapper(nestedSubDocId, context, subReaderContext); assert nestedObjectMapper != null; InternalSearchHit.InternalNestedIdentity nestedIdentity = getInternalNestedIdentity( context, nestedSubDocId, subReaderContext, documentMapper, nestedObjectMapper); BytesReference source = rootFieldsVisitor.source(); if (source != null) { Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(source, true); Map<String, Object> sourceAsMap = tuple.v2(); List<Map<String, Object>> nestedParsedSource; SearchHit.NestedIdentity nested = nestedIdentity; do { Object extractedValue = XContentMapValues.extractValue(nested.getField().string(), sourceAsMap); if (extractedValue == null) { // The nested objects may not exist in the _source, because it was filtered because of // _source filtering break; } else if (extractedValue instanceof List) { // nested field has an array value in the _source nestedParsedSource = (List<Map<String, Object>>) extractedValue; } else if (extractedValue instanceof Map) { // nested field has an object value in the _source. This just means the nested field has // just one inner object, which is valid, but uncommon. nestedParsedSource = ImmutableList.of((Map<String, Object>) extractedValue); } else { throw new IllegalStateException("extracted source isn't an object or an array"); } sourceAsMap = nestedParsedSource.get(nested.getOffset()); nested = nested.getChild(); } while (nested != null); context.lookup().source().setSource(sourceAsMap); XContentType contentType = tuple.v1(); BytesReference nestedSource = contentBuilder(contentType).map(sourceAsMap).bytes(); context.lookup().source().setSource(nestedSource); context.lookup().source().setSourceContentType(contentType); } InternalSearchHit searchHit = new InternalSearchHit( nestedTopDocId, rootFieldsVisitor.uid().id(), documentMapper.typeText(), nestedIdentity, searchFields); if (extractFieldNames != null) { for (String extractFieldName : extractFieldNames) { List<Object> values = context.lookup().source().extractRawValues(extractFieldName); if (!values.isEmpty()) { if (searchHit.fieldsOrNull() == null) { searchHit.fields(new HashMap<String, SearchHitField>(2)); } SearchHitField hitField = searchHit.fields().get(extractFieldName); if (hitField == null) { hitField = new InternalSearchHitField(extractFieldName, new ArrayList<>(2)); searchHit.fields().put(extractFieldName, hitField); } for (Object value : values) { hitField.values().add(value); } } } } return searchHit; }
@SuppressWarnings({"unchecked"}) @Inject public RabbitmqRiver( RiverName riverName, RiverSettings settings, Client client, ScriptService scriptService) { super(riverName, settings); this.client = client; if (settings.settings().containsKey("rabbitmq")) { Map<String, Object> rabbitSettings = (Map<String, Object>) settings.settings().get("rabbitmq"); if (rabbitSettings.containsKey("addresses")) { List<Address> addresses = new ArrayList<Address>(); for (Map<String, Object> address : (List<Map<String, Object>>) rabbitSettings.get("addresses")) { addresses.add( new Address( XContentMapValues.nodeStringValue(address.get("host"), "localhost"), XContentMapValues.nodeIntegerValue(address.get("port"), AMQP.PROTOCOL.PORT))); } rabbitAddresses = addresses.toArray(new Address[addresses.size()]); } else { String rabbitHost = XContentMapValues.nodeStringValue(rabbitSettings.get("host"), "localhost"); int rabbitPort = XContentMapValues.nodeIntegerValue(rabbitSettings.get("port"), AMQP.PROTOCOL.PORT); rabbitAddresses = new Address[] {new Address(rabbitHost, rabbitPort)}; } rabbitUser = XContentMapValues.nodeStringValue(rabbitSettings.get("user"), "guest"); rabbitPassword = XContentMapValues.nodeStringValue(rabbitSettings.get("pass"), "guest"); rabbitVhost = XContentMapValues.nodeStringValue(rabbitSettings.get("vhost"), "/"); rabbitQueue = XContentMapValues.nodeStringValue(rabbitSettings.get("queue"), "elasticsearch"); rabbitExchange = XContentMapValues.nodeStringValue(rabbitSettings.get("exchange"), "elasticsearch"); rabbitRoutingKey = XContentMapValues.nodeStringValue(rabbitSettings.get("routing_key"), "elasticsearch"); rabbitExchangeDeclare = XContentMapValues.nodeBooleanValue(rabbitSettings.get("exchange_declare"), true); if (rabbitExchangeDeclare) { rabbitExchangeType = XContentMapValues.nodeStringValue(rabbitSettings.get("exchange_type"), "direct"); rabbitExchangeDurable = XContentMapValues.nodeBooleanValue(rabbitSettings.get("exchange_durable"), true); } else { rabbitExchangeType = "direct"; rabbitExchangeDurable = true; } rabbitQueueDeclare = XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_declare"), true); if (rabbitQueueDeclare) { rabbitQueueDurable = XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_durable"), true); rabbitQueueAutoDelete = XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_auto_delete"), false); if (rabbitSettings.containsKey("args")) { rabbitQueueArgs = (Map<String, Object>) rabbitSettings.get("args"); } } else { rabbitQueueDurable = true; rabbitQueueAutoDelete = false; } rabbitQueueBind = XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_bind"), true); rabbitQosPrefetchSize = XContentMapValues.nodeIntegerValue(rabbitSettings.get("qos_prefetch_size"), 0); rabbitQosPrefetchCount = XContentMapValues.nodeIntegerValue(rabbitSettings.get("qos_prefetch_count"), 10); rabbitHeartbeat = TimeValue.parseTimeValue( XContentMapValues.nodeStringValue(rabbitSettings.get("heartbeat"), "30m"), TimeValue.timeValueMinutes(30)); } else { rabbitAddresses = new Address[] {new Address("localhost", AMQP.PROTOCOL.PORT)}; rabbitUser = "******"; rabbitPassword = "******"; rabbitVhost = "/"; rabbitQueue = "elasticsearch"; rabbitQueueAutoDelete = false; rabbitQueueDurable = true; rabbitExchange = "elasticsearch"; rabbitExchangeType = "direct"; rabbitExchangeDurable = true; rabbitRoutingKey = "elasticsearch"; rabbitExchangeDeclare = true; rabbitQueueDeclare = true; rabbitQueueBind = true; rabbitQosPrefetchSize = 0; rabbitQosPrefetchCount = 10; rabbitHeartbeat = TimeValue.timeValueMinutes(30); } if (settings.settings().containsKey("index")) { Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index"); bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size"), 100); if (indexSettings.containsKey("bulk_timeout")) { bulkTimeout = TimeValue.parseTimeValue( XContentMapValues.nodeStringValue(indexSettings.get("bulk_timeout"), "10ms"), TimeValue.timeValueMillis(10)); } else { bulkTimeout = TimeValue.timeValueMillis(10); } ordered = XContentMapValues.nodeBooleanValue(indexSettings.get("ordered"), false); } else { bulkSize = 100; bulkTimeout = TimeValue.timeValueMillis(10); ordered = false; } if (settings.settings().containsKey("bulk_script_filter")) { Map<String, Object> scriptSettings = (Map<String, Object>) settings.settings().get("bulk_script_filter"); if (scriptSettings.containsKey("script")) { String scriptLang = "native"; if (scriptSettings.containsKey("script_lang")) { scriptLang = scriptSettings.get("script_lang").toString(); } Map<String, Object> scriptParams = null; if (scriptSettings.containsKey("script_params")) { scriptParams = (Map<String, Object>) scriptSettings.get("script_params"); } else { scriptParams = Maps.newHashMap(); } bulkScript = scriptService.executable( scriptLang, scriptSettings.get("script").toString(), scriptParams); } else { bulkScript = null; } } else { bulkScript = null; } if (settings.settings().containsKey("script_filter")) { Map<String, Object> scriptSettings = (Map<String, Object>) settings.settings().get("script_filter"); if (scriptSettings.containsKey("script")) { String scriptLang = "mvel"; if (scriptSettings.containsKey("script_lang")) { scriptLang = scriptSettings.get("script_lang").toString(); } Map<String, Object> scriptParams = null; if (scriptSettings.containsKey("script_params")) { scriptParams = (Map<String, Object>) scriptSettings.get("script_params"); } else { scriptParams = Maps.newHashMap(); } script = scriptService.executable( scriptLang, scriptSettings.get("script").toString(), scriptParams); } else { script = null; } } else { script = null; } }
@SuppressWarnings("unchecked") public static synchronized MongoDBRiverDefinition parseSettings( String riverName, String riverIndexName, RiverSettings settings, ScriptService scriptService) { logger.trace("Parse river settings for {}", riverName); Preconditions.checkNotNull(riverName, "No riverName specified"); Preconditions.checkNotNull(riverIndexName, "No riverIndexName specified"); Preconditions.checkNotNull(settings, "No settings specified"); Builder builder = new Builder(); builder.riverName(riverName); builder.riverIndexName(riverIndexName); List<ServerAddress> mongoServers = new ArrayList<ServerAddress>(); String mongoHost; int mongoPort; if (settings.settings().containsKey(MongoDBRiver.TYPE)) { Map<String, Object> mongoSettings = (Map<String, Object>) settings.settings().get(MongoDBRiver.TYPE); if (mongoSettings.containsKey(SERVERS_FIELD)) { Object mongoServersSettings = mongoSettings.get(SERVERS_FIELD); logger.trace("mongoServersSettings: " + mongoServersSettings); boolean array = XContentMapValues.isArray(mongoServersSettings); if (array) { ArrayList<Map<String, Object>> feeds = (ArrayList<Map<String, Object>>) mongoServersSettings; for (Map<String, Object> feed : feeds) { mongoHost = XContentMapValues.nodeStringValue(feed.get(HOST_FIELD), null); mongoPort = XContentMapValues.nodeIntegerValue(feed.get(PORT_FIELD), DEFAULT_DB_PORT); logger.trace("Server: " + mongoHost + " - " + mongoPort); try { mongoServers.add(new ServerAddress(mongoHost, mongoPort)); } catch (UnknownHostException uhEx) { logger.warn("Cannot add mongo server {}:{}", uhEx, mongoHost, mongoPort); } } } } else { mongoHost = XContentMapValues.nodeStringValue(mongoSettings.get(HOST_FIELD), DEFAULT_DB_HOST); mongoPort = XContentMapValues.nodeIntegerValue(mongoSettings.get(PORT_FIELD), DEFAULT_DB_PORT); try { mongoServers.add(new ServerAddress(mongoHost, mongoPort)); } catch (UnknownHostException uhEx) { logger.warn("Cannot add mongo server {}:{}", uhEx, mongoHost, mongoPort); } } builder.mongoServers(mongoServers); MongoClientOptions.Builder mongoClientOptionsBuilder = MongoClientOptions.builder().socketKeepAlive(true); // MongoDB options if (mongoSettings.containsKey(OPTIONS_FIELD)) { Map<String, Object> mongoOptionsSettings = (Map<String, Object>) mongoSettings.get(OPTIONS_FIELD); logger.trace("mongoOptionsSettings: " + mongoOptionsSettings); builder.mongoSecondaryReadPreference( XContentMapValues.nodeBooleanValue( mongoOptionsSettings.get(SECONDARY_READ_PREFERENCE_FIELD), false)); builder.connectTimeout( XContentMapValues.nodeIntegerValue( mongoOptionsSettings.get(CONNECT_TIMEOUT), DEFAULT_CONNECT_TIMEOUT)); builder.socketTimeout( XContentMapValues.nodeIntegerValue( mongoOptionsSettings.get(SOCKET_TIMEOUT), DEFAULT_SOCKET_TIMEOUT)); builder.dropCollection( XContentMapValues.nodeBooleanValue( mongoOptionsSettings.get(DROP_COLLECTION_FIELD), false)); String isMongos = XContentMapValues.nodeStringValue(mongoOptionsSettings.get(IS_MONGOS_FIELD), null); if (isMongos != null) { builder.isMongos(Boolean.valueOf(isMongos)); } builder.mongoUseSSL( XContentMapValues.nodeBooleanValue( mongoOptionsSettings.get(SSL_CONNECTION_FIELD), false)); builder.mongoSSLVerifyCertificate( XContentMapValues.nodeBooleanValue( mongoOptionsSettings.get(SSL_VERIFY_CERT_FIELD), true)); builder.advancedTransformation( XContentMapValues.nodeBooleanValue( mongoOptionsSettings.get(ADVANCED_TRANSFORMATION_FIELD), false)); builder.skipInitialImport( XContentMapValues.nodeBooleanValue( mongoOptionsSettings.get(SKIP_INITIAL_IMPORT_FIELD), false)); builder.connectionsPerHost( XContentMapValues.nodeIntegerValue( mongoOptionsSettings.get(CONNECTIONS_PER_HOST), DEFAULT_CONNECTIONS_PER_HOST)); builder.threadsAllowedToBlockForConnectionMultiplier( XContentMapValues.nodeIntegerValue( mongoOptionsSettings.get(THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER), DEFAULT_THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER)); mongoClientOptionsBuilder .connectTimeout(builder.connectTimeout) .socketTimeout(builder.socketTimeout) .connectionsPerHost(builder.connectionsPerHost) .threadsAllowedToBlockForConnectionMultiplier( builder.threadsAllowedToBlockForConnectionMultiplier); if (builder.mongoSecondaryReadPreference) { mongoClientOptionsBuilder.readPreference(ReadPreference.secondaryPreferred()); } if (builder.mongoUseSSL) { mongoClientOptionsBuilder.socketFactory(getSSLSocketFactory()); } if (mongoOptionsSettings.containsKey(PARENT_TYPES_FIELD)) { Set<String> parentTypes = new HashSet<String>(); Object parentTypesSettings = mongoOptionsSettings.get(PARENT_TYPES_FIELD); logger.trace("parentTypesSettings: " + parentTypesSettings); boolean array = XContentMapValues.isArray(parentTypesSettings); if (array) { ArrayList<String> fields = (ArrayList<String>) parentTypesSettings; for (String field : fields) { logger.trace("Field: " + field); parentTypes.add(field); } } builder.parentTypes(parentTypes); } if (mongoOptionsSettings.containsKey(STORE_STATISTICS_FIELD)) { Object storeStatistics = mongoOptionsSettings.get(STORE_STATISTICS_FIELD); boolean object = XContentMapValues.isObject(storeStatistics); if (object) { Map<String, Object> storeStatisticsSettings = (Map<String, Object>) storeStatistics; builder.storeStatistics(true); builder.statisticsIndexName( XContentMapValues.nodeStringValue( storeStatisticsSettings.get(INDEX_OBJECT), riverName + "-stats")); builder.statisticsTypeName( XContentMapValues.nodeStringValue( storeStatisticsSettings.get(TYPE_FIELD), "stats")); } else { builder.storeStatistics(XContentMapValues.nodeBooleanValue(storeStatistics, false)); if (builder.storeStatistics) { builder.statisticsIndexName(riverName + "-stats"); builder.statisticsTypeName("stats"); } } } // builder.storeStatistics(XContentMapValues.nodeBooleanValue(mongoOptionsSettings.get(STORE_STATISTICS_FIELD), // false)); builder.importAllCollections( XContentMapValues.nodeBooleanValue( mongoOptionsSettings.get(IMPORT_ALL_COLLECTIONS_FIELD), false)); builder.disableIndexRefresh( XContentMapValues.nodeBooleanValue( mongoOptionsSettings.get(DISABLE_INDEX_REFRESH_FIELD), false)); builder.includeCollection( XContentMapValues.nodeStringValue( mongoOptionsSettings.get(INCLUDE_COLLECTION_FIELD), "")); if (mongoOptionsSettings.containsKey(INCLUDE_FIELDS_FIELD)) { Set<String> includeFields = new HashSet<String>(); Object includeFieldsSettings = mongoOptionsSettings.get(INCLUDE_FIELDS_FIELD); logger.trace("includeFieldsSettings: " + includeFieldsSettings); boolean array = XContentMapValues.isArray(includeFieldsSettings); if (array) { ArrayList<String> fields = (ArrayList<String>) includeFieldsSettings; for (String field : fields) { logger.trace("Field: " + field); includeFields.add(field); } } if (!includeFields.contains(MongoDBRiver.MONGODB_ID_FIELD)) { includeFields.add(MongoDBRiver.MONGODB_ID_FIELD); } builder.includeFields(includeFields); } else if (mongoOptionsSettings.containsKey(EXCLUDE_FIELDS_FIELD)) { Set<String> excludeFields = new HashSet<String>(); Object excludeFieldsSettings = mongoOptionsSettings.get(EXCLUDE_FIELDS_FIELD); logger.trace("excludeFieldsSettings: " + excludeFieldsSettings); boolean array = XContentMapValues.isArray(excludeFieldsSettings); if (array) { ArrayList<String> fields = (ArrayList<String>) excludeFieldsSettings; for (String field : fields) { logger.trace("Field: " + field); excludeFields.add(field); } } builder.excludeFields(excludeFields); } if (mongoOptionsSettings.containsKey(INITIAL_TIMESTAMP_FIELD)) { BSONTimestamp timeStamp = null; try { Map<String, Object> initalTimestampSettings = (Map<String, Object>) mongoOptionsSettings.get(INITIAL_TIMESTAMP_FIELD); String scriptType = "js"; if (initalTimestampSettings.containsKey(INITIAL_TIMESTAMP_SCRIPT_TYPE_FIELD)) { scriptType = initalTimestampSettings.get(INITIAL_TIMESTAMP_SCRIPT_TYPE_FIELD).toString(); } if (initalTimestampSettings.containsKey(INITIAL_TIMESTAMP_SCRIPT_FIELD)) { ExecutableScript scriptExecutable = scriptService.executable( scriptType, initalTimestampSettings.get(INITIAL_TIMESTAMP_SCRIPT_FIELD).toString(), ScriptService.ScriptType.INLINE, Maps.newHashMap()); Object ctx = scriptExecutable.run(); logger.trace("initialTimestamp script returned: {}", ctx); if (ctx != null) { long timestamp = Long.parseLong(ctx.toString()); timeStamp = new BSONTimestamp((int) (new Date(timestamp).getTime() / 1000), 1); } } } catch (Throwable t) { logger.error("Could not set initial timestamp", t); } finally { builder.initialTimestamp(timeStamp); } } } builder.mongoClientOptions(mongoClientOptionsBuilder.build()); // Credentials if (mongoSettings.containsKey(CREDENTIALS_FIELD)) { String dbCredential; String mau = ""; String map = ""; String maad = ""; String mlu = ""; String mlp = ""; String mlad = ""; // String mdu = ""; // String mdp = ""; Object mongoCredentialsSettings = mongoSettings.get(CREDENTIALS_FIELD); boolean array = XContentMapValues.isArray(mongoCredentialsSettings); if (array) { ArrayList<Map<String, Object>> credentials = (ArrayList<Map<String, Object>>) mongoCredentialsSettings; for (Map<String, Object> credential : credentials) { dbCredential = XContentMapValues.nodeStringValue(credential.get(DB_FIELD), null); if (ADMIN_DB_FIELD.equals(dbCredential)) { mau = XContentMapValues.nodeStringValue(credential.get(USER_FIELD), null); map = XContentMapValues.nodeStringValue(credential.get(PASSWORD_FIELD), null); maad = XContentMapValues.nodeStringValue(credential.get(AUTH_FIELD), null); } else if (LOCAL_DB_FIELD.equals(dbCredential)) { mlu = XContentMapValues.nodeStringValue(credential.get(USER_FIELD), null); mlp = XContentMapValues.nodeStringValue(credential.get(PASSWORD_FIELD), null); mlad = XContentMapValues.nodeStringValue(credential.get(AUTH_FIELD), null); // } else { // mdu = XContentMapValues.nodeStringValue( // credential.get(USER_FIELD), null); // mdp = XContentMapValues.nodeStringValue( // credential.get(PASSWORD_FIELD), null); } } } builder.mongoAdminUser(mau); builder.mongoAdminPassword(map); builder.mongoAdminAuthDatabase(maad); builder.mongoLocalUser(mlu); builder.mongoLocalPassword(mlp); builder.mongoLocalAuthDatabase(mlad); // mongoDbUser = mdu; // mongoDbPassword = mdp; } builder.mongoDb(XContentMapValues.nodeStringValue(mongoSettings.get(DB_FIELD), riverName)); builder.mongoCollection( XContentMapValues.nodeStringValue(mongoSettings.get(COLLECTION_FIELD), riverName)); builder.mongoGridFS( XContentMapValues.nodeBooleanValue(mongoSettings.get(GRIDFS_FIELD), false)); if (mongoSettings.containsKey(FILTER_FIELD)) { String filter = XContentMapValues.nodeStringValue(mongoSettings.get(FILTER_FIELD), ""); filter = removePrefix("o.", filter); builder.mongoCollectionFilter(convertToBasicDBObject(filter)); // DBObject bsonObject = (DBObject) JSON.parse(filter); // builder.mongoOplogFilter(convertToBasicDBObject(addPrefix("o.", // filter))); builder.mongoOplogFilter(convertToBasicDBObject(removePrefix("o.", filter))); // } else { // builder.mongoOplogFilter(""); } if (mongoSettings.containsKey(SCRIPT_FIELD)) { String scriptType = "js"; builder.script(mongoSettings.get(SCRIPT_FIELD).toString()); if (mongoSettings.containsKey("scriptType")) { scriptType = mongoSettings.get("scriptType").toString(); } else if (mongoSettings.containsKey(SCRIPT_TYPE_FIELD)) { scriptType = mongoSettings.get(SCRIPT_TYPE_FIELD).toString(); } builder.scriptType(scriptType); } } else { mongoHost = DEFAULT_DB_HOST; mongoPort = DEFAULT_DB_PORT; try { mongoServers.add(new ServerAddress(mongoHost, mongoPort)); builder.mongoServers(mongoServers); } catch (UnknownHostException e) { e.printStackTrace(); } builder.mongoDb(riverName); builder.mongoCollection(riverName); } if (settings.settings().containsKey(INDEX_OBJECT)) { Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get(INDEX_OBJECT); builder.indexName( XContentMapValues.nodeStringValue(indexSettings.get(NAME_FIELD), builder.mongoDb)); builder.typeName( XContentMapValues.nodeStringValue(indexSettings.get(TYPE_FIELD), builder.mongoDb)); Bulk.Builder bulkBuilder = new Bulk.Builder(); if (indexSettings.containsKey(BULK_FIELD)) { Map<String, Object> bulkSettings = (Map<String, Object>) indexSettings.get(BULK_FIELD); int bulkActions = XContentMapValues.nodeIntegerValue( bulkSettings.get(ACTIONS_FIELD), DEFAULT_BULK_ACTIONS); bulkBuilder.bulkActions(bulkActions); String size = XContentMapValues.nodeStringValue( bulkSettings.get(SIZE_FIELD), DEFAULT_BULK_SIZE.toString()); bulkBuilder.bulkSize(ByteSizeValue.parseBytesSizeValue(size)); bulkBuilder.concurrentRequests( XContentMapValues.nodeIntegerValue( bulkSettings.get(CONCURRENT_REQUESTS_FIELD), EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY))); bulkBuilder.flushInterval( XContentMapValues.nodeTimeValue( bulkSettings.get(FLUSH_INTERVAL_FIELD), DEFAULT_FLUSH_INTERVAL)); builder.throttleSize( XContentMapValues.nodeIntegerValue( indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5)); } else { int bulkActions = XContentMapValues.nodeIntegerValue( indexSettings.get(BULK_SIZE_FIELD), DEFAULT_BULK_ACTIONS); bulkBuilder.bulkActions(bulkActions); bulkBuilder.bulkSize(DEFAULT_BULK_SIZE); bulkBuilder.flushInterval( XContentMapValues.nodeTimeValue( indexSettings.get(BULK_TIMEOUT_FIELD), DEFAULT_FLUSH_INTERVAL)); bulkBuilder.concurrentRequests( XContentMapValues.nodeIntegerValue( indexSettings.get(CONCURRENT_BULK_REQUESTS_FIELD), EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY))); builder.throttleSize( XContentMapValues.nodeIntegerValue( indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5)); } builder.bulk(bulkBuilder.build()); } else { builder.indexName(builder.mongoDb); builder.typeName(builder.mongoDb); builder.bulk(new Bulk.Builder().build()); } return builder.build(); }
@Override public Mapper.Builder parse( String fieldName, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException { StringFieldMapper.Builder builder = new StringFieldMapper.Builder(fieldName); // hack for the fact that string can't just accept true/false for // the index property and still accepts no/not_analyzed/analyzed final Object index = node.remove("index"); if (index != null) { final String normalizedIndex = Strings.toUnderscoreCase(index.toString()); switch (normalizedIndex) { case "analyzed": builder.tokenized(true); node.put("index", true); break; case "not_analyzed": builder.tokenized(false); node.put("index", true); break; case "no": node.put("index", false); break; default: throw new IllegalArgumentException( "Can't parse [index] value [" + index + "] for field [" + fieldName + "], expected [true], [false], [no], [not_analyzed] or [analyzed]"); } } parseTextField(builder, fieldName, node, parserContext); for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext(); ) { Map.Entry<String, Object> entry = iterator.next(); String propName = Strings.toUnderscoreCase(entry.getKey()); Object propNode = entry.getValue(); if (propName.equals("null_value")) { if (propNode == null) { throw new MapperParsingException("Property [null_value] cannot be null."); } builder.nullValue(propNode.toString()); iterator.remove(); } else if (propName.equals("position_increment_gap")) { int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1); if (newPositionIncrementGap < 0) { throw new MapperParsingException("positions_increment_gap less than 0 aren't allowed."); } builder.positionIncrementGap(newPositionIncrementGap); // we need to update to actual analyzers if they are not set in this case... // so we can inject the position increment gap... if (builder.fieldType().indexAnalyzer() == null) { builder .fieldType() .setIndexAnalyzer(parserContext.analysisService().defaultIndexAnalyzer()); } if (builder.fieldType().searchAnalyzer() == null) { builder .fieldType() .setSearchAnalyzer(parserContext.analysisService().defaultSearchAnalyzer()); } if (builder.fieldType().searchQuoteAnalyzer() == null) { builder .fieldType() .setSearchQuoteAnalyzer( parserContext.analysisService().defaultSearchQuoteAnalyzer()); } iterator.remove(); } else if (propName.equals("ignore_above")) { builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1)); iterator.remove(); } else if (parseMultiField(builder, fieldName, parserContext, propName, propNode)) { iterator.remove(); } } return builder; }
public void testFilter() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder() .startObject() .field("test1", "value1") .field("test2", "value2") .field("something_else", "value3") .endObject(); Map<String, Object> source; try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) { source = parser.map(); } Map<String, Object> filter = XContentMapValues.filter(source, new String[] {"test1"}, Strings.EMPTY_ARRAY); assertThat(filter.size(), equalTo(1)); assertThat(filter.get("test1").toString(), equalTo("value1")); filter = XContentMapValues.filter(source, new String[] {"test*"}, Strings.EMPTY_ARRAY); assertThat(filter.size(), equalTo(2)); assertThat(filter.get("test1").toString(), equalTo("value1")); assertThat(filter.get("test2").toString(), equalTo("value2")); filter = XContentMapValues.filter(source, Strings.EMPTY_ARRAY, new String[] {"test1"}); assertThat(filter.size(), equalTo(2)); assertThat(filter.get("test2").toString(), equalTo("value2")); assertThat(filter.get("something_else").toString(), equalTo("value3")); // more complex object... builder = XContentFactory.jsonBuilder() .startObject() .startObject("path1") .startArray("path2") .startObject() .field("test", "value1") .endObject() .startObject() .field("test", "value2") .endObject() .endArray() .endObject() .field("test1", "value1") .endObject(); try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) { source = parser.map(); } filter = XContentMapValues.filter(source, new String[] {"path1"}, Strings.EMPTY_ARRAY); assertThat(filter.size(), equalTo(1)); filter = XContentMapValues.filter(source, new String[] {"path1*"}, Strings.EMPTY_ARRAY); assertThat(filter.get("path1"), equalTo(source.get("path1"))); assertThat(filter.containsKey("test1"), equalTo(false)); filter = XContentMapValues.filter(source, new String[] {"test1*"}, Strings.EMPTY_ARRAY); assertThat(filter.get("test1"), equalTo(source.get("test1"))); assertThat(filter.containsKey("path1"), equalTo(false)); filter = XContentMapValues.filter(source, new String[] {"path1.path2.*"}, Strings.EMPTY_ARRAY); assertThat(filter.get("path1"), equalTo(source.get("path1"))); assertThat(filter.containsKey("test1"), equalTo(false)); }