@Test
  public void testIpMultiField() throws Exception {
    assertAcked(
        client()
            .admin()
            .indices()
            .prepareCreate("my-index")
            .addMapping("my-type", createMappingSource("ip")));

    GetMappingsResponse getMappingsResponse =
        client().admin().indices().prepareGetMappings("my-index").get();
    MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type");
    assertThat(mappingMetaData, not(nullValue()));
    Map<String, Object> mappingSource = mappingMetaData.sourceAsMap();
    Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource));
    assertThat(aField.size(), equalTo(2));
    assertThat(aField.get("type").toString(), equalTo("ip"));
    assertThat(aField.get("fields"), notNullValue());

    Map bField = ((Map) XContentMapValues.extractValue("properties.a.fields.b", mappingSource));
    assertThat(bField.size(), equalTo(2));
    assertThat(bField.get("type").toString(), equalTo("string"));
    assertThat(bField.get("index").toString(), equalTo("not_analyzed"));

    client()
        .prepareIndex("my-index", "my-type", "1")
        .setSource("a", "127.0.0.1")
        .setRefresh(true)
        .get();
    CountResponse countResponse =
        client().prepareCount("my-index").setQuery(matchQuery("a.b", "127.0.0.1")).get();
    assertThat(countResponse.getCount(), equalTo(1l));
  }
 @Override
 public HashMap<String, String> loadKey(String username) {
   logger.info("loading password for username {}", username);
   HashMap<String, String> ret = new HashMap<>();
   String riverIndexName = getRiverIndexName();
   refreshSearchIndex(riverIndexName);
   GetResponse resp =
       client.prepareGet(riverIndexName, riverName().name(), "_pwd").execute().actionGet();
   if (resp.isExists()) {
     if (logger.isDebugEnabled()) {
       logger.debug("Password document: {}", resp.getSourceAsString());
     }
     Map<String, Object> newset = resp.getSource();
     Set<String> keys = newset.keySet();
     for (String s : keys) {
       logger.info(
           "Added key {} with a value of {}",
           s,
           XContentMapValues.nodeStringValue(newset.get(s), null));
       ret.put(s, XContentMapValues.nodeStringValue(newset.get(s), null));
     }
   }
   if (ret.isEmpty()) {
     return null;
   }
   return ret;
 }
  @SuppressWarnings("unchecked")
  public void testNestedFiltering() {
    Map<String, Object> map = new HashMap<>();
    map.put("field", "value");
    map.put(
        "array",
        Arrays.asList(
            1,
            new HashMap<String, Object>() {
              {
                put("nested", 2);
                put("nested_2", 3);
              }
            }));
    Map<String, Object> falteredMap =
        XContentMapValues.filter(map, new String[] {"array.nested"}, Strings.EMPTY_ARRAY);
    assertThat(falteredMap.size(), equalTo(1));

    // Selecting members of objects within arrays (ex. [ 1, { nested: "value"} ])  always returns
    // all values in the array (1 in the ex)
    // this is expected behavior as this types of objects are not supported in ES
    assertThat((Integer) ((List) falteredMap.get("array")).get(0), equalTo(1));
    assertThat(((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).size(), equalTo(1));
    assertThat(
        (Integer) ((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).get("nested"),
        equalTo(2));

    falteredMap = XContentMapValues.filter(map, new String[] {"array.*"}, Strings.EMPTY_ARRAY);
    assertThat(falteredMap.size(), equalTo(1));
    assertThat((Integer) ((List) falteredMap.get("array")).get(0), equalTo(1));
    assertThat(((Map<String, Object>) ((List) falteredMap.get("array")).get(1)).size(), equalTo(2));

    map.clear();
    map.put("field", "value");
    map.put(
        "obj",
        new HashMap<String, Object>() {
          {
            put("field", "value");
            put("field2", "value2");
          }
        });
    falteredMap = XContentMapValues.filter(map, new String[] {"obj.field"}, Strings.EMPTY_ARRAY);
    assertThat(falteredMap.size(), equalTo(1));
    assertThat(((Map<String, Object>) falteredMap.get("obj")).size(), equalTo(1));
    assertThat(
        (String) ((Map<String, Object>) falteredMap.get("obj")).get("field"), equalTo("value"));

    falteredMap = XContentMapValues.filter(map, new String[] {"obj.*"}, Strings.EMPTY_ARRAY);
    assertThat(falteredMap.size(), equalTo(1));
    assertThat(((Map<String, Object>) falteredMap.get("obj")).size(), equalTo(2));
    assertThat(
        (String) ((Map<String, Object>) falteredMap.get("obj")).get("field"), equalTo("value"));
    assertThat(
        (String) ((Map<String, Object>) falteredMap.get("obj")).get("field2"), equalTo("value2"));
  }
  public void testExtractRawValue() throws Exception {
    XContentBuilder builder =
        XContentFactory.jsonBuilder().startObject().field("test", "value").endObject();

    Map<String, Object> map;
    try (XContentParser parser =
        XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
      map = parser.map();
    }
    assertThat(XContentMapValues.extractRawValues("test", map).get(0).toString(), equalTo("value"));

    builder = XContentFactory.jsonBuilder().startObject().field("test.me", "value").endObject();

    try (XContentParser parser =
        XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
      map = parser.map();
    }
    assertThat(
        XContentMapValues.extractRawValues("test.me", map).get(0).toString(), equalTo("value"));

    builder =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("path1")
            .startObject("path2")
            .field("test", "value")
            .endObject()
            .endObject()
            .endObject();

    try (XContentParser parser =
        XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
      map = parser.map();
    }
    assertThat(
        XContentMapValues.extractRawValues("path1.path2.test", map).get(0).toString(),
        equalTo("value"));

    builder =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("path1.xxx")
            .startObject("path2.yyy")
            .field("test", "value")
            .endObject()
            .endObject()
            .endObject();

    try (XContentParser parser =
        XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
      map = parser.map();
    }
    assertThat(
        XContentMapValues.extractRawValues("path1.xxx.path2.yyy.test", map).get(0).toString(),
        equalTo("value"));
  }
  public void testSupplementaryCharactersInPaths() {
    Map<String, Object> map = new HashMap<>();
    map.put("搜索", 2);
    map.put("指数", 3);

    assertEquals(
        Collections.singletonMap("搜索", 2),
        XContentMapValues.filter(map, new String[] {"搜索"}, new String[0]));
    assertEquals(
        Collections.singletonMap("指数", 3),
        XContentMapValues.filter(map, new String[0], new String[] {"搜索"}));
  }
  @SuppressWarnings("unchecked")
  public void testCompleteObjectFiltering() {
    Map<String, Object> map = new HashMap<>();
    map.put("field", "value");
    map.put(
        "obj",
        new HashMap<String, Object>() {
          {
            put("field", "value");
            put("field2", "value2");
          }
        });
    map.put(
        "array",
        Arrays.asList(
            1,
            new HashMap<String, Object>() {
              {
                put("field", "value");
                put("field2", "value2");
              }
            }));

    Map<String, Object> filteredMap =
        XContentMapValues.filter(map, new String[] {"obj"}, Strings.EMPTY_ARRAY);
    assertThat(filteredMap.size(), equalTo(1));
    assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(2));
    assertThat(
        ((Map<String, Object>) filteredMap.get("obj")).get("field").toString(), equalTo("value"));
    assertThat(
        ((Map<String, Object>) filteredMap.get("obj")).get("field2").toString(), equalTo("value2"));

    filteredMap = XContentMapValues.filter(map, new String[] {"obj"}, new String[] {"*.field2"});
    assertThat(filteredMap.size(), equalTo(1));
    assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(1));
    assertThat(
        ((Map<String, Object>) filteredMap.get("obj")).get("field").toString(), equalTo("value"));

    filteredMap = XContentMapValues.filter(map, new String[] {"array"}, new String[] {});
    assertThat(filteredMap.size(), equalTo(1));
    assertThat(((List) filteredMap.get("array")).size(), equalTo(2));
    assertThat((Integer) ((List) filteredMap.get("array")).get(0), equalTo(1));
    assertThat(((Map<String, Object>) ((List) filteredMap.get("array")).get(1)).size(), equalTo(2));

    filteredMap = XContentMapValues.filter(map, new String[] {"array"}, new String[] {"*.field2"});
    assertThat(filteredMap.size(), equalTo(1));
    assertThat(((List) filteredMap.get("array")).size(), equalTo(2));
    assertThat((Integer) ((List) filteredMap.get("array")).get(0), equalTo(1));
    assertThat(((Map<String, Object>) ((List) filteredMap.get("array")).get(1)).size(), equalTo(1));
    assertThat(
        ((Map<String, Object>) ((List) filteredMap.get("array")).get(1)).get("field").toString(),
        equalTo("value"));
  }
  public void testSharedPrefixes() {
    Map<String, Object> map = new HashMap<>();
    map.put("foobar", 2);
    map.put("foobaz", 3);

    assertEquals(
        Collections.singletonMap("foobar", 2),
        XContentMapValues.filter(map, new String[] {"foobar"}, new String[0]));
    assertEquals(
        Collections.singletonMap("foobaz", 3),
        XContentMapValues.filter(map, new String[0], new String[] {"foobar"}));
  }
 @Override
 public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
     throws MapperParsingException {
   StringFieldMapper.Builder builder = stringField(name);
   parseField(builder, name, node, parserContext);
   for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
       iterator.hasNext(); ) {
     Map.Entry<String, Object> entry = iterator.next();
     String propName = Strings.toUnderscoreCase(entry.getKey());
     Object propNode = entry.getValue();
     if (propName.equals("null_value")) {
       if (propNode == null) {
         throw new MapperParsingException("Property [null_value] cannot be null.");
       }
       builder.nullValue(propNode.toString());
       iterator.remove();
     } else if (propName.equals("search_quote_analyzer")) {
       NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
       if (analyzer == null) {
         throw new MapperParsingException(
             "Analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
       }
       builder.searchQuotedAnalyzer(analyzer);
       iterator.remove();
     } else if (propName.equals("position_offset_gap")) {
       builder.positionOffsetGap(XContentMapValues.nodeIntegerValue(propNode, -1));
       // we need to update to actual analyzers if they are not set in this case...
       // so we can inject the position offset gap...
       if (builder.indexAnalyzer == null) {
         builder.indexAnalyzer = parserContext.analysisService().defaultIndexAnalyzer();
       }
       if (builder.searchAnalyzer == null) {
         builder.searchAnalyzer = parserContext.analysisService().defaultSearchAnalyzer();
       }
       if (builder.searchQuotedAnalyzer == null) {
         builder.searchQuotedAnalyzer =
             parserContext.analysisService().defaultSearchQuoteAnalyzer();
       }
       iterator.remove();
     } else if (propName.equals("ignore_above")) {
       builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
       iterator.remove();
     } else if (parseMultiField(builder, name, parserContext, propName, propNode)) {
       iterator.remove();
     }
   }
   return builder;
 }
 /** The template source definition. */
 public PutIndexTemplateRequest source(Map templateSource) {
   Map<String, Object> source = templateSource;
   if (source.containsKey("template")) {
     template(source.get("template").toString());
   }
   if (source.containsKey("order")) {
     order(XContentMapValues.nodeIntegerValue(source.get("order"), order()));
   }
   if (source.containsKey("settings")) {
     if (!(source.get("settings") instanceof Map)) {
       throw new ElasticSearchIllegalArgumentException(
           "Malformed settings section, should include an inner object");
     }
     settings((Map<String, Object>) source.get("settings"));
   }
   if (source.containsKey("mappings")) {
     Map<String, Object> mappings = (Map<String, Object>) source.get("mappings");
     for (Map.Entry<String, Object> entry : mappings.entrySet()) {
       if (!(entry.getValue() instanceof Map)) {
         throw new ElasticSearchIllegalArgumentException(
             "Malformed mappings section for type ["
                 + entry.getKey()
                 + "], should include an inner object describing the mapping");
       }
       mapping(entry.getKey(), (Map<String, Object>) entry.getValue());
     }
   }
   return this;
 }
 public void testFilterWithEmptyIncludesExcludes() {
   Map<String, Object> map = new HashMap<>();
   map.put("field", "value");
   Map<String, Object> filteredMap =
       XContentMapValues.filter(map, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY);
   assertThat(filteredMap.size(), equalTo(1));
   assertThat(filteredMap.get("field").toString(), equalTo("value"));
 }
  @SuppressWarnings("unchecked")
  public void testFilterIncludesUsingStarPrefix() {
    Map<String, Object> map = new HashMap<>();
    map.put("field", "value");
    map.put(
        "obj",
        new HashMap<String, Object>() {
          {
            put("field", "value");
            put("field2", "value2");
          }
        });
    map.put(
        "n_obj",
        new HashMap<String, Object>() {
          {
            put("n_field", "value");
            put("n_field2", "value2");
          }
        });

    Map<String, Object> filteredMap =
        XContentMapValues.filter(map, new String[] {"*.field2"}, Strings.EMPTY_ARRAY);
    assertThat(filteredMap.size(), equalTo(1));
    assertThat(filteredMap, hasKey("obj"));
    assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(1));
    assertThat(((Map<String, Object>) filteredMap.get("obj")), hasKey("field2"));

    // only objects
    filteredMap = XContentMapValues.filter(map, new String[] {"*.*"}, Strings.EMPTY_ARRAY);
    assertThat(filteredMap.size(), equalTo(2));
    assertThat(filteredMap, hasKey("obj"));
    assertThat(((Map<String, Object>) filteredMap.get("obj")).size(), equalTo(2));
    assertThat(filteredMap, hasKey("n_obj"));
    assertThat(((Map<String, Object>) filteredMap.get("n_obj")).size(), equalTo(2));

    filteredMap = XContentMapValues.filter(map, new String[] {"*"}, new String[] {"*.*2"});
    assertThat(filteredMap.size(), equalTo(3));
    assertThat(filteredMap, hasKey("field"));
    assertThat(filteredMap, hasKey("obj"));
    assertThat(((Map) filteredMap.get("obj")).size(), equalTo(1));
    assertThat(((Map<String, Object>) filteredMap.get("obj")), hasKey("field"));
    assertThat(filteredMap, hasKey("n_obj"));
    assertThat(((Map<String, Object>) filteredMap.get("n_obj")).size(), equalTo(1));
    assertThat(((Map<String, Object>) filteredMap.get("n_obj")), hasKey("n_field"));
  }
  @SuppressWarnings({"unchecked"})
  @Inject
  public SimpleRiver(
      RiverName riverName, RiverSettings settings, Client client, ThreadPool threadPool) {
    super(riverName, settings);
    this.client = client;

    if (settings.settings().containsKey("simple")) {
      Map<String, Object> simpleSettings = (Map<String, Object>) settings.settings().get("simple");
      simpleNumber = XContentMapValues.nodeIntegerValue(simpleSettings.get("number"), 100);
      fieldName = XContentMapValues.nodeStringValue(simpleSettings.get("field"), "test");
      poll =
          XContentMapValues.nodeTimeValue(
              simpleSettings.get("poll"), TimeValue.timeValueMinutes(60));
    }

    logger.info(
        "creating simple stream river for [{} numbers] with field [{}]", simpleNumber, fieldName);

    if (settings.settings().containsKey("index")) {
      Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index");
      indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name());
      typeName = XContentMapValues.nodeStringValue(indexSettings.get("type"), "simple_type");
      bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size"), 100);
      bulkThreshold = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_threshold"), 10);
    } else {
      indexName = riverName.name();
      typeName = "simple_type";
      bulkSize = 100;
      bulkThreshold = 10;
    }
  }
 public void testPrefixedNamesFilteringTest() {
   Map<String, Object> map = new HashMap<>();
   map.put("obj", "value");
   map.put("obj_name", "value_name");
   Map<String, Object> filterdMap =
       XContentMapValues.filter(map, new String[] {"obj_name"}, Strings.EMPTY_ARRAY);
   assertThat(filterdMap.size(), equalTo(1));
   assertThat((String) filterdMap.get("obj_name"), equalTo("value_name"));
 }
  public void testThatFilterIncludesEmptyObjectWhenUsingIncludes() throws Exception {
    XContentBuilder builder =
        XContentFactory.jsonBuilder().startObject().startObject("obj").endObject().endObject();

    Tuple<XContentType, Map<String, Object>> mapTuple =
        XContentHelper.convertToMap(builder.bytes(), true);
    Map<String, Object> filteredSource =
        XContentMapValues.filter(mapTuple.v2(), new String[] {"obj"}, Strings.EMPTY_ARRAY);

    assertThat(mapTuple.v2(), equalTo(filteredSource));
  }
  public void testDotsInFieldNames() {
    Map<String, Object> map = new HashMap<>();
    map.put("foo.bar", 2);
    Map<String, Object> sub = new HashMap<>();
    sub.put("baz", 3);
    map.put("foo", sub);
    map.put("quux", 5);

    // dots in field names in includes
    Map<String, Object> filtered =
        XContentMapValues.filter(map, new String[] {"foo"}, new String[0]);
    Map<String, Object> expected = new HashMap<>(map);
    expected.remove("quux");
    assertEquals(expected, filtered);

    // dots in field names in excludes
    filtered = XContentMapValues.filter(map, new String[0], new String[] {"foo"});
    expected = new HashMap<>(map);
    expected.keySet().retainAll(Collections.singleton("quux"));
    assertEquals(expected, filtered);
  }
  @SuppressWarnings({"unchecked"})
  public void testNotOmittingObjectWithNestedExcludedObject() throws Exception {
    XContentBuilder builder =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("obj1")
            .startObject("obj2")
            .startObject("obj3")
            .endObject()
            .endObject()
            .endObject()
            .endObject();

    // implicit include
    Tuple<XContentType, Map<String, Object>> mapTuple =
        XContentHelper.convertToMap(builder.bytes(), true);
    Map<String, Object> filteredSource =
        XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[] {"*.obj2"});

    assertThat(filteredSource.size(), equalTo(1));
    assertThat(filteredSource, hasKey("obj1"));
    assertThat(((Map) filteredSource.get("obj1")).size(), Matchers.equalTo(0));

    // explicit include
    filteredSource =
        XContentMapValues.filter(mapTuple.v2(), new String[] {"obj1"}, new String[] {"*.obj2"});
    assertThat(filteredSource.size(), equalTo(1));
    assertThat(filteredSource, hasKey("obj1"));
    assertThat(((Map) filteredSource.get("obj1")).size(), Matchers.equalTo(0));

    // wild card include
    filteredSource =
        XContentMapValues.filter(mapTuple.v2(), new String[] {"*.obj2"}, new String[] {"*.obj3"});
    assertThat(filteredSource.size(), equalTo(1));
    assertThat(filteredSource, hasKey("obj1"));
    assertThat(((Map<String, Object>) filteredSource.get("obj1")), hasKey("obj2"));
    assertThat(((Map) ((Map) filteredSource.get("obj1")).get("obj2")).size(), Matchers.equalTo(0));
  }
    @Nullable
    @Override
    public Object referenceValue(Reference reference) {
      if (updatedColumnValues == null) {
        return super.referenceValue(reference);
      }

      Object value = updatedColumnValues.get(reference.ident().columnIdent().fqn());
      if (value == null && !reference.ident().isColumn()) {
        value =
            XContentMapValues.extractValue(
                reference.ident().columnIdent().fqn(), updatedColumnValues);
      }
      return reference.valueType().value(value);
    }
 @Override
 public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
     throws MapperParsingException {
   Builder builder = geoShapeField(name);
   for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
       iterator.hasNext(); ) {
     Map.Entry<String, Object> entry = iterator.next();
     String fieldName = Strings.toUnderscoreCase(entry.getKey());
     Object fieldNode = entry.getValue();
     if (Names.TREE.equals(fieldName)) {
       builder.fieldType().setTree(fieldNode.toString());
       iterator.remove();
     } else if (Names.TREE_LEVELS.equals(fieldName)) {
       builder.fieldType().setTreeLevels(Integer.parseInt(fieldNode.toString()));
       iterator.remove();
     } else if (Names.TREE_PRESISION.equals(fieldName)) {
       builder
           .fieldType()
           .setPrecisionInMeters(
               DistanceUnit.parse(
                   fieldNode.toString(), DistanceUnit.DEFAULT, DistanceUnit.DEFAULT));
       iterator.remove();
     } else if (Names.DISTANCE_ERROR_PCT.equals(fieldName)) {
       builder.fieldType().setDistanceErrorPct(Double.parseDouble(fieldNode.toString()));
       iterator.remove();
     } else if (Names.ORIENTATION.equals(fieldName)) {
       builder
           .fieldType()
           .setOrientation(ShapeBuilder.orientationFromString(fieldNode.toString()));
       iterator.remove();
     } else if (Names.STRATEGY.equals(fieldName)) {
       builder.fieldType().setStrategyName(fieldNode.toString());
       iterator.remove();
     } else if (Names.COERCE.equals(fieldName)) {
       builder.coerce(nodeBooleanValue(fieldNode));
       iterator.remove();
     } else if (Names.STRATEGY_POINTS_ONLY.equals(fieldName)
         && builder.fieldType().strategyName.equals(SpatialStrategy.TERM.getStrategyName())
             == false) {
       builder.fieldType().setPointsOnly(XContentMapValues.nodeBooleanValue(fieldNode));
       iterator.remove();
     }
   }
   return builder;
 }
  public void testNotOmittingObjectsWithExcludedProperties() throws Exception {
    XContentBuilder builder =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("obj")
            .field("f1", "v1")
            .endObject()
            .endObject();

    Tuple<XContentType, Map<String, Object>> mapTuple =
        XContentHelper.convertToMap(builder.bytes(), true);
    Map<String, Object> filteredSource =
        XContentMapValues.filter(mapTuple.v2(), Strings.EMPTY_ARRAY, new String[] {"obj.f1"});

    assertThat(filteredSource.size(), equalTo(1));
    assertThat(filteredSource, hasKey("obj"));
    assertThat(((Map) filteredSource.get("obj")).size(), equalTo(0));
  }
 /** The template source definition. */
 public PutIndexTemplateRequest source(Map templateSource) {
   Map<String, Object> source = templateSource;
   for (Map.Entry<String, Object> entry : source.entrySet()) {
     String name = entry.getKey();
     if (name.equals("template")) {
       template(entry.getValue().toString());
     } else if (name.equals("order")) {
       order(XContentMapValues.nodeIntegerValue(entry.getValue(), order()));
     } else if (name.equals("settings")) {
       if (!(entry.getValue() instanceof Map)) {
         throw new ElasticSearchIllegalArgumentException(
             "Malformed settings section, should include an inner object");
       }
       settings((Map<String, Object>) entry.getValue());
     } else if (name.equals("mappings")) {
       Map<String, Object> mappings = (Map<String, Object>) entry.getValue();
       for (Map.Entry<String, Object> entry1 : mappings.entrySet()) {
         if (!(entry1.getValue() instanceof Map)) {
           throw new ElasticSearchIllegalArgumentException(
               "Malformed mappings section for type ["
                   + entry1.getKey()
                   + "], should include an inner object describing the mapping");
         }
         mapping(entry1.getKey(), (Map<String, Object>) entry1.getValue());
       }
     } else {
       // maybe custom?
       IndexMetaData.Custom.Factory factory = IndexMetaData.lookupFactory(name);
       if (factory != null) {
         try {
           customs.put(name, factory.fromMap((Map<String, Object>) entry.getValue()));
         } catch (IOException e) {
           throw new ElasticSearchParseException(
               "failed to parse custom metadata for [" + name + "]");
         }
       }
     }
   }
   return this;
 }
 @Override
 public Mapper.Builder parse(
     String fieldName, Map<String, Object> node, ParserContext parserContext)
     throws MapperParsingException {
   TextFieldMapper.Builder builder = new TextFieldMapper.Builder(fieldName);
   builder.fieldType().setIndexAnalyzer(parserContext.analysisService().defaultIndexAnalyzer());
   builder
       .fieldType()
       .setSearchAnalyzer(parserContext.analysisService().defaultSearchAnalyzer());
   builder
       .fieldType()
       .setSearchQuoteAnalyzer(parserContext.analysisService().defaultSearchQuoteAnalyzer());
   parseTextField(builder, fieldName, node, parserContext);
   for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
       iterator.hasNext(); ) {
     Map.Entry<String, Object> entry = iterator.next();
     String propName = entry.getKey();
     Object propNode = entry.getValue();
     if (propName.equals("position_increment_gap")) {
       int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1);
       builder.positionIncrementGap(newPositionIncrementGap);
       iterator.remove();
     } else if (propName.equals("fielddata")) {
       builder.fielddata(XContentMapValues.nodeBooleanValue(propNode));
       iterator.remove();
     } else if (propName.equals("eager_global_ordinals")) {
       builder.eagerGlobalOrdinals(XContentMapValues.nodeBooleanValue(propNode));
       iterator.remove();
     } else if (propName.equals("fielddata_frequency_filter")) {
       Map<?, ?> frequencyFilter = (Map<?, ?>) propNode;
       double minFrequency = XContentMapValues.nodeDoubleValue(frequencyFilter.remove("min"), 0);
       double maxFrequency =
           XContentMapValues.nodeDoubleValue(frequencyFilter.remove("max"), Integer.MAX_VALUE);
       int minSegmentSize =
           XContentMapValues.nodeIntegerValue(frequencyFilter.remove("min_segment_size"), 0);
       builder.fielddataFrequencyFilter(minFrequency, maxFrequency, minSegmentSize);
       DocumentMapperParser.checkNoRemainingFields(
           propName, frequencyFilter, parserContext.indexVersionCreated());
       iterator.remove();
     }
   }
   return builder;
 }
  /**
   * Configure the river.
   *
   * @param settings used for configuration.
   */
  @SuppressWarnings({"unchecked"})
  protected void configure(Map<String, Object> settings) {

    if (!closed) throw new IllegalStateException("Remote River must be stopped to configure it!");

    if (settings.containsKey("remote")) {
      Map<String, Object> remoteSettings = (Map<String, Object>) settings.get("remote");
      maxIndexingThreads =
          XContentMapValues.nodeIntegerValue(remoteSettings.get("maxIndexingThreads"), 1);

      SpaceIndexingMode sim =
          SpaceIndexingMode.parseConfiguration((String) remoteSettings.get("listDocumentsMode"));
      if (sim != null) spaceIndexingMode = sim;
      else if (XContentMapValues.nodeBooleanValue(remoteSettings.get("simpleGetDocuments"), false))
        spaceIndexingMode = SpaceIndexingMode.SIMPLE;

      indexFullUpdatePeriod =
          Utils.parseTimeValue(remoteSettings, "indexFullUpdatePeriod", 12, TimeUnit.HOURS);

      String ifuce = Utils.trimToNull((String) remoteSettings.get("indexFullUpdateCronExpression"));
      if (ifuce != null) {
        try {
          this.indexFullUpdateCronExpression = new CronExpression(ifuce);
        } catch (ParseException e) {
          throw new SettingsException(
              "Cron expression in indexFullUpdateCronExpression is invalid: " + e.getMessage());
        }
      }

      if (spaceIndexingMode.isIncrementalUpdateSupported()
          || (indexFullUpdatePeriod < 1 && indexFullUpdateCronExpression == null))
        indexUpdatePeriod =
            Utils.parseTimeValue(remoteSettings, "indexUpdatePeriod", 5, TimeUnit.MINUTES);
      else indexUpdatePeriod = 0;

      if (remoteSettings.containsKey("spacesIndexed")) {
        allIndexedSpacesKeys =
            Utils.parseCsvString(
                XContentMapValues.nodeStringValue(remoteSettings.get("spacesIndexed"), null));
        if (allIndexedSpacesKeys != null) {
          // stop spaces loading from remote system
          allIndexedSpacesKeysNextRefresh = Long.MAX_VALUE;
        }
      }
      if (remoteSettings.containsKey("spaceKeysExcluded")) {
        spaceKeysExcluded =
            Utils.parseCsvString(
                XContentMapValues.nodeStringValue(remoteSettings.get("spaceKeysExcluded"), null));
      }
      String remoteClientClass =
          Utils.trimToNull(
              XContentMapValues.nodeStringValue(remoteSettings.get("remoteClientClass"), null));
      if (remoteClientClass != null) {
        try {
          remoteSystemClient = (IRemoteSystemClient) Class.forName(remoteClientClass).newInstance();
        } catch (Exception e) {
          throw new SettingsException(
              "Unable to instantiate class defined by 'remote/remoteClientClass': "
                  + e.getMessage());
        }
      } else {
        remoteSystemClient = new GetJSONClient();
      }
      remoteSystemClient.init(
          this, remoteSettings, allIndexedSpacesKeysNextRefresh != Long.MAX_VALUE, this);
    } else {
      throw new SettingsException("'remote' element of river configuration structure not found");
    }

    Map<String, Object> indexSettings = null;
    if (settings.containsKey("index")) {
      indexSettings = (Map<String, Object>) settings.get("index");
      indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name());
      typeName =
          XContentMapValues.nodeStringValue(
              indexSettings.get("type"), INDEX_DOCUMENT_TYPE_NAME_DEFAULT);
    } else {
      indexName = riverName.name();
      typeName = INDEX_DOCUMENT_TYPE_NAME_DEFAULT;
    }

    Map<String, Object> activityLogSettings = null;
    if (settings.containsKey("activity_log")) {
      activityLogSettings = (Map<String, Object>) settings.get("activity_log");
      activityLogIndexName =
          Utils.trimToNull(
              XContentMapValues.nodeStringValue(activityLogSettings.get("index"), null));
      if (activityLogIndexName == null) {
        throw new SettingsException(
            "'activity_log/index' element of river configuration structure must be defined with some string");
      }
      activityLogTypeName =
          Utils.trimToNull(
              XContentMapValues.nodeStringValue(
                  activityLogSettings.get("type"), INDEX_ACTIVITY_TYPE_NAME_DEFAULT));
    }

    documentIndexStructureBuilder =
        new DocumentWithCommentsIndexStructureBuilder(
            this, indexName, typeName, indexSettings, spaceIndexingMode.isUpdateDateMandatory());
    preparePreprocessors(indexSettings, documentIndexStructureBuilder);

    remoteSystemClient.setIndexStructureBuilder(documentIndexStructureBuilder);

    logger.info(
        "Configured Remote River '{}'. Search index name '{}', document type for issues '{}'. Indexing mode '{}'.",
        riverName.getName(),
        indexName,
        typeName,
        spaceIndexingMode.getConfigValue());
    if (activityLogIndexName != null) {
      logger.info(
          "Activity log for Remote River '{}' is enabled. Search index name '{}', document type for index updates '{}'.",
          riverName.getName(),
          activityLogIndexName,
          activityLogTypeName);
    }
  }
 public Object extractValue(String path) {
   return XContentMapValues.extractValue(path, loadSourceIfNeeded());
 }
 public Object filter(String[] includes, String[] excludes) {
   return XContentMapValues.filter(loadSourceIfNeeded(), includes, excludes);
 }
 /**
  * Returns the values associated with the path. Those are "low" level values, and it can handle
  * path expression where an array/list is navigated within.
  */
 public List<Object> extractRawValues(String path) {
   return XContentMapValues.extractRawValues(path, loadSourceIfNeeded());
 }
Beispiel #26
0
  private InternalSearchHit createNestedSearchHit(
      SearchContext context,
      int nestedTopDocId,
      int nestedSubDocId,
      int rootSubDocId,
      List<String> extractFieldNames,
      boolean loadAllStored,
      Set<String> fieldNames,
      LeafReaderContext subReaderContext)
      throws IOException {
    // Also if highlighting is requested on nested documents we need to fetch the _source from the
    // root document,
    // otherwise highlighting will attempt to fetch the _source from the nested doc, which will
    // fail,
    // because the entire _source is only stored with the root document.
    final FieldsVisitor rootFieldsVisitor =
        new FieldsVisitor(
            context.sourceRequested() || extractFieldNames != null || context.highlight() != null);
    loadStoredFields(context, subReaderContext, rootFieldsVisitor, rootSubDocId);
    rootFieldsVisitor.postProcess(context.mapperService());

    Map<String, SearchHitField> searchFields =
        getSearchFields(context, nestedSubDocId, loadAllStored, fieldNames, subReaderContext);
    DocumentMapper documentMapper =
        context.mapperService().documentMapper(rootFieldsVisitor.uid().type());
    SourceLookup sourceLookup = context.lookup().source();
    sourceLookup.setSegmentAndDocument(subReaderContext, nestedSubDocId);

    ObjectMapper nestedObjectMapper =
        documentMapper.findNestedObjectMapper(nestedSubDocId, context, subReaderContext);
    assert nestedObjectMapper != null;
    InternalSearchHit.InternalNestedIdentity nestedIdentity =
        getInternalNestedIdentity(
            context, nestedSubDocId, subReaderContext, documentMapper, nestedObjectMapper);

    BytesReference source = rootFieldsVisitor.source();
    if (source != null) {
      Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(source, true);
      Map<String, Object> sourceAsMap = tuple.v2();

      List<Map<String, Object>> nestedParsedSource;
      SearchHit.NestedIdentity nested = nestedIdentity;
      do {
        Object extractedValue =
            XContentMapValues.extractValue(nested.getField().string(), sourceAsMap);
        if (extractedValue == null) {
          // The nested objects may not exist in the _source, because it was filtered because of
          // _source filtering
          break;
        } else if (extractedValue instanceof List) {
          // nested field has an array value in the _source
          nestedParsedSource = (List<Map<String, Object>>) extractedValue;
        } else if (extractedValue instanceof Map) {
          // nested field has an object value in the _source. This just means the nested field has
          // just one inner object, which is valid, but uncommon.
          nestedParsedSource = ImmutableList.of((Map<String, Object>) extractedValue);
        } else {
          throw new IllegalStateException("extracted source isn't an object or an array");
        }
        sourceAsMap = nestedParsedSource.get(nested.getOffset());
        nested = nested.getChild();
      } while (nested != null);

      context.lookup().source().setSource(sourceAsMap);
      XContentType contentType = tuple.v1();
      BytesReference nestedSource = contentBuilder(contentType).map(sourceAsMap).bytes();
      context.lookup().source().setSource(nestedSource);
      context.lookup().source().setSourceContentType(contentType);
    }

    InternalSearchHit searchHit =
        new InternalSearchHit(
            nestedTopDocId,
            rootFieldsVisitor.uid().id(),
            documentMapper.typeText(),
            nestedIdentity,
            searchFields);
    if (extractFieldNames != null) {
      for (String extractFieldName : extractFieldNames) {
        List<Object> values = context.lookup().source().extractRawValues(extractFieldName);
        if (!values.isEmpty()) {
          if (searchHit.fieldsOrNull() == null) {
            searchHit.fields(new HashMap<String, SearchHitField>(2));
          }

          SearchHitField hitField = searchHit.fields().get(extractFieldName);
          if (hitField == null) {
            hitField = new InternalSearchHitField(extractFieldName, new ArrayList<>(2));
            searchHit.fields().put(extractFieldName, hitField);
          }
          for (Object value : values) {
            hitField.values().add(value);
          }
        }
      }
    }

    return searchHit;
  }
  @SuppressWarnings({"unchecked"})
  @Inject
  public RabbitmqRiver(
      RiverName riverName, RiverSettings settings, Client client, ScriptService scriptService) {
    super(riverName, settings);
    this.client = client;

    if (settings.settings().containsKey("rabbitmq")) {
      Map<String, Object> rabbitSettings =
          (Map<String, Object>) settings.settings().get("rabbitmq");

      if (rabbitSettings.containsKey("addresses")) {
        List<Address> addresses = new ArrayList<Address>();
        for (Map<String, Object> address :
            (List<Map<String, Object>>) rabbitSettings.get("addresses")) {
          addresses.add(
              new Address(
                  XContentMapValues.nodeStringValue(address.get("host"), "localhost"),
                  XContentMapValues.nodeIntegerValue(address.get("port"), AMQP.PROTOCOL.PORT)));
        }
        rabbitAddresses = addresses.toArray(new Address[addresses.size()]);
      } else {
        String rabbitHost =
            XContentMapValues.nodeStringValue(rabbitSettings.get("host"), "localhost");
        int rabbitPort =
            XContentMapValues.nodeIntegerValue(rabbitSettings.get("port"), AMQP.PROTOCOL.PORT);
        rabbitAddresses = new Address[] {new Address(rabbitHost, rabbitPort)};
      }

      rabbitUser = XContentMapValues.nodeStringValue(rabbitSettings.get("user"), "guest");
      rabbitPassword = XContentMapValues.nodeStringValue(rabbitSettings.get("pass"), "guest");
      rabbitVhost = XContentMapValues.nodeStringValue(rabbitSettings.get("vhost"), "/");

      rabbitQueue = XContentMapValues.nodeStringValue(rabbitSettings.get("queue"), "elasticsearch");
      rabbitExchange =
          XContentMapValues.nodeStringValue(rabbitSettings.get("exchange"), "elasticsearch");
      rabbitRoutingKey =
          XContentMapValues.nodeStringValue(rabbitSettings.get("routing_key"), "elasticsearch");

      rabbitExchangeDeclare =
          XContentMapValues.nodeBooleanValue(rabbitSettings.get("exchange_declare"), true);
      if (rabbitExchangeDeclare) {

        rabbitExchangeType =
            XContentMapValues.nodeStringValue(rabbitSettings.get("exchange_type"), "direct");
        rabbitExchangeDurable =
            XContentMapValues.nodeBooleanValue(rabbitSettings.get("exchange_durable"), true);
      } else {
        rabbitExchangeType = "direct";
        rabbitExchangeDurable = true;
      }

      rabbitQueueDeclare =
          XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_declare"), true);
      if (rabbitQueueDeclare) {
        rabbitQueueDurable =
            XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_durable"), true);
        rabbitQueueAutoDelete =
            XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_auto_delete"), false);
        if (rabbitSettings.containsKey("args")) {
          rabbitQueueArgs = (Map<String, Object>) rabbitSettings.get("args");
        }
      } else {
        rabbitQueueDurable = true;
        rabbitQueueAutoDelete = false;
      }
      rabbitQueueBind = XContentMapValues.nodeBooleanValue(rabbitSettings.get("queue_bind"), true);

      rabbitQosPrefetchSize =
          XContentMapValues.nodeIntegerValue(rabbitSettings.get("qos_prefetch_size"), 0);
      rabbitQosPrefetchCount =
          XContentMapValues.nodeIntegerValue(rabbitSettings.get("qos_prefetch_count"), 10);

      rabbitHeartbeat =
          TimeValue.parseTimeValue(
              XContentMapValues.nodeStringValue(rabbitSettings.get("heartbeat"), "30m"),
              TimeValue.timeValueMinutes(30));

    } else {
      rabbitAddresses = new Address[] {new Address("localhost", AMQP.PROTOCOL.PORT)};
      rabbitUser = "******";
      rabbitPassword = "******";
      rabbitVhost = "/";

      rabbitQueue = "elasticsearch";
      rabbitQueueAutoDelete = false;
      rabbitQueueDurable = true;
      rabbitExchange = "elasticsearch";
      rabbitExchangeType = "direct";
      rabbitExchangeDurable = true;
      rabbitRoutingKey = "elasticsearch";

      rabbitExchangeDeclare = true;
      rabbitQueueDeclare = true;
      rabbitQueueBind = true;

      rabbitQosPrefetchSize = 0;
      rabbitQosPrefetchCount = 10;

      rabbitHeartbeat = TimeValue.timeValueMinutes(30);
    }

    if (settings.settings().containsKey("index")) {
      Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index");
      bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size"), 100);
      if (indexSettings.containsKey("bulk_timeout")) {
        bulkTimeout =
            TimeValue.parseTimeValue(
                XContentMapValues.nodeStringValue(indexSettings.get("bulk_timeout"), "10ms"),
                TimeValue.timeValueMillis(10));
      } else {
        bulkTimeout = TimeValue.timeValueMillis(10);
      }
      ordered = XContentMapValues.nodeBooleanValue(indexSettings.get("ordered"), false);
    } else {
      bulkSize = 100;
      bulkTimeout = TimeValue.timeValueMillis(10);
      ordered = false;
    }

    if (settings.settings().containsKey("bulk_script_filter")) {
      Map<String, Object> scriptSettings =
          (Map<String, Object>) settings.settings().get("bulk_script_filter");
      if (scriptSettings.containsKey("script")) {
        String scriptLang = "native";
        if (scriptSettings.containsKey("script_lang")) {
          scriptLang = scriptSettings.get("script_lang").toString();
        }
        Map<String, Object> scriptParams = null;
        if (scriptSettings.containsKey("script_params")) {
          scriptParams = (Map<String, Object>) scriptSettings.get("script_params");
        } else {
          scriptParams = Maps.newHashMap();
        }
        bulkScript =
            scriptService.executable(
                scriptLang, scriptSettings.get("script").toString(), scriptParams);
      } else {
        bulkScript = null;
      }
    } else {
      bulkScript = null;
    }

    if (settings.settings().containsKey("script_filter")) {
      Map<String, Object> scriptSettings =
          (Map<String, Object>) settings.settings().get("script_filter");
      if (scriptSettings.containsKey("script")) {
        String scriptLang = "mvel";
        if (scriptSettings.containsKey("script_lang")) {
          scriptLang = scriptSettings.get("script_lang").toString();
        }
        Map<String, Object> scriptParams = null;
        if (scriptSettings.containsKey("script_params")) {
          scriptParams = (Map<String, Object>) scriptSettings.get("script_params");
        } else {
          scriptParams = Maps.newHashMap();
        }
        script =
            scriptService.executable(
                scriptLang, scriptSettings.get("script").toString(), scriptParams);
      } else {
        script = null;
      }
    } else {
      script = null;
    }
  }
  @SuppressWarnings("unchecked")
  public static synchronized MongoDBRiverDefinition parseSettings(
      String riverName,
      String riverIndexName,
      RiverSettings settings,
      ScriptService scriptService) {

    logger.trace("Parse river settings for {}", riverName);
    Preconditions.checkNotNull(riverName, "No riverName specified");
    Preconditions.checkNotNull(riverIndexName, "No riverIndexName specified");
    Preconditions.checkNotNull(settings, "No settings specified");

    Builder builder = new Builder();
    builder.riverName(riverName);
    builder.riverIndexName(riverIndexName);

    List<ServerAddress> mongoServers = new ArrayList<ServerAddress>();
    String mongoHost;
    int mongoPort;

    if (settings.settings().containsKey(MongoDBRiver.TYPE)) {
      Map<String, Object> mongoSettings =
          (Map<String, Object>) settings.settings().get(MongoDBRiver.TYPE);
      if (mongoSettings.containsKey(SERVERS_FIELD)) {
        Object mongoServersSettings = mongoSettings.get(SERVERS_FIELD);
        logger.trace("mongoServersSettings: " + mongoServersSettings);
        boolean array = XContentMapValues.isArray(mongoServersSettings);

        if (array) {
          ArrayList<Map<String, Object>> feeds =
              (ArrayList<Map<String, Object>>) mongoServersSettings;
          for (Map<String, Object> feed : feeds) {
            mongoHost = XContentMapValues.nodeStringValue(feed.get(HOST_FIELD), null);
            mongoPort = XContentMapValues.nodeIntegerValue(feed.get(PORT_FIELD), DEFAULT_DB_PORT);
            logger.trace("Server: " + mongoHost + " - " + mongoPort);
            try {
              mongoServers.add(new ServerAddress(mongoHost, mongoPort));
            } catch (UnknownHostException uhEx) {
              logger.warn("Cannot add mongo server {}:{}", uhEx, mongoHost, mongoPort);
            }
          }
        }
      } else {
        mongoHost =
            XContentMapValues.nodeStringValue(mongoSettings.get(HOST_FIELD), DEFAULT_DB_HOST);
        mongoPort =
            XContentMapValues.nodeIntegerValue(mongoSettings.get(PORT_FIELD), DEFAULT_DB_PORT);
        try {
          mongoServers.add(new ServerAddress(mongoHost, mongoPort));
        } catch (UnknownHostException uhEx) {
          logger.warn("Cannot add mongo server {}:{}", uhEx, mongoHost, mongoPort);
        }
      }
      builder.mongoServers(mongoServers);

      MongoClientOptions.Builder mongoClientOptionsBuilder =
          MongoClientOptions.builder().socketKeepAlive(true);

      // MongoDB options
      if (mongoSettings.containsKey(OPTIONS_FIELD)) {
        Map<String, Object> mongoOptionsSettings =
            (Map<String, Object>) mongoSettings.get(OPTIONS_FIELD);
        logger.trace("mongoOptionsSettings: " + mongoOptionsSettings);
        builder.mongoSecondaryReadPreference(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(SECONDARY_READ_PREFERENCE_FIELD), false));
        builder.connectTimeout(
            XContentMapValues.nodeIntegerValue(
                mongoOptionsSettings.get(CONNECT_TIMEOUT), DEFAULT_CONNECT_TIMEOUT));
        builder.socketTimeout(
            XContentMapValues.nodeIntegerValue(
                mongoOptionsSettings.get(SOCKET_TIMEOUT), DEFAULT_SOCKET_TIMEOUT));
        builder.dropCollection(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(DROP_COLLECTION_FIELD), false));
        String isMongos =
            XContentMapValues.nodeStringValue(mongoOptionsSettings.get(IS_MONGOS_FIELD), null);
        if (isMongos != null) {
          builder.isMongos(Boolean.valueOf(isMongos));
        }
        builder.mongoUseSSL(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(SSL_CONNECTION_FIELD), false));
        builder.mongoSSLVerifyCertificate(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(SSL_VERIFY_CERT_FIELD), true));
        builder.advancedTransformation(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(ADVANCED_TRANSFORMATION_FIELD), false));
        builder.skipInitialImport(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(SKIP_INITIAL_IMPORT_FIELD), false));
        builder.connectionsPerHost(
            XContentMapValues.nodeIntegerValue(
                mongoOptionsSettings.get(CONNECTIONS_PER_HOST), DEFAULT_CONNECTIONS_PER_HOST));
        builder.threadsAllowedToBlockForConnectionMultiplier(
            XContentMapValues.nodeIntegerValue(
                mongoOptionsSettings.get(THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER),
                DEFAULT_THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER));

        mongoClientOptionsBuilder
            .connectTimeout(builder.connectTimeout)
            .socketTimeout(builder.socketTimeout)
            .connectionsPerHost(builder.connectionsPerHost)
            .threadsAllowedToBlockForConnectionMultiplier(
                builder.threadsAllowedToBlockForConnectionMultiplier);

        if (builder.mongoSecondaryReadPreference) {
          mongoClientOptionsBuilder.readPreference(ReadPreference.secondaryPreferred());
        }

        if (builder.mongoUseSSL) {
          mongoClientOptionsBuilder.socketFactory(getSSLSocketFactory());
        }

        if (mongoOptionsSettings.containsKey(PARENT_TYPES_FIELD)) {
          Set<String> parentTypes = new HashSet<String>();
          Object parentTypesSettings = mongoOptionsSettings.get(PARENT_TYPES_FIELD);
          logger.trace("parentTypesSettings: " + parentTypesSettings);
          boolean array = XContentMapValues.isArray(parentTypesSettings);

          if (array) {
            ArrayList<String> fields = (ArrayList<String>) parentTypesSettings;
            for (String field : fields) {
              logger.trace("Field: " + field);
              parentTypes.add(field);
            }
          }

          builder.parentTypes(parentTypes);
        }

        if (mongoOptionsSettings.containsKey(STORE_STATISTICS_FIELD)) {
          Object storeStatistics = mongoOptionsSettings.get(STORE_STATISTICS_FIELD);
          boolean object = XContentMapValues.isObject(storeStatistics);
          if (object) {
            Map<String, Object> storeStatisticsSettings = (Map<String, Object>) storeStatistics;
            builder.storeStatistics(true);
            builder.statisticsIndexName(
                XContentMapValues.nodeStringValue(
                    storeStatisticsSettings.get(INDEX_OBJECT), riverName + "-stats"));
            builder.statisticsTypeName(
                XContentMapValues.nodeStringValue(
                    storeStatisticsSettings.get(TYPE_FIELD), "stats"));
          } else {
            builder.storeStatistics(XContentMapValues.nodeBooleanValue(storeStatistics, false));
            if (builder.storeStatistics) {
              builder.statisticsIndexName(riverName + "-stats");
              builder.statisticsTypeName("stats");
            }
          }
        }
        // builder.storeStatistics(XContentMapValues.nodeBooleanValue(mongoOptionsSettings.get(STORE_STATISTICS_FIELD),
        // false));
        builder.importAllCollections(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(IMPORT_ALL_COLLECTIONS_FIELD), false));
        builder.disableIndexRefresh(
            XContentMapValues.nodeBooleanValue(
                mongoOptionsSettings.get(DISABLE_INDEX_REFRESH_FIELD), false));
        builder.includeCollection(
            XContentMapValues.nodeStringValue(
                mongoOptionsSettings.get(INCLUDE_COLLECTION_FIELD), ""));

        if (mongoOptionsSettings.containsKey(INCLUDE_FIELDS_FIELD)) {
          Set<String> includeFields = new HashSet<String>();
          Object includeFieldsSettings = mongoOptionsSettings.get(INCLUDE_FIELDS_FIELD);
          logger.trace("includeFieldsSettings: " + includeFieldsSettings);
          boolean array = XContentMapValues.isArray(includeFieldsSettings);

          if (array) {
            ArrayList<String> fields = (ArrayList<String>) includeFieldsSettings;
            for (String field : fields) {
              logger.trace("Field: " + field);
              includeFields.add(field);
            }
          }

          if (!includeFields.contains(MongoDBRiver.MONGODB_ID_FIELD)) {
            includeFields.add(MongoDBRiver.MONGODB_ID_FIELD);
          }
          builder.includeFields(includeFields);
        } else if (mongoOptionsSettings.containsKey(EXCLUDE_FIELDS_FIELD)) {
          Set<String> excludeFields = new HashSet<String>();
          Object excludeFieldsSettings = mongoOptionsSettings.get(EXCLUDE_FIELDS_FIELD);
          logger.trace("excludeFieldsSettings: " + excludeFieldsSettings);
          boolean array = XContentMapValues.isArray(excludeFieldsSettings);

          if (array) {
            ArrayList<String> fields = (ArrayList<String>) excludeFieldsSettings;
            for (String field : fields) {
              logger.trace("Field: " + field);
              excludeFields.add(field);
            }
          }

          builder.excludeFields(excludeFields);
        }

        if (mongoOptionsSettings.containsKey(INITIAL_TIMESTAMP_FIELD)) {
          BSONTimestamp timeStamp = null;
          try {
            Map<String, Object> initalTimestampSettings =
                (Map<String, Object>) mongoOptionsSettings.get(INITIAL_TIMESTAMP_FIELD);
            String scriptType = "js";
            if (initalTimestampSettings.containsKey(INITIAL_TIMESTAMP_SCRIPT_TYPE_FIELD)) {
              scriptType =
                  initalTimestampSettings.get(INITIAL_TIMESTAMP_SCRIPT_TYPE_FIELD).toString();
            }
            if (initalTimestampSettings.containsKey(INITIAL_TIMESTAMP_SCRIPT_FIELD)) {

              ExecutableScript scriptExecutable =
                  scriptService.executable(
                      scriptType,
                      initalTimestampSettings.get(INITIAL_TIMESTAMP_SCRIPT_FIELD).toString(),
                      ScriptService.ScriptType.INLINE,
                      Maps.newHashMap());
              Object ctx = scriptExecutable.run();
              logger.trace("initialTimestamp script returned: {}", ctx);
              if (ctx != null) {
                long timestamp = Long.parseLong(ctx.toString());
                timeStamp = new BSONTimestamp((int) (new Date(timestamp).getTime() / 1000), 1);
              }
            }
          } catch (Throwable t) {
            logger.error("Could not set initial timestamp", t);
          } finally {
            builder.initialTimestamp(timeStamp);
          }
        }
      }
      builder.mongoClientOptions(mongoClientOptionsBuilder.build());

      // Credentials
      if (mongoSettings.containsKey(CREDENTIALS_FIELD)) {
        String dbCredential;
        String mau = "";
        String map = "";
        String maad = "";
        String mlu = "";
        String mlp = "";
        String mlad = "";
        // String mdu = "";
        // String mdp = "";
        Object mongoCredentialsSettings = mongoSettings.get(CREDENTIALS_FIELD);
        boolean array = XContentMapValues.isArray(mongoCredentialsSettings);

        if (array) {
          ArrayList<Map<String, Object>> credentials =
              (ArrayList<Map<String, Object>>) mongoCredentialsSettings;
          for (Map<String, Object> credential : credentials) {
            dbCredential = XContentMapValues.nodeStringValue(credential.get(DB_FIELD), null);
            if (ADMIN_DB_FIELD.equals(dbCredential)) {
              mau = XContentMapValues.nodeStringValue(credential.get(USER_FIELD), null);
              map = XContentMapValues.nodeStringValue(credential.get(PASSWORD_FIELD), null);
              maad = XContentMapValues.nodeStringValue(credential.get(AUTH_FIELD), null);
            } else if (LOCAL_DB_FIELD.equals(dbCredential)) {
              mlu = XContentMapValues.nodeStringValue(credential.get(USER_FIELD), null);
              mlp = XContentMapValues.nodeStringValue(credential.get(PASSWORD_FIELD), null);
              mlad = XContentMapValues.nodeStringValue(credential.get(AUTH_FIELD), null);
              // } else {
              // mdu = XContentMapValues.nodeStringValue(
              // credential.get(USER_FIELD), null);
              // mdp = XContentMapValues.nodeStringValue(
              // credential.get(PASSWORD_FIELD), null);
            }
          }
        }
        builder.mongoAdminUser(mau);
        builder.mongoAdminPassword(map);
        builder.mongoAdminAuthDatabase(maad);
        builder.mongoLocalUser(mlu);
        builder.mongoLocalPassword(mlp);
        builder.mongoLocalAuthDatabase(mlad);
        // mongoDbUser = mdu;
        // mongoDbPassword = mdp;
      }

      builder.mongoDb(XContentMapValues.nodeStringValue(mongoSettings.get(DB_FIELD), riverName));
      builder.mongoCollection(
          XContentMapValues.nodeStringValue(mongoSettings.get(COLLECTION_FIELD), riverName));
      builder.mongoGridFS(
          XContentMapValues.nodeBooleanValue(mongoSettings.get(GRIDFS_FIELD), false));
      if (mongoSettings.containsKey(FILTER_FIELD)) {
        String filter = XContentMapValues.nodeStringValue(mongoSettings.get(FILTER_FIELD), "");
        filter = removePrefix("o.", filter);
        builder.mongoCollectionFilter(convertToBasicDBObject(filter));
        // DBObject bsonObject = (DBObject) JSON.parse(filter);
        // builder.mongoOplogFilter(convertToBasicDBObject(addPrefix("o.",
        // filter)));
        builder.mongoOplogFilter(convertToBasicDBObject(removePrefix("o.", filter)));
        // } else {
        // builder.mongoOplogFilter("");
      }

      if (mongoSettings.containsKey(SCRIPT_FIELD)) {
        String scriptType = "js";
        builder.script(mongoSettings.get(SCRIPT_FIELD).toString());
        if (mongoSettings.containsKey("scriptType")) {
          scriptType = mongoSettings.get("scriptType").toString();
        } else if (mongoSettings.containsKey(SCRIPT_TYPE_FIELD)) {
          scriptType = mongoSettings.get(SCRIPT_TYPE_FIELD).toString();
        }
        builder.scriptType(scriptType);
      }
    } else {
      mongoHost = DEFAULT_DB_HOST;
      mongoPort = DEFAULT_DB_PORT;
      try {
        mongoServers.add(new ServerAddress(mongoHost, mongoPort));
        builder.mongoServers(mongoServers);
      } catch (UnknownHostException e) {
        e.printStackTrace();
      }
      builder.mongoDb(riverName);
      builder.mongoCollection(riverName);
    }

    if (settings.settings().containsKey(INDEX_OBJECT)) {
      Map<String, Object> indexSettings =
          (Map<String, Object>) settings.settings().get(INDEX_OBJECT);
      builder.indexName(
          XContentMapValues.nodeStringValue(indexSettings.get(NAME_FIELD), builder.mongoDb));
      builder.typeName(
          XContentMapValues.nodeStringValue(indexSettings.get(TYPE_FIELD), builder.mongoDb));

      Bulk.Builder bulkBuilder = new Bulk.Builder();
      if (indexSettings.containsKey(BULK_FIELD)) {
        Map<String, Object> bulkSettings = (Map<String, Object>) indexSettings.get(BULK_FIELD);
        int bulkActions =
            XContentMapValues.nodeIntegerValue(
                bulkSettings.get(ACTIONS_FIELD), DEFAULT_BULK_ACTIONS);
        bulkBuilder.bulkActions(bulkActions);
        String size =
            XContentMapValues.nodeStringValue(
                bulkSettings.get(SIZE_FIELD), DEFAULT_BULK_SIZE.toString());
        bulkBuilder.bulkSize(ByteSizeValue.parseBytesSizeValue(size));
        bulkBuilder.concurrentRequests(
            XContentMapValues.nodeIntegerValue(
                bulkSettings.get(CONCURRENT_REQUESTS_FIELD),
                EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY)));
        bulkBuilder.flushInterval(
            XContentMapValues.nodeTimeValue(
                bulkSettings.get(FLUSH_INTERVAL_FIELD), DEFAULT_FLUSH_INTERVAL));
        builder.throttleSize(
            XContentMapValues.nodeIntegerValue(
                indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5));
      } else {
        int bulkActions =
            XContentMapValues.nodeIntegerValue(
                indexSettings.get(BULK_SIZE_FIELD), DEFAULT_BULK_ACTIONS);
        bulkBuilder.bulkActions(bulkActions);
        bulkBuilder.bulkSize(DEFAULT_BULK_SIZE);
        bulkBuilder.flushInterval(
            XContentMapValues.nodeTimeValue(
                indexSettings.get(BULK_TIMEOUT_FIELD), DEFAULT_FLUSH_INTERVAL));
        bulkBuilder.concurrentRequests(
            XContentMapValues.nodeIntegerValue(
                indexSettings.get(CONCURRENT_BULK_REQUESTS_FIELD),
                EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY)));
        builder.throttleSize(
            XContentMapValues.nodeIntegerValue(
                indexSettings.get(THROTTLE_SIZE_FIELD), bulkActions * 5));
      }
      builder.bulk(bulkBuilder.build());
    } else {
      builder.indexName(builder.mongoDb);
      builder.typeName(builder.mongoDb);
      builder.bulk(new Bulk.Builder().build());
    }
    return builder.build();
  }
 @Override
 public Mapper.Builder parse(
     String fieldName, Map<String, Object> node, ParserContext parserContext)
     throws MapperParsingException {
   StringFieldMapper.Builder builder = new StringFieldMapper.Builder(fieldName);
   // hack for the fact that string can't just accept true/false for
   // the index property and still accepts no/not_analyzed/analyzed
   final Object index = node.remove("index");
   if (index != null) {
     final String normalizedIndex = Strings.toUnderscoreCase(index.toString());
     switch (normalizedIndex) {
       case "analyzed":
         builder.tokenized(true);
         node.put("index", true);
         break;
       case "not_analyzed":
         builder.tokenized(false);
         node.put("index", true);
         break;
       case "no":
         node.put("index", false);
         break;
       default:
         throw new IllegalArgumentException(
             "Can't parse [index] value ["
                 + index
                 + "] for field ["
                 + fieldName
                 + "], expected [true], [false], [no], [not_analyzed] or [analyzed]");
     }
   }
   parseTextField(builder, fieldName, node, parserContext);
   for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator();
       iterator.hasNext(); ) {
     Map.Entry<String, Object> entry = iterator.next();
     String propName = Strings.toUnderscoreCase(entry.getKey());
     Object propNode = entry.getValue();
     if (propName.equals("null_value")) {
       if (propNode == null) {
         throw new MapperParsingException("Property [null_value] cannot be null.");
       }
       builder.nullValue(propNode.toString());
       iterator.remove();
     } else if (propName.equals("position_increment_gap")) {
       int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(propNode, -1);
       if (newPositionIncrementGap < 0) {
         throw new MapperParsingException("positions_increment_gap less than 0 aren't allowed.");
       }
       builder.positionIncrementGap(newPositionIncrementGap);
       // we need to update to actual analyzers if they are not set in this case...
       // so we can inject the position increment gap...
       if (builder.fieldType().indexAnalyzer() == null) {
         builder
             .fieldType()
             .setIndexAnalyzer(parserContext.analysisService().defaultIndexAnalyzer());
       }
       if (builder.fieldType().searchAnalyzer() == null) {
         builder
             .fieldType()
             .setSearchAnalyzer(parserContext.analysisService().defaultSearchAnalyzer());
       }
       if (builder.fieldType().searchQuoteAnalyzer() == null) {
         builder
             .fieldType()
             .setSearchQuoteAnalyzer(
                 parserContext.analysisService().defaultSearchQuoteAnalyzer());
       }
       iterator.remove();
     } else if (propName.equals("ignore_above")) {
       builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
       iterator.remove();
     } else if (parseMultiField(builder, fieldName, parserContext, propName, propNode)) {
       iterator.remove();
     }
   }
   return builder;
 }
  public void testFilter() throws Exception {
    XContentBuilder builder =
        XContentFactory.jsonBuilder()
            .startObject()
            .field("test1", "value1")
            .field("test2", "value2")
            .field("something_else", "value3")
            .endObject();

    Map<String, Object> source;
    try (XContentParser parser =
        XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
      source = parser.map();
    }
    Map<String, Object> filter =
        XContentMapValues.filter(source, new String[] {"test1"}, Strings.EMPTY_ARRAY);
    assertThat(filter.size(), equalTo(1));
    assertThat(filter.get("test1").toString(), equalTo("value1"));

    filter = XContentMapValues.filter(source, new String[] {"test*"}, Strings.EMPTY_ARRAY);
    assertThat(filter.size(), equalTo(2));
    assertThat(filter.get("test1").toString(), equalTo("value1"));
    assertThat(filter.get("test2").toString(), equalTo("value2"));

    filter = XContentMapValues.filter(source, Strings.EMPTY_ARRAY, new String[] {"test1"});
    assertThat(filter.size(), equalTo(2));
    assertThat(filter.get("test2").toString(), equalTo("value2"));
    assertThat(filter.get("something_else").toString(), equalTo("value3"));

    // more complex object...
    builder =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("path1")
            .startArray("path2")
            .startObject()
            .field("test", "value1")
            .endObject()
            .startObject()
            .field("test", "value2")
            .endObject()
            .endArray()
            .endObject()
            .field("test1", "value1")
            .endObject();

    try (XContentParser parser =
        XContentFactory.xContent(XContentType.JSON).createParser(builder.string())) {
      source = parser.map();
    }
    filter = XContentMapValues.filter(source, new String[] {"path1"}, Strings.EMPTY_ARRAY);
    assertThat(filter.size(), equalTo(1));

    filter = XContentMapValues.filter(source, new String[] {"path1*"}, Strings.EMPTY_ARRAY);
    assertThat(filter.get("path1"), equalTo(source.get("path1")));
    assertThat(filter.containsKey("test1"), equalTo(false));

    filter = XContentMapValues.filter(source, new String[] {"test1*"}, Strings.EMPTY_ARRAY);
    assertThat(filter.get("test1"), equalTo(source.get("test1")));
    assertThat(filter.containsKey("path1"), equalTo(false));

    filter = XContentMapValues.filter(source, new String[] {"path1.path2.*"}, Strings.EMPTY_ARRAY);
    assertThat(filter.get("path1"), equalTo(source.get("path1")));
    assertThat(filter.containsKey("test1"), equalTo(false));
  }