// #6187: if no fields were boosted, we shouldn't use AllTokenStream
 public void testNoBoost() throws Exception {
   String mapping =
       copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json");
   DocumentMapper docMapper =
       createIndex("test").mapperService().documentMapperParser().parse(mapping);
   byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
   Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
   AllField field = (AllField) doc.getField("_all");
   // no fields have boost, so we should not see AllTokenStream:
   assertThat(
       field.tokenStream(docMapper.mappers().indexAnalyzer(), null),
       Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
 }
 public void testAllMappersNoBoost() throws Exception {
   String mapping =
       copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json");
   IndexService index = createIndex("test");
   DocumentMapper docMapper = index.mapperService().documentMapperParser().parse(mapping);
   byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
   Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
   AllField field = (AllField) doc.getField("_all");
   AllEntries allEntries = field.getAllEntries();
   assertThat(allEntries.fields().size(), equalTo(3));
   assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
   assertThat(allEntries.fields().contains("name.last"), equalTo(true));
   assertThat(allEntries.fields().contains("simple1"), equalTo(true));
   assertThat(field.fieldType().omitNorms(), equalTo(false));
 }
 // #6187: if _all doesn't index positions then we never use AllTokenStream, even if some fields
 // have boost
 public void testBoostWithOmitPositions() throws Exception {
   String mapping =
       copyToStringFromClasspath(
           "/org/elasticsearch/index/mapper/all/mapping_boost_omit_positions_on_all.json");
   DocumentMapper docMapper =
       createIndex("test").mapperService().documentMapperParser().parse(mapping);
   byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
   Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
   AllField field = (AllField) doc.getField("_all");
   // _all field omits positions, so we should not get AllTokenStream even though fields are
   // boosted
   assertThat(
       field.tokenStream(docMapper.mappers().indexAnalyzer(), null),
       Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
 }
  public void testSimpleAllMappersWithStore() throws Exception {
    String mapping =
        copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/store-mapping.json");
    DocumentMapper docMapper =
        createIndex("test").mapperService().documentMapperParser().parse(mapping);
    byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
    Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
    AllField field = (AllField) doc.getField("_all");
    AllEntries allEntries = field.getAllEntries();
    assertThat(allEntries.fields().size(), equalTo(2));
    assertThat(allEntries.fields().contains("name.last"), equalTo(true));
    assertThat(allEntries.fields().contains("simple1"), equalTo(true));

    String text = field.stringValue();
    assertThat(text, equalTo(allEntries.buildText()));
    assertThat(field.fieldType().omitNorms(), equalTo(false));
  }
  public void testSimpleAllMappersWithReparse() throws Exception {
    DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
    String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
    DocumentMapper docMapper = parser.parse(mapping);
    String builtMapping = docMapper.mappingSource().string();
    // reparse it
    DocumentMapper builtDocMapper = parser.parse(builtMapping);
    byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
    Document doc = builtDocMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();

    AllField field = (AllField) doc.getField("_all");
    AllEntries allEntries = field.getAllEntries();
    assertThat(allEntries.fields().toString(), allEntries.fields().size(), equalTo(3));
    assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
    assertThat(allEntries.fields().contains("name.last"), equalTo(true));
    assertThat(allEntries.fields().contains("simple1"), equalTo(true));
    assertThat(field.fieldType().omitNorms(), equalTo(true));
  }
  public void testIncludeInObjectBackcompat() throws Exception {
    String mapping =
        jsonBuilder().startObject().startObject("type").endObject().endObject().string();
    Settings settings =
        Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
    DocumentMapper docMapper =
        createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);
    ParsedDocument doc =
        docMapper.parse(
            "test",
            "type",
            "1",
            XContentFactory.jsonBuilder().startObject().field("_all", "foo").endObject().bytes());

    assertNull(doc.rootDoc().get("_all"));
    AllField field = (AllField) doc.rootDoc().getField("_all");
    // the backcompat behavior is actually ignoring directly specifying _all
    assertFalse(field.getAllEntries().fields().iterator().hasNext());
  }
 @Test
 public void testSimpleAllMappers() throws Exception {
   String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
   DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
   byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
   Document doc = docMapper.parse(new BytesArray(json)).rootDoc();
   AllField field = (AllField) doc.getField("_all");
   AllEntries allEntries =
       ((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries();
   assertThat(allEntries.fields().size(), equalTo(3));
   assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
   assertThat(allEntries.fields().contains("name.last"), equalTo(true));
   assertThat(allEntries.fields().contains("simple1"), equalTo(true));
   FieldMapper mapper = docMapper.mappers().smartNameFieldMapper("_all");
   assertThat(field.fieldType().omitNorms(), equalTo(true));
   assertThat(
       mapper.queryStringTermQuery(new Term("_all", "foobar")),
       Matchers.instanceOf(AllTermQuery.class));
 }
  @Test
  public void testSimpleAllMappersWithReparse() throws Exception {
    String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
    DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
    String builtMapping = docMapper.mappingSource().string();
    // reparse it
    DocumentMapper builtDocMapper = MapperTestUtils.newParser().parse(builtMapping);
    byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
    Document doc = builtDocMapper.parse(new BytesArray(json)).rootDoc();

    AllField field = (AllField) doc.getField("_all");
    AllEntries allEntries =
        ((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries();
    assertThat(allEntries.fields().size(), equalTo(3));
    assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
    assertThat(allEntries.fields().contains("name.last"), equalTo(true));
    assertThat(allEntries.fields().contains("simple1"), equalTo(true));
    assertThat(field.fieldType().omitNorms(), equalTo(true));
  }
 public void testAllMappersTermQuery() throws Exception {
   String mapping =
       copyToStringFromClasspath(
           "/org/elasticsearch/index/mapper/all/mapping_omit_positions_on_all.json");
   DocumentMapper docMapper =
       createIndex("test").mapperService().documentMapperParser().parse(mapping);
   byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
   Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
   AllField field = (AllField) doc.getField("_all");
   AllEntries allEntries = field.getAllEntries();
   assertThat(allEntries.fields().size(), equalTo(3));
   assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
   assertThat(allEntries.fields().contains("name.last"), equalTo(true));
   assertThat(allEntries.fields().contains("simple1"), equalTo(true));
   AllFieldMapper mapper = docMapper.allFieldMapper();
   assertThat(field.fieldType().omitNorms(), equalTo(false));
   assertThat(
       mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
       Matchers.instanceOf(AllTermQuery.class));
 }
  public void testMultiField_includeInAllSetToFalse() throws IOException {
    String mapping =
        copyToStringFromClasspath(
            "/org/elasticsearch/index/mapper/all/multifield-mapping_include_in_all_set_to_false.json");
    DocumentMapper docMapper =
        createIndex("test").mapperService().documentMapperParser().parse(mapping);

    XContentBuilder builder = XContentFactory.jsonBuilder();
    builder
        .startObject()
        .field("foo")
        .startObject()
        .field("bar", "Elasticsearch rules!")
        .endObject()
        .endObject();

    Document doc = docMapper.parse("test", "test", "1", builder.bytes()).rootDoc();
    AllField field = (AllField) doc.getField("_all");
    AllEntries allEntries = field.getAllEntries();
    assertThat(allEntries.fields(), empty());
  }
  public void testRandom() throws Exception {
    boolean omitNorms = false;
    boolean stored = false;
    boolean enabled = true;
    boolean tv_stored = false;
    boolean tv_payloads = false;
    boolean tv_offsets = false;
    boolean tv_positions = false;
    String similarity = null;
    XContentBuilder mappingBuilder = jsonBuilder();
    mappingBuilder.startObject().startObject("test");
    List<Tuple<String, Boolean>> booleanOptionList = new ArrayList<>();
    boolean allDefault = true;
    if (frequently()) {
      allDefault = false;
      mappingBuilder.startObject("_all");
      if (randomBoolean()) {
        booleanOptionList.add(new Tuple<>("omit_norms", omitNorms = randomBoolean()));
      }
      if (randomBoolean()) {
        booleanOptionList.add(new Tuple<>("store", stored = randomBoolean()));
      }
      if (randomBoolean()) {
        booleanOptionList.add(new Tuple<>("store_term_vectors", tv_stored = randomBoolean()));
      }
      if (randomBoolean()) {
        booleanOptionList.add(new Tuple<>("enabled", enabled = randomBoolean()));
      }
      if (randomBoolean()) {
        booleanOptionList.add(
            new Tuple<>("store_term_vector_offsets", tv_offsets = randomBoolean()));
      }
      if (randomBoolean()) {
        booleanOptionList.add(
            new Tuple<>("store_term_vector_positions", tv_positions = randomBoolean()));
      }
      if (randomBoolean()) {
        booleanOptionList.add(
            new Tuple<>("store_term_vector_payloads", tv_payloads = randomBoolean()));
      }
      Collections.shuffle(booleanOptionList, getRandom());
      for (Tuple<String, Boolean> option : booleanOptionList) {
        mappingBuilder.field(option.v1(), option.v2().booleanValue());
      }
      tv_stored |= tv_positions || tv_payloads || tv_offsets;
      if (randomBoolean()) {
        mappingBuilder.field("similarity", similarity = randomBoolean() ? "BM25" : "TF/IDF");
      }
      mappingBuilder.endObject();
    }

    DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
    String mapping = mappingBuilder.endObject().endObject().bytes().toUtf8();
    logger.info(mapping);
    DocumentMapper docMapper = parser.parse(mapping);
    String builtMapping = docMapper.mappingSource().string();
    // reparse it
    DocumentMapper builtDocMapper = parser.parse(builtMapping);

    byte[] json =
        jsonBuilder()
            .startObject()
            .field("foo", "bar")
            .field("foobar", "foobar")
            .endObject()
            .bytes()
            .toBytes();
    Document doc = builtDocMapper.parse("test", "test", "1", new BytesArray(json)).rootDoc();
    AllField field = (AllField) doc.getField("_all");
    if (enabled) {
      assertThat(field.fieldType().omitNorms(), equalTo(omitNorms));
      assertThat(field.fieldType().stored(), equalTo(stored));
      assertThat(field.fieldType().storeTermVectorOffsets(), equalTo(tv_offsets));
      assertThat(field.fieldType().storeTermVectorPayloads(), equalTo(tv_payloads));
      assertThat(field.fieldType().storeTermVectorPositions(), equalTo(tv_positions));
      assertThat(field.fieldType().storeTermVectors(), equalTo(tv_stored));
      AllEntries allEntries = field.getAllEntries();
      assertThat(allEntries.fields().size(), equalTo(2));
      assertThat(allEntries.fields().contains("foobar"), equalTo(true));
      assertThat(allEntries.fields().contains("foo"), equalTo(true));
      if (!stored) {
        assertThat(field.stringValue(), nullValue());
      }
      String text = stored ? field.stringValue() : "bar foobar";
      assertThat(text.trim(), equalTo(allEntries.buildText().trim()));
    } else {
      assertThat(field, nullValue());
    }
    if (similarity == null || similarity.equals("TF/IDF")) {
      assertThat(builtDocMapper.allFieldMapper().fieldType().similarity(), nullValue());
    } else {
      assertThat(
          similarity, equalTo(builtDocMapper.allFieldMapper().fieldType().similarity().name()));
    }
    if (allDefault) {
      BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(0);
      XContentBuilder b = new XContentBuilder(XContentType.JSON.xContent(), bytesStreamOutput);
      XContentBuilder xContentBuilder =
          builtDocMapper.allFieldMapper().toXContent(b, ToXContent.EMPTY_PARAMS);
      xContentBuilder.flush();
      assertThat(bytesStreamOutput.size(), equalTo(0));
    }
  }