コード例 #1
0
 // #6187: if no fields were boosted, we shouldn't use AllTokenStream
 public void testNoBoost() throws Exception {
   String mapping =
       copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json");
   DocumentMapper docMapper =
       createIndex("test").mapperService().documentMapperParser().parse(mapping);
   byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
   Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
   AllField field = (AllField) doc.getField("_all");
   // no fields have boost, so we should not see AllTokenStream:
   assertThat(
       field.tokenStream(docMapper.mappers().indexAnalyzer(), null),
       Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
 }
コード例 #2
0
 public void testAllMappersNoBoost() throws Exception {
   String mapping =
       copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/noboost-mapping.json");
   IndexService index = createIndex("test");
   DocumentMapper docMapper = index.mapperService().documentMapperParser().parse(mapping);
   byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
   Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
   AllField field = (AllField) doc.getField("_all");
   AllEntries allEntries = field.getAllEntries();
   assertThat(allEntries.fields().size(), equalTo(3));
   assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
   assertThat(allEntries.fields().contains("name.last"), equalTo(true));
   assertThat(allEntries.fields().contains("simple1"), equalTo(true));
   assertThat(field.fieldType().omitNorms(), equalTo(false));
 }
コード例 #3
0
  public void testSimple() throws Exception {
    String mapping =
        copyToStringFromClasspath(
            "/org/elasticsearch/index/mapper/dynamictemplate/genericstore/test-mapping.json");
    IndexService index = createIndex("test");
    client().admin().indices().preparePutMapping("test").setType("person").setSource(mapping).get();
    DocumentMapper docMapper = index.mapperService().documentMapper("person");
    byte[] json =
        copyToBytesFromClasspath(
            "/org/elasticsearch/index/mapper/dynamictemplate/genericstore/test-data.json");
    ParsedDocument parsedDoc = docMapper.parse("test", "person", "1", new BytesArray(json));
    client()
        .admin()
        .indices()
        .preparePutMapping("test")
        .setType("person")
        .setSource(parsedDoc.dynamicMappingsUpdate().toString())
        .get();
    docMapper = index.mapperService().documentMapper("person");
    Document doc = parsedDoc.rootDoc();

    IndexableField f = doc.getField("name");
    assertThat(f.name(), equalTo("name"));
    assertThat(f.stringValue(), equalTo("some name"));
    assertThat(f.fieldType().stored(), equalTo(true));

    FieldMapper fieldMapper = docMapper.mappers().getMapper("name");
    assertThat(fieldMapper.fieldType().stored(), equalTo(true));

    f = doc.getField("age");
    assertThat(f.name(), equalTo("age"));
    assertThat(f.fieldType().stored(), equalTo(true));

    fieldMapper = docMapper.mappers().getMapper("age");
    assertThat(fieldMapper.fieldType().stored(), equalTo(true));
  }
コード例 #4
0
 // #6187: if _all doesn't index positions then we never use AllTokenStream, even if some fields
 // have boost
 public void testBoostWithOmitPositions() throws Exception {
   String mapping =
       copyToStringFromClasspath(
           "/org/elasticsearch/index/mapper/all/mapping_boost_omit_positions_on_all.json");
   DocumentMapper docMapper =
       createIndex("test").mapperService().documentMapperParser().parse(mapping);
   byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
   Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
   AllField field = (AllField) doc.getField("_all");
   // _all field omits positions, so we should not get AllTokenStream even though fields are
   // boosted
   assertThat(
       field.tokenStream(docMapper.mappers().indexAnalyzer(), null),
       Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
 }
コード例 #5
0
  public void testSimpleAllMappersWithStore() throws Exception {
    String mapping =
        copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/store-mapping.json");
    DocumentMapper docMapper =
        createIndex("test").mapperService().documentMapperParser().parse(mapping);
    byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
    Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
    AllField field = (AllField) doc.getField("_all");
    AllEntries allEntries = field.getAllEntries();
    assertThat(allEntries.fields().size(), equalTo(2));
    assertThat(allEntries.fields().contains("name.last"), equalTo(true));
    assertThat(allEntries.fields().contains("simple1"), equalTo(true));

    String text = field.stringValue();
    assertThat(text, equalTo(allEntries.buildText()));
    assertThat(field.fieldType().omitNorms(), equalTo(false));
  }
コード例 #6
0
  public void testSimpleAllMappersWithReparse() throws Exception {
    DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
    String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
    DocumentMapper docMapper = parser.parse(mapping);
    String builtMapping = docMapper.mappingSource().string();
    // reparse it
    DocumentMapper builtDocMapper = parser.parse(builtMapping);
    byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
    Document doc = builtDocMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();

    AllField field = (AllField) doc.getField("_all");
    AllEntries allEntries = field.getAllEntries();
    assertThat(allEntries.fields().toString(), allEntries.fields().size(), equalTo(3));
    assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
    assertThat(allEntries.fields().contains("name.last"), equalTo(true));
    assertThat(allEntries.fields().contains("simple1"), equalTo(true));
    assertThat(field.fieldType().omitNorms(), equalTo(true));
  }
コード例 #7
0
 @Test
 public void testSimpleAllMappers() throws Exception {
   String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
   DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
   byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
   Document doc = docMapper.parse(new BytesArray(json)).rootDoc();
   AllField field = (AllField) doc.getField("_all");
   AllEntries allEntries =
       ((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries();
   assertThat(allEntries.fields().size(), equalTo(3));
   assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
   assertThat(allEntries.fields().contains("name.last"), equalTo(true));
   assertThat(allEntries.fields().contains("simple1"), equalTo(true));
   FieldMapper mapper = docMapper.mappers().smartNameFieldMapper("_all");
   assertThat(field.fieldType().omitNorms(), equalTo(true));
   assertThat(
       mapper.queryStringTermQuery(new Term("_all", "foobar")),
       Matchers.instanceOf(AllTermQuery.class));
 }
コード例 #8
0
  @Test
  public void testSimpleAllMappersWithReparse() throws Exception {
    String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/all/mapping.json");
    DocumentMapper docMapper = MapperTestUtils.newParser().parse(mapping);
    String builtMapping = docMapper.mappingSource().string();
    // reparse it
    DocumentMapper builtDocMapper = MapperTestUtils.newParser().parse(builtMapping);
    byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
    Document doc = builtDocMapper.parse(new BytesArray(json)).rootDoc();

    AllField field = (AllField) doc.getField("_all");
    AllEntries allEntries =
        ((AllTokenStream) field.tokenStream(docMapper.mappers().indexAnalyzer())).allEntries();
    assertThat(allEntries.fields().size(), equalTo(3));
    assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
    assertThat(allEntries.fields().contains("name.last"), equalTo(true));
    assertThat(allEntries.fields().contains("simple1"), equalTo(true));
    assertThat(field.fieldType().omitNorms(), equalTo(true));
  }
コード例 #9
0
 public void testAllMappersTermQuery() throws Exception {
   String mapping =
       copyToStringFromClasspath(
           "/org/elasticsearch/index/mapper/all/mapping_omit_positions_on_all.json");
   DocumentMapper docMapper =
       createIndex("test").mapperService().documentMapperParser().parse(mapping);
   byte[] json = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/all/test1.json");
   Document doc = docMapper.parse("test", "person", "1", new BytesArray(json)).rootDoc();
   AllField field = (AllField) doc.getField("_all");
   AllEntries allEntries = field.getAllEntries();
   assertThat(allEntries.fields().size(), equalTo(3));
   assertThat(allEntries.fields().contains("address.last.location"), equalTo(true));
   assertThat(allEntries.fields().contains("name.last"), equalTo(true));
   assertThat(allEntries.fields().contains("simple1"), equalTo(true));
   AllFieldMapper mapper = docMapper.allFieldMapper();
   assertThat(field.fieldType().omitNorms(), equalTo(false));
   assertThat(
       mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
       Matchers.instanceOf(AllTermQuery.class));
 }
コード例 #10
0
  public void testMultiField_includeInAllSetToFalse() throws IOException {
    String mapping =
        copyToStringFromClasspath(
            "/org/elasticsearch/index/mapper/all/multifield-mapping_include_in_all_set_to_false.json");
    DocumentMapper docMapper =
        createIndex("test").mapperService().documentMapperParser().parse(mapping);

    XContentBuilder builder = XContentFactory.jsonBuilder();
    builder
        .startObject()
        .field("foo")
        .startObject()
        .field("bar", "Elasticsearch rules!")
        .endObject()
        .endObject();

    Document doc = docMapper.parse("test", "test", "1", builder.bytes()).rootDoc();
    AllField field = (AllField) doc.getField("_all");
    AllEntries allEntries = field.getAllEntries();
    assertThat(allEntries.fields(), empty());
  }
コード例 #11
0
  public void testRandom() throws Exception {
    boolean omitNorms = false;
    boolean stored = false;
    boolean enabled = true;
    boolean tv_stored = false;
    boolean tv_payloads = false;
    boolean tv_offsets = false;
    boolean tv_positions = false;
    String similarity = null;
    XContentBuilder mappingBuilder = jsonBuilder();
    mappingBuilder.startObject().startObject("test");
    List<Tuple<String, Boolean>> booleanOptionList = new ArrayList<>();
    boolean allDefault = true;
    if (frequently()) {
      allDefault = false;
      mappingBuilder.startObject("_all");
      if (randomBoolean()) {
        booleanOptionList.add(new Tuple<>("omit_norms", omitNorms = randomBoolean()));
      }
      if (randomBoolean()) {
        booleanOptionList.add(new Tuple<>("store", stored = randomBoolean()));
      }
      if (randomBoolean()) {
        booleanOptionList.add(new Tuple<>("store_term_vectors", tv_stored = randomBoolean()));
      }
      if (randomBoolean()) {
        booleanOptionList.add(new Tuple<>("enabled", enabled = randomBoolean()));
      }
      if (randomBoolean()) {
        booleanOptionList.add(
            new Tuple<>("store_term_vector_offsets", tv_offsets = randomBoolean()));
      }
      if (randomBoolean()) {
        booleanOptionList.add(
            new Tuple<>("store_term_vector_positions", tv_positions = randomBoolean()));
      }
      if (randomBoolean()) {
        booleanOptionList.add(
            new Tuple<>("store_term_vector_payloads", tv_payloads = randomBoolean()));
      }
      Collections.shuffle(booleanOptionList, getRandom());
      for (Tuple<String, Boolean> option : booleanOptionList) {
        mappingBuilder.field(option.v1(), option.v2().booleanValue());
      }
      tv_stored |= tv_positions || tv_payloads || tv_offsets;
      if (randomBoolean()) {
        mappingBuilder.field("similarity", similarity = randomBoolean() ? "BM25" : "TF/IDF");
      }
      mappingBuilder.endObject();
    }

    DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
    String mapping = mappingBuilder.endObject().endObject().bytes().toUtf8();
    logger.info(mapping);
    DocumentMapper docMapper = parser.parse(mapping);
    String builtMapping = docMapper.mappingSource().string();
    // reparse it
    DocumentMapper builtDocMapper = parser.parse(builtMapping);

    byte[] json =
        jsonBuilder()
            .startObject()
            .field("foo", "bar")
            .field("foobar", "foobar")
            .endObject()
            .bytes()
            .toBytes();
    Document doc = builtDocMapper.parse("test", "test", "1", new BytesArray(json)).rootDoc();
    AllField field = (AllField) doc.getField("_all");
    if (enabled) {
      assertThat(field.fieldType().omitNorms(), equalTo(omitNorms));
      assertThat(field.fieldType().stored(), equalTo(stored));
      assertThat(field.fieldType().storeTermVectorOffsets(), equalTo(tv_offsets));
      assertThat(field.fieldType().storeTermVectorPayloads(), equalTo(tv_payloads));
      assertThat(field.fieldType().storeTermVectorPositions(), equalTo(tv_positions));
      assertThat(field.fieldType().storeTermVectors(), equalTo(tv_stored));
      AllEntries allEntries = field.getAllEntries();
      assertThat(allEntries.fields().size(), equalTo(2));
      assertThat(allEntries.fields().contains("foobar"), equalTo(true));
      assertThat(allEntries.fields().contains("foo"), equalTo(true));
      if (!stored) {
        assertThat(field.stringValue(), nullValue());
      }
      String text = stored ? field.stringValue() : "bar foobar";
      assertThat(text.trim(), equalTo(allEntries.buildText().trim()));
    } else {
      assertThat(field, nullValue());
    }
    if (similarity == null || similarity.equals("TF/IDF")) {
      assertThat(builtDocMapper.allFieldMapper().fieldType().similarity(), nullValue());
    } else {
      assertThat(
          similarity, equalTo(builtDocMapper.allFieldMapper().fieldType().similarity().name()));
    }
    if (allDefault) {
      BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(0);
      XContentBuilder b = new XContentBuilder(XContentType.JSON.xContent(), bytesStreamOutput);
      XContentBuilder xContentBuilder =
          builtDocMapper.allFieldMapper().toXContent(b, ToXContent.EMPTY_PARAMS);
      xContentBuilder.flush();
      assertThat(bytesStreamOutput.size(), equalTo(0));
    }
  }
コード例 #12
0
  public void testFieldLevelBoost() throws Exception {
    String mapping =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("person")
            .startObject("properties")
            .startObject("str_field")
            .field("type", "string")
            .endObject()
            .startObject("int_field")
            .field("type", "integer")
            .startObject("norms")
            .field("enabled", true)
            .endObject()
            .endObject()
            .startObject("byte_field")
            .field("type", "byte")
            .startObject("norms")
            .field("enabled", true)
            .endObject()
            .endObject()
            .startObject("date_field")
            .field("type", "date")
            .startObject("norms")
            .field("enabled", true)
            .endObject()
            .endObject()
            .startObject("double_field")
            .field("type", "double")
            .startObject("norms")
            .field("enabled", true)
            .endObject()
            .endObject()
            .startObject("float_field")
            .field("type", "float")
            .startObject("norms")
            .field("enabled", true)
            .endObject()
            .endObject()
            .startObject("long_field")
            .field("type", "long")
            .startObject("norms")
            .field("enabled", true)
            .endObject()
            .endObject()
            .startObject("short_field")
            .field("type", "short")
            .startObject("norms")
            .field("enabled", true)
            .endObject()
            .endObject()
            .string();

    DocumentMapper docMapper =
        createIndex("test")
            .mapperService()
            .documentMapperParser()
            .parse("person", new CompressedXContent(mapping));
    BytesReference json =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("str_field")
            .field("boost", 2.0)
            .field("value", "some name")
            .endObject()
            .startObject("int_field")
            .field("boost", 3.0)
            .field("value", 10)
            .endObject()
            .startObject("byte_field")
            .field("boost", 4.0)
            .field("value", 20)
            .endObject()
            .startObject("date_field")
            .field("boost", 5.0)
            .field("value", "2012-01-10")
            .endObject()
            .startObject("double_field")
            .field("boost", 6.0)
            .field("value", 30.0)
            .endObject()
            .startObject("float_field")
            .field("boost", 7.0)
            .field("value", 40.0)
            .endObject()
            .startObject("long_field")
            .field("boost", 8.0)
            .field("value", 50)
            .endObject()
            .startObject("short_field")
            .field("boost", 9.0)
            .field("value", 60)
            .endObject()
            .bytes();
    Document doc = docMapper.parse("test", "person", "1", json).rootDoc();

    IndexableField f = doc.getField("str_field");
    assertThat((double) f.boost(), closeTo(2.0, 0.001));

    f = doc.getField("int_field");
    assertThat((double) f.boost(), closeTo(3.0, 0.001));

    f = doc.getField("byte_field");
    assertThat((double) f.boost(), closeTo(4.0, 0.001));

    f = doc.getField("date_field");
    assertThat((double) f.boost(), closeTo(5.0, 0.001));

    f = doc.getField("double_field");
    assertThat((double) f.boost(), closeTo(6.0, 0.001));

    f = doc.getField("float_field");
    assertThat((double) f.boost(), closeTo(7.0, 0.001));

    f = doc.getField("long_field");
    assertThat((double) f.boost(), closeTo(8.0, 0.001));

    f = doc.getField("short_field");
    assertThat((double) f.boost(), closeTo(9.0, 0.001));
  }
コード例 #13
0
  public void parse(ParseContext context) throws IOException {
    if (!enabled) {
      context.parser().skipChildren();
      return;
    }
    XContentParser parser = context.parser();

    String currentFieldName = parser.currentName();
    XContentParser.Token token = parser.currentToken();
    if (token == XContentParser.Token.VALUE_NULL) {
      // the object is null ("obj1" : null), simply bail
      return;
    }

    if (token.isValue() && !allowValue()) {
      // if we are parsing an object but it is just a value, its only allowed on root level parsers
      // with there
      // is a field name with the same name as the type
      throw new MapperParsingException(
          "object mapping for [" + name + "] tried to parse as object, but found a concrete value");
    }

    if (nested.isNested()) {
      context = context.createNestedContext(fullPath);
      Document nestedDoc = context.doc();
      Document parentDoc = nestedDoc.getParent();
      // pre add the uid field if possible (id was already provided)
      IndexableField uidField = parentDoc.getField(UidFieldMapper.NAME);
      if (uidField != null) {
        // we don't need to add it as a full uid field in nested docs, since we don't need
        // versioning
        // we also rely on this for UidField#loadVersion

        // this is a deeply nested field
        nestedDoc.add(
            new Field(
                UidFieldMapper.NAME,
                uidField.stringValue(),
                UidFieldMapper.Defaults.NESTED_FIELD_TYPE));
      }
      // the type of the nested doc starts with __, so we can identify that its a nested one in
      // filters
      // note, we don't prefix it with the type of the doc since it allows us to execute a nested
      // query
      // across types (for example, with similar nested objects)
      nestedDoc.add(
          new Field(
              TypeFieldMapper.NAME, nestedTypePathAsString, TypeFieldMapper.Defaults.FIELD_TYPE));
    }

    ContentPath.Type origPathType = context.path().pathType();
    context.path().pathType(pathType);

    // if we are at the end of the previous object, advance
    if (token == XContentParser.Token.END_OBJECT) {
      token = parser.nextToken();
    }
    if (token == XContentParser.Token.START_OBJECT) {
      // if we are just starting an OBJECT, advance, this is the object we are parsing, we need the
      // name first
      token = parser.nextToken();
    }

    while (token != XContentParser.Token.END_OBJECT) {
      if (token == XContentParser.Token.START_OBJECT) {
        serializeObject(context, currentFieldName);
      } else if (token == XContentParser.Token.START_ARRAY) {
        serializeArray(context, currentFieldName);
      } else if (token == XContentParser.Token.FIELD_NAME) {
        currentFieldName = parser.currentName();
      } else if (token == XContentParser.Token.VALUE_NULL) {
        serializeNullValue(context, currentFieldName);
      } else if (token == null) {
        throw new MapperParsingException(
            "object mapping for ["
                + name
                + "] tried to parse as object, but got EOF, has a concrete value been provided to it?");
      } else if (token.isValue()) {
        serializeValue(context, currentFieldName, token);
      }
      token = parser.nextToken();
    }
    // restore the enable path flag
    context.path().pathType(origPathType);
    if (nested.isNested()) {
      Document nestedDoc = context.doc();
      Document parentDoc = nestedDoc.getParent();
      if (nested.isIncludeInParent()) {
        for (IndexableField field : nestedDoc.getFields()) {
          if (field.name().equals(UidFieldMapper.NAME)
              || field.name().equals(TypeFieldMapper.NAME)) {
            continue;
          } else {
            parentDoc.add(field);
          }
        }
      }
      if (nested.isIncludeInRoot()) {
        Document rootDoc = context.rootDoc();
        // don't add it twice, if its included in parent, and we are handling the master doc...
        if (!nested.isIncludeInParent() || parentDoc != rootDoc) {
          for (IndexableField field : nestedDoc.getFields()) {
            if (field.name().equals(UidFieldMapper.NAME)
                || field.name().equals(TypeFieldMapper.NAME)) {
              continue;
            } else {
              rootDoc.add(field);
            }
          }
        }
      }
    }
  }