Ejemplo n.º 1
0
  private ParsedDocument parseFetchedDoc(
      PercolateContext context,
      BytesReference fetchedDoc,
      IndexService documentIndexService,
      String type) {
    ParsedDocument doc = null;
    XContentParser parser = null;
    try {
      parser = XContentFactory.xContent(fetchedDoc).createParser(fetchedDoc);
      MapperService mapperService = documentIndexService.mapperService();
      DocumentMapper docMapper = mapperService.documentMapperWithAutoCreate(type);
      doc = docMapper.parse(source(parser).type(type).flyweight(true));

      if (context.highlight() != null) {
        doc.setSource(fetchedDoc);
      }
    } catch (Throwable e) {
      throw new ElasticsearchParseException("failed to parse request", e);
    } finally {
      if (parser != null) {
        parser.close();
      }
    }

    if (doc == null) {
      throw new ElasticsearchParseException("No doc to percolate in the request");
    }

    return doc;
  }
 public void testNoParentNullFieldCreatedIfNoParentSpecified() throws Exception {
   Index index = new Index("_index", "testUUID");
   IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, Settings.EMPTY);
   NamedAnalyzer namedAnalyzer =
       new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer());
   IndexAnalyzers indexAnalyzers =
       new IndexAnalyzers(
           indexSettings, namedAnalyzer, namedAnalyzer, namedAnalyzer, Collections.emptyMap());
   SimilarityService similarityService =
       new SimilarityService(indexSettings, Collections.emptyMap());
   MapperService mapperService =
       new MapperService(
           indexSettings,
           indexAnalyzers,
           similarityService,
           new IndicesModule(emptyList()).getMapperRegistry(),
           () -> null);
   XContentBuilder mappingSource =
       jsonBuilder()
           .startObject()
           .startObject("some_type")
           .startObject("properties")
           .endObject()
           .endObject()
           .endObject();
   mapperService.merge(
       "some_type",
       new CompressedXContent(mappingSource.string()),
       MergeReason.MAPPING_UPDATE,
       false);
   Set<String> allFields = new HashSet<>(mapperService.simpleMatchToIndexNames("*"));
   assertTrue(allFields.contains("_parent"));
   assertFalse(allFields.contains("_parent#null"));
 }
  @Test
  public void testDefaultMappingAndWithMappingOverrideWithMapperService() throws Exception {
    String defaultMapping =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject(MapperService.DEFAULT_MAPPING)
            .startObject("_source")
            .field("enabled", false)
            .endObject()
            .endObject()
            .endObject()
            .string();

    MapperService mapperService = MapperTests.newMapperService();
    mapperService.add(MapperService.DEFAULT_MAPPING, defaultMapping);

    String mapping =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type")
            .startObject("_source")
            .field("enabled", true)
            .endObject()
            .endObject()
            .endObject()
            .string();
    mapperService.add("my_type", mapping);

    DocumentMapper mapper = mapperService.documentMapper("my_type");
    assertThat(mapper.type(), equalTo("my_type"));
    assertThat(mapper.sourceMapper().enabled(), equalTo(true));
  }
  @Test
  public void testPutMappingWithDocumentMapperBuilder() throws IOException {
    RootObjectMapper.Builder rootObjectMapperBuilder =
        new RootObjectMapper.Builder(INDEX_TYPE)
            .add(new StringFieldMapper.Builder("message_2").store(true));

    GetSettingsResponse getSettingsResponse =
        client()
            .admin()
            .indices()
            .getSettings(new GetSettingsRequest().indices(INDEX_NAME))
            .actionGet();
    MapperService mapperService = getMapperService();
    DocumentMapper documentMapper =
        new DocumentMapper.Builder(
                getSettingsResponse.getIndexToSettings().get(INDEX_NAME),
                rootObjectMapperBuilder,
                mapperService)
            .build(mapperService, mapperService.documentMapperParser());
    String expectedMappingSource = documentMapper.mappingSource().toString();
    PutMapping putMapping =
        new PutMapping.Builder(INDEX_NAME, INDEX_TYPE, expectedMappingSource).build();

    JestResult result = client.execute(putMapping);
    assertTrue(result.getErrorMessage(), result.isSucceeded());
  }
  public void testPercolatorFieldMapper_noQuery() throws Exception {
    addQueryMapping();
    ParsedDocument doc =
        mapperService
            .documentMapper(typeName)
            .parse(
                "test",
                typeName,
                "1",
                XContentFactory.jsonBuilder().startObject().endObject().bytes());
    assertThat(doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName()).length, equalTo(0));

    try {
      mapperService
          .documentMapper(typeName)
          .parse(
              "test",
              typeName,
              "1",
              XContentFactory.jsonBuilder().startObject().nullField(fieldName).endObject().bytes());
    } catch (MapperParsingException e) {
      assertThat(
          e.getDetailedMessage(), containsString("query malformed, must start with start_object"));
    }
  }
 /** Checks the mappings for compatibility with the current version */
 private void checkMappingsCompatibility(IndexMetaData indexMetaData) {
   Index index = new Index(indexMetaData.getIndex());
   Settings settings = indexMetaData.getSettings();
   try {
     SimilarityService similarityService = new SimilarityService(index, settings);
     // We cannot instantiate real analysis server at this point because the node might not have
     // been started yet. However, we don't really need real analyzers at this stage - so we can
     // fake it
     try (AnalysisService analysisService = new FakeAnalysisService(index, settings)) {
       try (MapperService mapperService =
           new MapperService(index, settings, analysisService, similarityService, scriptService)) {
         for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
           MappingMetaData mappingMetaData = cursor.value;
           mapperService.merge(mappingMetaData.type(), mappingMetaData.source(), false, false);
         }
       }
     }
   } catch (Exception ex) {
     // Wrap the inner exception so we have the index name in the exception message
     throw new IllegalStateException(
         "unable to upgrade the mappings for the index ["
             + indexMetaData.getIndex()
             + "], reason: ["
             + ex.getMessage()
             + "]",
         ex);
   }
 }
  public void testNestedRangeQuery() throws IOException {
    // create a nested geo_point type with a subfield named "geohash" (explicit testing for ISSUE
    // #15179)
    MapperService mapperService = createShardContext().getMapperService();
    String nestedMapping =
        "{\"nested_doc\" : {\"properties\" : {"
            + "\"locations\": {\"properties\": {"
            + "\"geohash\": {\"type\": \"geo_point\"}},"
            + "\"type\": \"nested\"}"
            + "}}}";
    mapperService.merge(
        "nested_doc",
        new CompressedXContent(nestedMapping),
        MapperService.MergeReason.MAPPING_UPDATE,
        false);

    // create a range query on the nested locations.geohash sub-field
    String queryJson =
        "{\n"
            + "  \"nested\": {\n"
            + "    \"path\": \"locations\",\n"
            + "    \"query\": {\n"
            + "      \"geo_distance_range\": {\n"
            + "        \"from\": \"0.0km\",\n"
            + "        \"to\" : \"200.0km\",\n"
            + "        \"locations.geohash\": \"s7ws01wyd7ws\"\n"
            + "      }\n"
            + "    }\n"
            + "  }\n"
            + "}\n";
    NestedQueryBuilder builder = (NestedQueryBuilder) parseQuery(queryJson);
    QueryShardContext context = createShardContext();
    builder.toQuery(context);
  }
  public void testConflictNewType() throws Exception {
    XContentBuilder mapping =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type1")
            .startObject("properties")
            .startObject("foo")
            .field("type", "long")
            .endObject()
            .endObject()
            .endObject()
            .endObject();
    MapperService mapperService =
        createIndex("test", Settings.settingsBuilder().build(), "type1", mapping).mapperService();

    XContentBuilder update =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type2")
            .startObject("properties")
            .startObject("foo")
            .field("type", "double")
            .endObject()
            .endObject()
            .endObject()
            .endObject();

    try {
      mapperService.merge(
          "type2",
          new CompressedXContent(update.string()),
          MapperService.MergeReason.MAPPING_UPDATE,
          false);
      fail();
    } catch (IllegalArgumentException e) {
      // expected
      assertTrue(
          e.getMessage(),
          e.getMessage().contains("mapper [foo] cannot be changed from type [long] to [double]"));
    }

    try {
      mapperService.merge(
          "type2",
          new CompressedXContent(update.string()),
          MapperService.MergeReason.MAPPING_UPDATE,
          false);
      fail();
    } catch (IllegalArgumentException e) {
      // expected
      assertTrue(
          e.getMessage(),
          e.getMessage().contains("mapper [foo] cannot be changed from type [long] to [double]"));
    }

    assertTrue(
        mapperService.documentMapper("type1").mapping().root().getMapper("foo")
            instanceof LongFieldMapper);
    assertNull(mapperService.documentMapper("type2"));
  }
 static SearchContext createSearchContext(String indexName, String parentType, String childType)
     throws IOException {
   Settings settings =
       Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_6_0).build();
   IndexService indexService = createIndex(indexName, settings);
   MapperService mapperService = indexService.mapperService();
   // Parent/child parsers require that the parent and child type to be presented in mapping
   // Sometimes we want a nested object field in the parent type that triggers nonNestedDocsFilter
   // to be used
   mapperService.merge(
       parentType,
       new CompressedXContent(
           PutMappingRequest.buildFromSimplifiedDef(
                   parentType,
                   "nested_field",
                   random().nextBoolean() ? "type=nested" : "type=object")
               .string()),
       true,
       false);
   mapperService.merge(
       childType,
       new CompressedXContent(
           PutMappingRequest.buildFromSimplifiedDef(
                   childType,
                   "_parent",
                   "type=" + parentType,
                   CHILD_SCORE_NAME,
                   "type=double,doc_values=false")
               .string()),
       true,
       false);
   return createSearchContext(indexService);
 }
  public void testAllowNoAdditionalSettings() throws Exception {
    addQueryMapping();
    IndexService indexService = createIndex("test1", Settings.EMPTY);
    MapperService mapperService = indexService.mapperService();

    String percolatorMapper =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject(typeName)
            .startObject("properties")
            .startObject(fieldName)
            .field("type", "percolator")
            .field("index", "no")
            .endObject()
            .endObject()
            .endObject()
            .endObject()
            .string();
    try {
      mapperService.merge(
          typeName,
          new CompressedXContent(percolatorMapper),
          MapperService.MergeReason.MAPPING_UPDATE,
          true);
      fail("MapperParsingException expected");
    } catch (MapperParsingException e) {
      assertThat(
          e.getMessage(),
          equalTo(
              "Mapping definition for ["
                  + fieldName
                  + "] has unsupported parameters:  [index : no]"));
    }
  }
Ejemplo n.º 11
0
  public void testConflictSameType() throws Exception {
    XContentBuilder mapping =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type")
            .startObject("properties")
            .startObject("foo")
            .field("type", "long")
            .endObject()
            .endObject()
            .endObject()
            .endObject();
    MapperService mapperService =
        createIndex("test", Settings.settingsBuilder().build(), "type", mapping).mapperService();

    XContentBuilder update =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type")
            .startObject("properties")
            .startObject("foo")
            .field("type", "double")
            .endObject()
            .endObject()
            .endObject()
            .endObject();

    try {
      mapperService.merge(
          "type",
          new CompressedXContent(update.string()),
          MapperService.MergeReason.MAPPING_UPDATE,
          false);
      fail();
    } catch (IllegalArgumentException e) {
      assertThat(
          e.getMessage(),
          containsString(
              "mapper [foo] of different type, current_type [long], merged_type [double]"));
    }

    try {
      mapperService.merge(
          "type",
          new CompressedXContent(update.string()),
          MapperService.MergeReason.MAPPING_UPDATE,
          false);
      fail();
    } catch (IllegalArgumentException e) {
      assertThat(
          e.getMessage(),
          containsString(
              "mapper [foo] of different type, current_type [long], merged_type [double]"));
    }

    assertTrue(
        mapperService.documentMapper("type").mapping().root().getMapper("foo")
            instanceof LongFieldMapper);
  }
Ejemplo n.º 12
0
    public Builder(
        String index,
        Settings indexSettings,
        RootObjectMapper.Builder builder,
        MapperService mapperService) {
      this.index = index;
      this.indexSettings = indexSettings;
      this.builderContext = new Mapper.BuilderContext(indexSettings, new ContentPath(1));
      this.rootObjectMapper = builder.build(builderContext);

      // TODO: find a cleaner way to handle existing root mappings and using their field type as the
      // default.
      // the vast majority of these root mappers only need the existing type for backwards
      // compatibility, since
      // the pre 2.0 field type settings could be modified

      // UID first so it will be the first stored field to load (so will benefit from "fields: []"
      // early termination
      this.rootMappers.put(
          UidFieldMapper.class,
          new UidFieldMapper(indexSettings, mapperService.fullName(UidFieldMapper.NAME)));
      this.rootMappers.put(
          IdFieldMapper.class,
          new IdFieldMapper(indexSettings, mapperService.fullName(IdFieldMapper.NAME)));
      this.rootMappers.put(
          RoutingFieldMapper.class,
          new RoutingFieldMapper(indexSettings, mapperService.fullName(RoutingFieldMapper.NAME)));
      // add default mappers, order is important (for example analyzer should come before the rest
      // to set context.analyzer)
      this.rootMappers.put(
          SizeFieldMapper.class,
          new SizeFieldMapper(indexSettings, mapperService.fullName(SizeFieldMapper.NAME)));
      this.rootMappers.put(
          IndexFieldMapper.class,
          new IndexFieldMapper(indexSettings, mapperService.fullName(IndexFieldMapper.NAME)));
      this.rootMappers.put(SourceFieldMapper.class, new SourceFieldMapper(indexSettings));
      this.rootMappers.put(
          TypeFieldMapper.class,
          new TypeFieldMapper(indexSettings, mapperService.fullName(TypeFieldMapper.NAME)));
      this.rootMappers.put(
          AllFieldMapper.class,
          new AllFieldMapper(indexSettings, mapperService.fullName(AllFieldMapper.NAME)));
      this.rootMappers.put(
          TimestampFieldMapper.class,
          new TimestampFieldMapper(
              indexSettings, mapperService.fullName(TimestampFieldMapper.NAME)));
      this.rootMappers.put(TTLFieldMapper.class, new TTLFieldMapper(indexSettings));
      this.rootMappers.put(VersionFieldMapper.class, new VersionFieldMapper(indexSettings));
      this.rootMappers.put(
          ParentFieldMapper.class,
          new ParentFieldMapper(indexSettings, mapperService.fullName(ParentFieldMapper.NAME)));
      // _field_names last so that it can see all other fields
      this.rootMappers.put(
          FieldNamesFieldMapper.class,
          new FieldNamesFieldMapper(
              indexSettings, mapperService.fullName(FieldNamesFieldMapper.NAME)));
    }
 @Override
 protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
   mapperService.merge(
       PARENT_TYPE,
       new CompressedXContent(
           PutMappingRequest.buildFromSimplifiedDef(
                   PARENT_TYPE,
                   STRING_FIELD_NAME,
                   "type=text",
                   STRING_FIELD_NAME_2,
                   "type=keyword",
                   INT_FIELD_NAME,
                   "type=integer",
                   DOUBLE_FIELD_NAME,
                   "type=double",
                   BOOLEAN_FIELD_NAME,
                   "type=boolean",
                   DATE_FIELD_NAME,
                   "type=date",
                   OBJECT_FIELD_NAME,
                   "type=object")
               .string()),
       MapperService.MergeReason.MAPPING_UPDATE,
       false);
   mapperService.merge(
       CHILD_TYPE,
       new CompressedXContent(
           PutMappingRequest.buildFromSimplifiedDef(
                   CHILD_TYPE,
                   "_parent",
                   "type=" + PARENT_TYPE,
                   STRING_FIELD_NAME,
                   "type=text",
                   STRING_FIELD_NAME_2,
                   "type=keyword",
                   INT_FIELD_NAME,
                   "type=integer",
                   DOUBLE_FIELD_NAME,
                   "type=double",
                   BOOLEAN_FIELD_NAME,
                   "type=boolean",
                   DATE_FIELD_NAME,
                   "type=date",
                   OBJECT_FIELD_NAME,
                   "type=object")
               .string()),
       MapperService.MergeReason.MAPPING_UPDATE,
       false);
   mapperService.merge(
       "just_a_type",
       new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("just_a_type").string()),
       MapperService.MergeReason.MAPPING_UPDATE,
       false);
 }
  public void testPercolatorFieldMapper() throws Exception {
    addQueryMapping();
    QueryBuilder queryBuilder = termQuery("field", "value");
    ParsedDocument doc =
        mapperService
            .documentMapper(typeName)
            .parse(
                "test",
                typeName,
                "1",
                XContentFactory.jsonBuilder()
                    .startObject()
                    .field(fieldName, queryBuilder)
                    .endObject()
                    .bytes());

    assertThat(doc.rootDoc().getFields(fieldType.getExtractedTermsField()).length, equalTo(1));
    assertThat(
        doc.rootDoc().getFields(fieldType.getExtractedTermsField())[0].binaryValue().utf8ToString(),
        equalTo("field\0value"));
    assertThat(doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName()).length, equalTo(1));
    assertThat(
        doc.rootDoc().getFields(fieldType.getExtractionResultFieldName()).length, equalTo(1));
    assertThat(
        doc.rootDoc().getFields(fieldType.getExtractionResultFieldName())[0].stringValue(),
        equalTo(EXTRACTION_COMPLETE));
    BytesRef qbSource =
        doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName())[0].binaryValue();
    assertQueryBuilder(qbSource, queryBuilder);

    // add an query for which we don't extract terms from
    queryBuilder = rangeQuery("field").from("a").to("z");
    doc =
        mapperService
            .documentMapper(typeName)
            .parse(
                "test",
                typeName,
                "1",
                XContentFactory.jsonBuilder()
                    .startObject()
                    .field(fieldName, queryBuilder)
                    .endObject()
                    .bytes());
    assertThat(
        doc.rootDoc().getFields(fieldType.getExtractionResultFieldName()).length, equalTo(1));
    assertThat(
        doc.rootDoc().getFields(fieldType.getExtractionResultFieldName())[0].stringValue(),
        equalTo(EXTRACTION_FAILED));
    assertThat(doc.rootDoc().getFields(fieldType.getExtractedTermsField()).length, equalTo(0));
    assertThat(doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName()).length, equalTo(1));
    qbSource = doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName())[0].binaryValue();
    assertQueryBuilder(qbSource, queryBuilder);
  }
  private void updateMappingOnMaster(final String index, final String type) {
    try {
      MapperService mapperService = indicesService.indexServiceSafe(index).mapperService();
      final DocumentMapper documentMapper = mapperService.documentMapper(type);
      if (documentMapper == null) { // should not happen
        return;
      }
      IndexMetaData metaData = clusterService.state().metaData().index(index);
      if (metaData == null) {
        return;
      }

      long orderId = mappingUpdatedAction.generateNextMappingUpdateOrder();
      documentMapper.refreshSource();

      DiscoveryNode node = clusterService.localNode();
      final MappingUpdatedAction.MappingUpdatedRequest request =
          new MappingUpdatedAction.MappingUpdatedRequest(
              index,
              metaData.uuid(),
              type,
              documentMapper.mappingSource(),
              orderId,
              node != null ? node.id() : null);
      mappingUpdatedAction.execute(
          request,
          new ActionListener<MappingUpdatedAction.MappingUpdatedResponse>() {
            @Override
            public void onResponse(
                MappingUpdatedAction.MappingUpdatedResponse mappingUpdatedResponse) {
              // all is well
            }

            @Override
            public void onFailure(Throwable e) {
              try {
                logger.warn(
                    "failed to update master on updated mapping for index [{}], type [{}] and source [{}]",
                    e,
                    index,
                    type,
                    documentMapper.mappingSource().string());
              } catch (IOException e1) {
                // ignore
              }
            }
          });
    } catch (Exception e) {
      logger.warn(
          "failed to update master on updated mapping for index [{}], type [{}]", e, index, type);
    }
  }
  private ParsedDocument parseDocument(String index, String type, BytesReference doc) {
    MapperService mapperService = indexShard.mapperService();
    IndexService indexService = indexShard.indexService();

    // TODO: make parsing not dynamically create fields not in the original mapping
    Tuple<DocumentMapper, Boolean> docMapper = mapperService.documentMapperWithAutoCreate(type);
    ParsedDocument parsedDocument =
        docMapper.v1().parse(source(doc).type(type).flyweight(true)).setMappingsModified(docMapper);
    if (parsedDocument.mappingsModified()) {
      mappingUpdatedAction.updateMappingOnMaster(index, docMapper.v1(), indexService.indexUUID());
    }
    return parsedDocument;
  }
 private Analyzer getAnalyzerAtField(
     String field, @Nullable Map<String, String> perFieldAnalyzer) {
   MapperService mapperService = indexShard.mapperService();
   Analyzer analyzer;
   if (perFieldAnalyzer != null && perFieldAnalyzer.containsKey(field)) {
     analyzer = mapperService.analysisService().analyzer(perFieldAnalyzer.get(field).toString());
   } else {
     analyzer = mapperService.smartNameFieldMapper(field).indexAnalyzer();
   }
   if (analyzer == null) {
     analyzer = mapperService.analysisService().defaultIndexAnalyzer();
   }
   return analyzer;
 }
  // multiple percolator fields are allowed in the mapping, but only one field can be used at index
  // time.
  public void testMultiplePercolatorFields() throws Exception {
    String typeName = "another_type";
    String percolatorMapper =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject(typeName)
            .startObject("_field_names")
            .field("enabled", false)
            .endObject() // makes testing easier
            .startObject("properties")
            .startObject("query_field1")
            .field("type", "percolator")
            .endObject()
            .startObject("query_field2")
            .field("type", "percolator")
            .endObject()
            .endObject()
            .endObject()
            .endObject()
            .string();
    mapperService.merge(
        typeName,
        new CompressedXContent(percolatorMapper),
        MapperService.MergeReason.MAPPING_UPDATE,
        true);

    QueryBuilder queryBuilder = matchQuery("field", "value");
    ParsedDocument doc =
        mapperService
            .documentMapper(typeName)
            .parse(
                "test",
                typeName,
                "1",
                jsonBuilder()
                    .startObject()
                    .field("query_field1", queryBuilder)
                    .field("query_field2", queryBuilder)
                    .endObject()
                    .bytes());
    assertThat(
        doc.rootDoc().getFields().size(),
        equalTo(11)); // also includes _uid (1), type (2), source (1)
    BytesRef queryBuilderAsBytes =
        doc.rootDoc().getField("query_field1.query_builder_field").binaryValue();
    assertQueryBuilder(queryBuilderAsBytes, queryBuilder);

    queryBuilderAsBytes = doc.rootDoc().getField("query_field2.query_builder_field").binaryValue();
    assertQueryBuilder(queryBuilderAsBytes, queryBuilder);
  }
Ejemplo n.º 19
0
    @Override
    public TerminationHandle warmReader(
        final IndexShard indexShard, final Engine.Searcher searcher) {
      final MapperService mapperService = indexShard.mapperService();
      final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
      for (DocumentMapper docMapper : mapperService.docMappers(false)) {
        for (FieldMapper fieldMapper : docMapper.mappers()) {
          final MappedFieldType fieldType = fieldMapper.fieldType();
          final String indexName = fieldType.name();
          if (fieldType.eagerGlobalOrdinals() == false) {
            continue;
          }
          warmUpGlobalOrdinals.put(indexName, fieldType);
        }
      }
      final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
      final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size());
      for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) {
        executor.execute(
            () -> {
              try {
                final long start = System.nanoTime();
                IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType);
                DirectoryReader reader = searcher.getDirectoryReader();
                IndexFieldData<?> global = ifd.loadGlobal(reader);
                if (reader.leaves().isEmpty() == false) {
                  global.load(reader.leaves().get(0));
                }

                if (indexShard.warmerService().logger().isTraceEnabled()) {
                  indexShard
                      .warmerService()
                      .logger()
                      .trace(
                          "warmed global ordinals for [{}], took [{}]",
                          fieldType.name(),
                          TimeValue.timeValueNanos(System.nanoTime() - start));
                }
              } catch (Exception e) {
                indexShard
                    .warmerService()
                    .logger()
                    .warn("failed to warm-up global ordinals for [{}]", e, fieldType.name());
              } finally {
                latch.countDown();
              }
            });
      }
      return () -> latch.await();
    }
 @Override
 protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
   mapperService.merge(
       "nested_doc",
       new CompressedXContent(
           PutMappingRequest.buildFromSimplifiedDef(
                   "nested_doc",
                   STRING_FIELD_NAME,
                   "type=text",
                   INT_FIELD_NAME,
                   "type=integer",
                   DOUBLE_FIELD_NAME,
                   "type=double",
                   BOOLEAN_FIELD_NAME,
                   "type=boolean",
                   DATE_FIELD_NAME,
                   "type=date",
                   OBJECT_FIELD_NAME,
                   "type=object",
                   GEO_POINT_FIELD_NAME,
                   "type=geo_point",
                   "nested1",
                   "type=nested")
               .string()),
       MapperService.MergeReason.MAPPING_UPDATE,
       false);
 }
Ejemplo n.º 21
0
 ObjectMapper(
     String name,
     String fullPath,
     boolean enabled,
     Nested nested,
     Dynamic dynamic,
     ContentPath.Type pathType,
     Map<String, Mapper> mappers,
     @Nullable @IndexSettings Settings settings) {
   this.name = name;
   this.fullPath = fullPath;
   this.enabled = enabled;
   this.nested = nested;
   this.dynamic = dynamic;
   this.pathType = pathType;
   this.mappers = UpdateInPlaceMap.of(MapperService.getFieldMappersCollectionSwitch(settings));
   if (mappers != null) {
     UpdateInPlaceMap<String, Mapper>.Mutator mappersMutator = this.mappers.mutator();
     mappersMutator.putAll(mappers);
     mappersMutator.close();
   }
   this.nestedTypePathAsString = "__" + fullPath;
   this.nestedTypePathAsBytes = new BytesRef(nestedTypePathAsString);
   this.nestedTypeFilter = new TermFilter(new Term(TypeFieldMapper.NAME, nestedTypePathAsBytes));
 }
Ejemplo n.º 22
0
 private FieldLookup loadFieldData(String name) {
   FieldLookup data = cachedFieldData.get(name);
   if (data == null) {
     MappedFieldType fieldType = mapperService.smartNameFieldType(name, types);
     if (fieldType == null) {
       throw new IllegalArgumentException(
           "No field found for ["
               + name
               + "] in mapping with types "
               + Arrays.toString(types)
               + "");
     }
     data = new FieldLookup(fieldType);
     cachedFieldData.put(name, data);
   }
   if (data.fields() == null) {
     String fieldName = data.fieldType().names().indexName();
     fieldVisitor.reset(fieldName);
     try {
       reader.document(docId, fieldVisitor);
       fieldVisitor.postProcess(data.fieldType());
       data.fields(
           ImmutableMap.of(name, fieldVisitor.fields().get(data.fieldType().names().indexName())));
     } catch (IOException e) {
       throw new ElasticsearchParseException("failed to load field [{}]", e, name);
     }
   }
   return data;
 }
  @Before
  public void init() throws Exception {
    indexService = createIndex("test", Settings.EMPTY);
    mapperService = indexService.mapperService();

    String mapper =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type")
            .startObject("_field_names")
            .field("enabled", false)
            .endObject() // makes testing easier
            .startObject("properties")
            .startObject("field")
            .field("type", "text")
            .endObject()
            .startObject("number_field")
            .field("type", "long")
            .endObject()
            .startObject("date_field")
            .field("type", "date")
            .endObject()
            .endObject()
            .endObject()
            .endObject()
            .string();
    mapperService.merge(
        "type", new CompressedXContent(mapper), MapperService.MergeReason.MAPPING_UPDATE, true);
  }
 private IndexShard newShard(
     boolean primary, DiscoveryNode node, IndexMetaData indexMetaData, Path homePath)
     throws IOException {
   // add node name to settings for propper logging
   final Settings nodeSettings = Settings.builder().put("node.name", node.getName()).build();
   final IndexSettings indexSettings = new IndexSettings(indexMetaData, nodeSettings);
   ShardRouting shardRouting =
       TestShardRouting.newShardRouting(
           shardId, node.getId(), primary, ShardRoutingState.INITIALIZING);
   final Path path = Files.createDirectories(homePath.resolve(node.getId()));
   final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(path);
   ShardPath shardPath =
       new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
   Store store = createStore(indexSettings, shardPath);
   IndexCache indexCache =
       new IndexCache(indexSettings, new DisabledQueryCache(indexSettings), null);
   MapperService mapperService =
       MapperTestUtils.newMapperService(homePath, indexSettings.getSettings());
   for (Map.Entry<String, String> type : indexMapping.entrySet()) {
     mapperService.merge(
         type.getKey(),
         new CompressedXContent(type.getValue()),
         MapperService.MergeReason.MAPPING_RECOVERY,
         true);
   }
   SimilarityService similarityService =
       new SimilarityService(indexSettings, Collections.emptyMap());
   final IndexEventListener indexEventListener = new IndexEventListener() {};
   final Engine.Warmer warmer = searcher -> {};
   return new IndexShard(
       shardRouting,
       indexSettings,
       shardPath,
       store,
       indexCache,
       mapperService,
       similarityService,
       null,
       null,
       indexEventListener,
       null,
       threadPool,
       BigArrays.NON_RECYCLING_INSTANCE,
       warmer,
       Collections.emptyList(),
       Collections.emptyList());
 }
 public void setUp() throws Exception {
   super.setUp();
   MapperService mapperService = queryParserService().mapperService;
   mapperService.merge(
       PARENT_TYPE,
       new CompressedXContent(
           PutMappingRequest.buildFromSimplifiedDef(
                   PARENT_TYPE,
                   STRING_FIELD_NAME,
                   "type=string",
                   INT_FIELD_NAME,
                   "type=integer",
                   DOUBLE_FIELD_NAME,
                   "type=double",
                   BOOLEAN_FIELD_NAME,
                   "type=boolean",
                   DATE_FIELD_NAME,
                   "type=date",
                   OBJECT_FIELD_NAME,
                   "type=object")
               .string()),
       false,
       false);
   mapperService.merge(
       CHILD_TYPE,
       new CompressedXContent(
           PutMappingRequest.buildFromSimplifiedDef(
                   CHILD_TYPE,
                   "_parent",
                   "type=" + PARENT_TYPE,
                   STRING_FIELD_NAME,
                   "type=string",
                   INT_FIELD_NAME,
                   "type=integer",
                   DOUBLE_FIELD_NAME,
                   "type=double",
                   BOOLEAN_FIELD_NAME,
                   "type=boolean",
                   DATE_FIELD_NAME,
                   "type=date",
                   OBJECT_FIELD_NAME,
                   "type=object")
               .string()),
       false,
       false);
 }
Ejemplo n.º 26
0
 public void testDefaultFloatingPointMappings() throws IOException {
   MapperService mapperService = createIndex("test").mapperService();
   String mapping =
       jsonBuilder()
           .startObject()
           .startObject("type")
           .field("numeric_detection", true)
           .endObject()
           .endObject()
           .string();
   mapperService.merge(
       "type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE, false);
   DocumentMapper mapper = mapperService.documentMapper("type");
   doTestDefaultFloatingPointMappings(mapper, XContentFactory.jsonBuilder());
   doTestDefaultFloatingPointMappings(mapper, XContentFactory.yamlBuilder());
   doTestDefaultFloatingPointMappings(mapper, XContentFactory.smileBuilder());
   doTestDefaultFloatingPointMappings(mapper, XContentFactory.cborBuilder());
 }
 @Override
 public PostingsFormat getPostingsFormatForField(String field) {
   final FieldMappers indexName = mapperService.indexName(field);
   if (indexName == null) {
     logger.warn("no index mapper found for field: [{}] returning default postings format", field);
     return defaultPostingFormat;
   }
   PostingsFormatProvider postingsFormat = indexName.mapper().postingsFormatProvider();
   return postingsFormat != null ? postingsFormat.get() : defaultPostingFormat;
 }
  public void testNestedRangeQuery() throws IOException {
    // geo distance range queries are no longer supported in 5.0 they are replaced by using
    // aggregations or sort
    if (createShardContext()
        .indexVersionCreated()
        .onOrAfter(LatLonPointFieldMapper.LAT_LON_FIELD_VERSION)) {
      return;
    }

    // create a nested geo_point type with a subfield named "geohash" (explicit testing for ISSUE
    // #15179)
    MapperService mapperService = createShardContext().getMapperService();
    String nestedMapping =
        "{\"nested_doc\" : {\"properties\" : {"
            + "\"locations\": {\"properties\": {"
            + "\"geohash\": {\"type\": \"geo_point\"}},"
            + "\"type\": \"nested\"}"
            + "}}}";
    mapperService.merge(
        "nested_doc",
        new CompressedXContent(nestedMapping),
        MapperService.MergeReason.MAPPING_UPDATE,
        false);

    // create a range query on the nested locations.geohash sub-field
    String queryJson =
        "{\n"
            + "  \"nested\": {\n"
            + "    \"path\": \"locations\",\n"
            + "    \"query\": {\n"
            + "      \"geo_distance_range\": {\n"
            + "        \"from\": \"0.0km\",\n"
            + "        \"to\" : \"200.0km\",\n"
            + "        \"locations.geohash\": \"s7ws01wyd7ws\"\n"
            + "      }\n"
            + "    }\n"
            + "  }\n"
            + "}\n";
    NestedQueryBuilder builder = (NestedQueryBuilder) parseQuery(queryJson);
    QueryShardContext context = createShardContext();
    builder.toQuery(context);
  }
 @Override
 public DocValuesFormat getDocValuesFormatForField(String field) {
   final FieldMappers indexName = mapperService.indexName(field);
   if (indexName == null) {
     logger.warn(
         "no index mapper found for field: [{}] returning default doc values format", field);
     return defaultDocValuesFormat;
   }
   DocValuesFormatProvider docValuesFormat = indexName.mapper().docValuesFormatProvider();
   return docValuesFormat != null ? docValuesFormat.get() : defaultDocValuesFormat;
 }
Ejemplo n.º 30
0
  private void addMappers(
      Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers) {
    assert mappingLock.isWriteLockedByCurrentThread();
    // first ensure we don't have any incompatible new fields
    mapperService.checkNewMappersCompatibility(objectMappers, fieldMappers, true);

    // update mappers for this document type
    MapBuilder<String, ObjectMapper> builder = MapBuilder.newMapBuilder(this.objectMappers);
    for (ObjectMapper objectMapper : objectMappers) {
      builder.put(objectMapper.fullPath(), objectMapper);
      if (objectMapper.nested().isNested()) {
        hasNestedObjects = true;
      }
    }
    this.objectMappers = builder.immutableMap();
    this.fieldMappers = this.fieldMappers.copyAndAllAll(fieldMappers);

    // finally update for the entire index
    mapperService.addMappers(objectMappers, fieldMappers);
  }