Exemplo n.º 1
0
    @Override
    public TerminationHandle warmNewReaders(
        final IndexShard indexShard,
        IndexMetaData indexMetaData,
        final WarmerContext context,
        ThreadPool threadPool) {
      final Loading defaultLoading =
          Loading.parse(indexMetaData.settings().get(NORMS_LOADING_KEY), Loading.LAZY);
      final MapperService mapperService = indexShard.mapperService();
      final ObjectSet<String> warmUp = new ObjectHashSet<>();
      for (DocumentMapper docMapper : mapperService.docMappers(false)) {
        for (FieldMapper fieldMapper : docMapper.mappers()) {
          final String indexName = fieldMapper.fieldType().names().indexName();
          Loading normsLoading = fieldMapper.fieldType().normsLoading();
          if (normsLoading == null) {
            normsLoading = defaultLoading;
          }
          if (fieldMapper.fieldType().indexOptions() != IndexOptions.NONE
              && !fieldMapper.fieldType().omitNorms()
              && normsLoading == Loading.EAGER) {
            warmUp.add(indexName);
          }
        }
      }

      final CountDownLatch latch = new CountDownLatch(1);
      // Norms loading may be I/O intensive but is not CPU intensive, so we execute it in a single
      // task
      threadPool
          .executor(executor())
          .execute(
              new Runnable() {
                @Override
                public void run() {
                  try {
                    for (ObjectCursor<String> stringObjectCursor : warmUp) {
                      final String indexName = stringObjectCursor.value;
                      final long start = System.nanoTime();
                      for (final LeafReaderContext ctx : context.searcher().reader().leaves()) {
                        final NumericDocValues values = ctx.reader().getNormValues(indexName);
                        if (values != null) {
                          values.get(0);
                        }
                      }
                      if (indexShard.warmerService().logger().isTraceEnabled()) {
                        indexShard
                            .warmerService()
                            .logger()
                            .trace(
                                "warmed norms for [{}], took [{}]",
                                indexName,
                                TimeValue.timeValueNanos(System.nanoTime() - start));
                      }
                    }
                  } catch (Throwable t) {
                    indexShard.warmerService().logger().warn("failed to warm-up norms", t);
                  } finally {
                    latch.countDown();
                  }
                }
              });

      return new TerminationHandle() {
        @Override
        public void awaitTermination() throws InterruptedException {
          latch.await();
        }
      };
    }
Exemplo n.º 2
0
  public static void parseField(
      FieldMapper.Builder builder,
      String name,
      Map<String, Object> fieldNode,
      Mapper.TypeParser.ParserContext parserContext) {
    NamedAnalyzer indexAnalyzer = builder.fieldType().indexAnalyzer();
    NamedAnalyzer searchAnalyzer = builder.fieldType().searchAnalyzer();
    Version indexVersionCreated = parserContext.indexVersionCreated();
    for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator();
        iterator.hasNext(); ) {
      Map.Entry<String, Object> entry = iterator.next();
      final String propName = Strings.toUnderscoreCase(entry.getKey());
      final Object propNode = entry.getValue();
      if (propName.equals("index_name") && indexVersionCreated.before(Version.V_2_0_0_beta1)) {
        builder.indexName(propNode.toString());
        iterator.remove();
      } else if (propName.equals("store")) {
        builder.store(parseStore(name, propNode.toString()));
        iterator.remove();
      } else if (propName.equals("index")) {
        parseIndex(name, propNode.toString(), builder);
        iterator.remove();
      } else if (propName.equals("tokenized")) {
        builder.tokenized(nodeBooleanValue(propNode));
        iterator.remove();
      } else if (propName.equals(DOC_VALUES)) {
        builder.docValues(nodeBooleanValue(propNode));
        iterator.remove();
      } else if (propName.equals("term_vector")) {
        parseTermVector(name, propNode.toString(), builder);
        iterator.remove();
      } else if (propName.equals("boost")) {
        builder.boost(nodeFloatValue(propNode));
        iterator.remove();
      } else if (propName.equals("store_term_vectors")) {
        builder.storeTermVectors(nodeBooleanValue(propNode));
        iterator.remove();
      } else if (propName.equals("store_term_vector_offsets")) {
        builder.storeTermVectorOffsets(nodeBooleanValue(propNode));
        iterator.remove();
      } else if (propName.equals("store_term_vector_positions")) {
        builder.storeTermVectorPositions(nodeBooleanValue(propNode));
        iterator.remove();
      } else if (propName.equals("store_term_vector_payloads")) {
        builder.storeTermVectorPayloads(nodeBooleanValue(propNode));
        iterator.remove();
      } else if (propName.equals(CQL_COLLECTION)) {
        switch (StringUtils.lowerCase(propNode.toString())) {
          case "list":
            builder.cqlCollection(CqlCollection.LIST);
            break;
          case "set":
            builder.cqlCollection(CqlCollection.SET);
            break;
          case "singleton":
            builder.cqlCollection(CqlCollection.SINGLETON);
            break;
        }
        iterator.remove();
      } else if (propName.equals(CQL_STRUCT)) {
        switch (StringUtils.lowerCase(propNode.toString())) {
          case "map":
            builder.cqlStruct(CqlStruct.MAP);
            break;
          case "udt":
            builder.cqlStruct(CqlStruct.UDT);
            break;
          case "tuple":
            builder.cqlStruct(CqlStruct.TUPLE);
            break;
        }
        iterator.remove();
      } else if (propName.equals(CQL_PARTIAL_UPDATE)) {
        builder.cqlPartialUpdate(nodeBooleanValue(propNode));
        iterator.remove();
      } else if (propName.equals("omit_norms")) {
        builder.omitNorms(nodeBooleanValue(propNode));
        iterator.remove();
      } else if (propName.equals("norms")) {
        final Map<String, Object> properties = nodeMapValue(propNode, "norms");
        for (Iterator<Entry<String, Object>> propsIterator = properties.entrySet().iterator();
            propsIterator.hasNext(); ) {
          Entry<String, Object> entry2 = propsIterator.next();
          final String propName2 = Strings.toUnderscoreCase(entry2.getKey());
          final Object propNode2 = entry2.getValue();
          if (propName2.equals("enabled")) {
            builder.omitNorms(!nodeBooleanValue(propNode2));
            propsIterator.remove();
          } else if (propName2.equals(Loading.KEY)) {
            builder.normsLoading(Loading.parse(nodeStringValue(propNode2, null), null));
            propsIterator.remove();
          }
        }
        DocumentMapperParser.checkNoRemainingFields(
            propName, properties, parserContext.indexVersionCreated());
        iterator.remove();
      } else if (propName.equals("omit_term_freq_and_positions")) {
        final IndexOptions op =
            nodeBooleanValue(propNode)
                ? IndexOptions.DOCS
                : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
        if (indexVersionCreated.onOrAfter(Version.V_1_0_0_RC2)) {
          throw new ElasticsearchParseException(
              "'omit_term_freq_and_positions' is not supported anymore - use ['index_options' : 'docs']  instead");
        }
        // deprecated option for BW compat
        builder.indexOptions(op);
        iterator.remove();
      } else if (propName.equals("index_options")) {
        builder.indexOptions(nodeIndexOptionValue(propNode));
        iterator.remove();
      } else if (propName.equals("analyzer")
          || // for backcompat, reading old indexes, remove for v3.0
          propName.equals("index_analyzer") && indexVersionCreated.before(Version.V_2_0_0_beta1)) {

        NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
        if (analyzer == null) {
          throw new MapperParsingException(
              "analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
        }
        indexAnalyzer = analyzer;
        iterator.remove();
      } else if (propName.equals("search_analyzer")) {
        NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
        if (analyzer == null) {
          throw new MapperParsingException(
              "analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
        }
        searchAnalyzer = analyzer;
        iterator.remove();
      } else if (propName.equals("include_in_all")) {
        builder.includeInAll(nodeBooleanValue(propNode));
        iterator.remove();
      } else if (propName.equals("postings_format")
          && indexVersionCreated.before(Version.V_2_0_0_beta1)) {
        // ignore for old indexes
        iterator.remove();
      } else if (propName.equals("doc_values_format")
          && indexVersionCreated.before(Version.V_2_0_0_beta1)) {
        // ignore for old indexes
        iterator.remove();
      } else if (propName.equals("similarity")) {
        builder.similarity(parserContext.similarityLookupService().similarity(propNode.toString()));
        iterator.remove();
      } else if (propName.equals("fielddata")) {
        final Settings settings =
            Settings.builder()
                .put(SettingsLoader.Helper.loadNestedFromMap(nodeMapValue(propNode, "fielddata")))
                .build();
        builder.fieldDataSettings(settings);
        iterator.remove();
      } else if (propName.equals("copy_to")) {
        if (parserContext.isWithinMultiField()) {
          if (indexVersionCreated.after(Version.V_2_1_0)
              || (indexVersionCreated.after(Version.V_2_0_1)
                  && indexVersionCreated.before(Version.V_2_1_0))) {
            throw new MapperParsingException(
                "copy_to in multi fields is not allowed. Found the copy_to in field ["
                    + name
                    + "] which is within a multi field.");
          } else {
            ESLoggerFactory.getLogger("mapping [" + parserContext.type() + "]")
                .warn(
                    "Found a copy_to in field ["
                        + name
                        + "] which is within a multi field. This feature has been removed and the copy_to will be ignored.");
            // we still parse this, otherwise the message will only appear once and the copy_to
            // removed. After that it will appear again. Better to have it always.
          }
        }
        parseCopyFields(propNode, builder);
        iterator.remove();
      }
    }

    if (indexAnalyzer == null) {
      if (searchAnalyzer != null) {
        // If the index was created before 2.0 then we are trying to upgrade the mappings so use the
        // default indexAnalyzer
        // instead of throwing an exception so the user is able to upgrade
        if (parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) {
          indexAnalyzer = parserContext.analysisService().defaultIndexAnalyzer();
        } else {
          throw new MapperParsingException(
              "analyzer on field [" + name + "] must be set when search_analyzer is set");
        }
      }
    } else if (searchAnalyzer == null) {
      searchAnalyzer = indexAnalyzer;
    }
    builder.indexAnalyzer(indexAnalyzer);
    builder.searchAnalyzer(searchAnalyzer);
  }