protected AbstractIndexOrdinalsFieldData(
     Index index,
     Settings indexSettings,
     Names fieldNames,
     FieldDataType fieldDataType,
     IndexFieldDataCache cache,
     CircuitBreakerService breakerService) {
   super(index, indexSettings, fieldNames, fieldDataType, cache);
   final Map<String, Settings> groups = fieldDataType.getSettings().getGroups("filter");
   frequency = groups.get("frequency");
   regex = groups.get("regex");
   this.breakerService = breakerService;
 }
Esempio n. 2
0
 @Override
 public TerminationHandle warmTopReader(
     final IndexShard indexShard,
     IndexMetaData indexMetaData,
     final WarmerContext context,
     ThreadPool threadPool) {
   final MapperService mapperService = indexShard.mapperService();
   final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
   for (DocumentMapper docMapper : mapperService.docMappers(false)) {
     for (FieldMapper fieldMapper : docMapper.mappers()) {
       final FieldDataType fieldDataType = fieldMapper.fieldType().fieldDataType();
       if (fieldDataType == null) {
         continue;
       }
       if (fieldDataType.getLoading() != Loading.EAGER_GLOBAL_ORDINALS) {
         continue;
       }
       final String indexName = fieldMapper.fieldType().names().indexName();
       if (warmUpGlobalOrdinals.containsKey(indexName)) {
         continue;
       }
       warmUpGlobalOrdinals.put(indexName, fieldMapper.fieldType());
     }
   }
   final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
   final Executor executor = threadPool.executor(executor());
   final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size());
   for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) {
     executor.execute(
         new Runnable() {
           @Override
           public void run() {
             try {
               final long start = System.nanoTime();
               IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType);
               ifd.loadGlobal(context.reader());
               if (indexShard.warmerService().logger().isTraceEnabled()) {
                 indexShard
                     .warmerService()
                     .logger()
                     .trace(
                         "warmed global ordinals for [{}], took [{}]",
                         fieldType.names().fullName(),
                         TimeValue.timeValueNanos(System.nanoTime() - start));
               }
             } catch (Throwable t) {
               indexShard
                   .warmerService()
                   .logger()
                   .warn(
                       "failed to warm-up global ordinals for [{}]",
                       t,
                       fieldType.names().fullName());
             } finally {
               latch.countDown();
             }
           }
         });
   }
   return new TerminationHandle() {
     @Override
     public void awaitTermination() throws InterruptedException {
       latch.await();
     }
   };
 }
  @Override
  public IndexFieldData.WithOrdinals build(
      final IndexReader indexReader,
      IndexFieldData.WithOrdinals indexFieldData,
      Settings settings,
      CircuitBreakerService breakerService)
      throws IOException {
    assert indexReader.leaves().size() > 1;
    long startTime = System.currentTimeMillis();

    // It makes sense to make the overhead ratio configurable for the mapping from segment ords to
    // global ords
    // However, other mappings are never the bottleneck and only used to get the original value from
    // an ord, so
    // it makes sense to force COMPACT for them
    final float acceptableOverheadRatio =
        settings.getAsFloat("acceptable_overhead_ratio", PackedInts.FAST);
    final AppendingPackedLongBuffer globalOrdToFirstSegment =
        new AppendingPackedLongBuffer(PackedInts.COMPACT);
    final MonotonicAppendingLongBuffer globalOrdToFirstSegmentDelta =
        new MonotonicAppendingLongBuffer(PackedInts.COMPACT);

    FieldDataType fieldDataType = indexFieldData.getFieldDataType();
    int defaultThreshold =
        settings.getAsInt(
            ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, ORDINAL_MAPPING_THRESHOLD_DEFAULT);
    int threshold =
        fieldDataType.getSettings().getAsInt(ORDINAL_MAPPING_THRESHOLD_KEY, defaultThreshold);
    OrdinalMappingSourceBuilder ordinalMappingBuilder =
        new OrdinalMappingSourceBuilder(
            indexReader.leaves().size(), acceptableOverheadRatio, threshold);

    long currentGlobalOrdinal = 0;
    final AtomicFieldData.WithOrdinals[] withOrdinals =
        new AtomicFieldData.WithOrdinals[indexReader.leaves().size()];
    TermIterator termIterator =
        new TermIterator(indexFieldData, indexReader.leaves(), withOrdinals);
    for (BytesRef term = termIterator.next(); term != null; term = termIterator.next()) {
      globalOrdToFirstSegment.add(termIterator.firstReaderIndex());
      long globalOrdinalDelta = currentGlobalOrdinal - termIterator.firstLocalOrdinal();
      globalOrdToFirstSegmentDelta.add(globalOrdinalDelta);
      for (TermIterator.LeafSource leafSource : termIterator.competitiveLeafs()) {
        ordinalMappingBuilder.onOrdinal(
            leafSource.context.ord, leafSource.tenum.ord(), currentGlobalOrdinal);
      }
      currentGlobalOrdinal++;
    }

    // ram used for the globalOrd to segmentOrd and segmentOrd to firstReaderIndex lookups
    long memorySizeInBytesCounter = 0;
    globalOrdToFirstSegment.freeze();
    memorySizeInBytesCounter += globalOrdToFirstSegment.ramBytesUsed();
    globalOrdToFirstSegmentDelta.freeze();
    memorySizeInBytesCounter += globalOrdToFirstSegmentDelta.ramBytesUsed();

    final long maxOrd = currentGlobalOrdinal;
    OrdinalMappingSource[] segmentOrdToGlobalOrdLookups = ordinalMappingBuilder.build(maxOrd);
    // add ram used for the main segmentOrd to globalOrd lookups
    memorySizeInBytesCounter += ordinalMappingBuilder.getMemorySizeInBytes();

    final long memorySizeInBytes = memorySizeInBytesCounter;
    breakerService.getBreaker().addWithoutBreaking(memorySizeInBytes);

    if (logger.isDebugEnabled()) {
      // this does include the [] from the array in the impl name
      String implName = segmentOrdToGlobalOrdLookups.getClass().getSimpleName();
      logger.debug(
          "Global-ordinals[{}][{}][{}] took {} ms",
          implName,
          indexFieldData.getFieldNames().fullName(),
          maxOrd,
          (System.currentTimeMillis() - startTime));
    }
    return new InternalGlobalOrdinalsIndexFieldData(
        indexFieldData.index(),
        settings,
        indexFieldData.getFieldNames(),
        fieldDataType,
        withOrdinals,
        globalOrdToFirstSegment,
        globalOrdToFirstSegmentDelta,
        segmentOrdToGlobalOrdLookups,
        memorySizeInBytes);
  }