@Override
    public TerminationHandle warmReader(
        final IndexShard indexShard, final Engine.Searcher searcher) {
      final MapperService mapperService = indexShard.mapperService();
      final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
      for (DocumentMapper docMapper : mapperService.docMappers(false)) {
        for (FieldMapper fieldMapper : docMapper.mappers()) {
          final MappedFieldType fieldType = fieldMapper.fieldType();
          final String indexName = fieldType.name();
          if (fieldType.eagerGlobalOrdinals() == false) {
            continue;
          }
          warmUpGlobalOrdinals.put(indexName, fieldType);
        }
      }
      final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
      final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size());
      for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) {
        executor.execute(
            () -> {
              try {
                final long start = System.nanoTime();
                IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType);
                DirectoryReader reader = searcher.getDirectoryReader();
                IndexFieldData<?> global = ifd.loadGlobal(reader);
                if (reader.leaves().isEmpty() == false) {
                  global.load(reader.leaves().get(0));
                }

                if (indexShard.warmerService().logger().isTraceEnabled()) {
                  indexShard
                      .warmerService()
                      .logger()
                      .trace(
                          "warmed global ordinals for [{}], took [{}]",
                          fieldType.name(),
                          TimeValue.timeValueNanos(System.nanoTime() - start));
                }
              } catch (Exception e) {
                indexShard
                    .warmerService()
                    .logger()
                    .warn("failed to warm-up global ordinals for [{}]", e, fieldType.name());
              } finally {
                latch.countDown();
              }
            });
      }
      return () -> latch.await();
    }
 /**
  * Check if the allocation of the replica is to be delayed. Compute the delay and if it is
  * delayed, add it to the ignore unassigned list Note: we only care about replica in delayed
  * allocation, since if we have an unassigned primary it will anyhow wait to find an existing copy
  * of the shard to be allocated Note: the other side of the equation is scheduling a reroute in a
  * timely manner, which happens in the RoutingService
  *
  * <p>PUBLIC FOR TESTS!
  *
  * @param unassignedIterator iterator over unassigned shards
  * @param shard the shard which might be delayed
  * @return true iff allocation is delayed for this shard
  */
 public boolean ignoreUnassignedIfDelayed(
     RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator, ShardRouting shard) {
   // calculate delay and store it in UnassignedInfo to be used by RoutingService
   long delay = shard.unassignedInfo().getLastComputedLeftDelayNanos();
   if (delay > 0) {
     logger.debug(
         "[{}][{}]: delaying allocation of [{}] for [{}]",
         shard.index(),
         shard.id(),
         shard,
         TimeValue.timeValueNanos(delay));
     /**
      * mark it as changed, since we want to kick a publishing to schedule future allocation, see
      * {@link
      * org.elasticsearch.cluster.routing.RoutingService#clusterChanged(ClusterChangedEvent)}).
      */
     unassignedIterator.removeAndIgnore();
     return true;
   }
   return false;
 }
Beispiel #3
0
  private String innerDetect() throws Exception {
    StringBuilder sb = new StringBuilder();

    sb.append("Hot threads at ");
    sb.append(DATE_TIME_FORMATTER.printer().print(System.currentTimeMillis()));
    sb.append(", interval=");
    sb.append(interval);
    sb.append(", busiestThreads=");
    sb.append(busiestThreads);
    sb.append(", ignoreIdleThreads=");
    sb.append(ignoreIdleThreads);
    sb.append(":\n");

    ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
    boolean enabledCpu = false;
    try {
      if (threadBean.isThreadCpuTimeSupported()) {
        if (!threadBean.isThreadCpuTimeEnabled()) {
          enabledCpu = true;
          threadBean.setThreadCpuTimeEnabled(true);
        }
      } else {
        throw new IllegalStateException("MBean doesn't support thread CPU Time");
      }
      Map<Long, MyThreadInfo> threadInfos = new HashMap<>();
      for (long threadId : threadBean.getAllThreadIds()) {
        // ignore our own thread...
        if (Thread.currentThread().getId() == threadId) {
          continue;
        }
        long cpu = threadBean.getThreadCpuTime(threadId);
        if (cpu == -1) {
          continue;
        }
        ThreadInfo info = threadBean.getThreadInfo(threadId, 0);
        if (info == null) {
          continue;
        }
        threadInfos.put(threadId, new MyThreadInfo(cpu, info));
      }
      Thread.sleep(interval.millis());
      for (long threadId : threadBean.getAllThreadIds()) {
        // ignore our own thread...
        if (Thread.currentThread().getId() == threadId) {
          continue;
        }
        long cpu = threadBean.getThreadCpuTime(threadId);
        if (cpu == -1) {
          threadInfos.remove(threadId);
          continue;
        }
        ThreadInfo info = threadBean.getThreadInfo(threadId, 0);
        if (info == null) {
          threadInfos.remove(threadId);
          continue;
        }
        MyThreadInfo data = threadInfos.get(threadId);
        if (data != null) {
          data.setDelta(cpu, info);
        } else {
          threadInfos.remove(threadId);
        }
      }
      // sort by delta CPU time on thread.
      List<MyThreadInfo> hotties = new ArrayList<>(threadInfos.values());
      final int busiestThreads = Math.min(this.busiestThreads, hotties.size());
      // skip that for now
      CollectionUtil.introSort(
          hotties,
          new Comparator<MyThreadInfo>() {
            @Override
            public int compare(MyThreadInfo o1, MyThreadInfo o2) {
              if ("cpu".equals(type)) {
                return (int) (o2.cpuTime - o1.cpuTime);
              } else if ("wait".equals(type)) {
                return (int) (o2.waitedTime - o1.waitedTime);
              } else if ("block".equals(type)) {
                return (int) (o2.blockedTime - o1.blockedTime);
              }
              throw new IllegalArgumentException();
            }
          });
      // analyse N stack traces for M busiest threads
      long[] ids = new long[busiestThreads];
      for (int i = 0; i < busiestThreads; i++) {
        MyThreadInfo info = hotties.get(i);
        ids[i] = info.info.getThreadId();
      }
      ThreadInfo[][] allInfos = new ThreadInfo[threadElementsSnapshotCount][];
      for (int j = 0; j < threadElementsSnapshotCount; j++) {
        // NOTE, javadoc of getThreadInfo says: If a thread of the given ID is not alive or does not
        // exist,
        // null will be set in the corresponding element in the returned array. A thread is alive if
        // it has
        // been started and has not yet died.
        allInfos[j] = threadBean.getThreadInfo(ids, Integer.MAX_VALUE);
        Thread.sleep(threadElementsSnapshotDelay.millis());
      }
      for (int t = 0; t < busiestThreads; t++) {
        long time = 0;
        if ("cpu".equals(type)) {
          time = hotties.get(t).cpuTime;
        } else if ("wait".equals(type)) {
          time = hotties.get(t).waitedTime;
        } else if ("block".equals(type)) {
          time = hotties.get(t).blockedTime;
        }
        String threadName = null;
        for (ThreadInfo[] info : allInfos) {
          if (info != null && info[t] != null) {
            if (ignoreIdleThreads && isIdleThread(info[t])) {
              info[t] = null;
              continue;
            }
            threadName = info[t].getThreadName();
            break;
          }
        }
        if (threadName == null) {
          continue; // thread is not alive yet or died before the first snapshot - ignore it!
        }
        double percent = (((double) time) / interval.nanos()) * 100;
        sb.append(
            String.format(
                Locale.ROOT,
                "%n%4.1f%% (%s out of %s) %s usage by thread '%s'%n",
                percent,
                TimeValue.timeValueNanos(time),
                interval,
                type,
                threadName));
        // for each snapshot (2nd array index) find later snapshot for same thread with max number
        // of
        // identical StackTraceElements (starting from end of each)
        boolean[] done = new boolean[threadElementsSnapshotCount];
        for (int i = 0; i < threadElementsSnapshotCount; i++) {
          if (done[i]) continue;
          int maxSim = 1;
          boolean[] similars = new boolean[threadElementsSnapshotCount];
          for (int j = i + 1; j < threadElementsSnapshotCount; j++) {
            if (done[j]) continue;
            int similarity = similarity(allInfos[i][t], allInfos[j][t]);
            if (similarity > maxSim) {
              maxSim = similarity;
              similars = new boolean[threadElementsSnapshotCount];
            }
            if (similarity == maxSim) similars[j] = true;
          }
          // print out trace maxSim levels of i, and mark similar ones as done
          int count = 1;
          for (int j = i + 1; j < threadElementsSnapshotCount; j++) {
            if (similars[j]) {
              done[j] = true;
              count++;
            }
          }
          if (allInfos[i][t] != null) {
            final StackTraceElement[] show = allInfos[i][t].getStackTrace();
            if (count == 1) {
              sb.append(String.format(Locale.ROOT, "  unique snapshot%n"));
              for (int l = 0; l < show.length; l++) {
                sb.append(String.format(Locale.ROOT, "    %s%n", show[l]));
              }
            } else {
              sb.append(
                  String.format(
                      Locale.ROOT,
                      "  %d/%d snapshots sharing following %d elements%n",
                      count,
                      threadElementsSnapshotCount,
                      maxSim));
              for (int l = show.length - maxSim; l < show.length; l++) {
                sb.append(String.format(Locale.ROOT, "    %s%n", show[l]));
              }
            }
          }
        }
      }
      return sb.toString();
    } finally {
      if (enabledCpu) {
        threadBean.setThreadCpuTimeEnabled(false);
      }
    }
  }