@Override public TerminationHandle warmReader( final IndexShard indexShard, final Engine.Searcher searcher) { final MapperService mapperService = indexShard.mapperService(); final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>(); for (DocumentMapper docMapper : mapperService.docMappers(false)) { for (FieldMapper fieldMapper : docMapper.mappers()) { final MappedFieldType fieldType = fieldMapper.fieldType(); final String indexName = fieldType.name(); if (fieldType.eagerGlobalOrdinals() == false) { continue; } warmUpGlobalOrdinals.put(indexName, fieldType); } } final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService(); final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size()); for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) { executor.execute( () -> { try { final long start = System.nanoTime(); IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType); DirectoryReader reader = searcher.getDirectoryReader(); IndexFieldData<?> global = ifd.loadGlobal(reader); if (reader.leaves().isEmpty() == false) { global.load(reader.leaves().get(0)); } if (indexShard.warmerService().logger().isTraceEnabled()) { indexShard .warmerService() .logger() .trace( "warmed global ordinals for [{}], took [{}]", fieldType.name(), TimeValue.timeValueNanos(System.nanoTime() - start)); } } catch (Exception e) { indexShard .warmerService() .logger() .warn("failed to warm-up global ordinals for [{}]", e, fieldType.name()); } finally { latch.countDown(); } }); } return () -> latch.await(); }
/** * Check if the allocation of the replica is to be delayed. Compute the delay and if it is * delayed, add it to the ignore unassigned list Note: we only care about replica in delayed * allocation, since if we have an unassigned primary it will anyhow wait to find an existing copy * of the shard to be allocated Note: the other side of the equation is scheduling a reroute in a * timely manner, which happens in the RoutingService * * <p>PUBLIC FOR TESTS! * * @param unassignedIterator iterator over unassigned shards * @param shard the shard which might be delayed * @return true iff allocation is delayed for this shard */ public boolean ignoreUnassignedIfDelayed( RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator, ShardRouting shard) { // calculate delay and store it in UnassignedInfo to be used by RoutingService long delay = shard.unassignedInfo().getLastComputedLeftDelayNanos(); if (delay > 0) { logger.debug( "[{}][{}]: delaying allocation of [{}] for [{}]", shard.index(), shard.id(), shard, TimeValue.timeValueNanos(delay)); /** * mark it as changed, since we want to kick a publishing to schedule future allocation, see * {@link * org.elasticsearch.cluster.routing.RoutingService#clusterChanged(ClusterChangedEvent)}). */ unassignedIterator.removeAndIgnore(); return true; } return false; }
private String innerDetect() throws Exception { StringBuilder sb = new StringBuilder(); sb.append("Hot threads at "); sb.append(DATE_TIME_FORMATTER.printer().print(System.currentTimeMillis())); sb.append(", interval="); sb.append(interval); sb.append(", busiestThreads="); sb.append(busiestThreads); sb.append(", ignoreIdleThreads="); sb.append(ignoreIdleThreads); sb.append(":\n"); ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); boolean enabledCpu = false; try { if (threadBean.isThreadCpuTimeSupported()) { if (!threadBean.isThreadCpuTimeEnabled()) { enabledCpu = true; threadBean.setThreadCpuTimeEnabled(true); } } else { throw new IllegalStateException("MBean doesn't support thread CPU Time"); } Map<Long, MyThreadInfo> threadInfos = new HashMap<>(); for (long threadId : threadBean.getAllThreadIds()) { // ignore our own thread... if (Thread.currentThread().getId() == threadId) { continue; } long cpu = threadBean.getThreadCpuTime(threadId); if (cpu == -1) { continue; } ThreadInfo info = threadBean.getThreadInfo(threadId, 0); if (info == null) { continue; } threadInfos.put(threadId, new MyThreadInfo(cpu, info)); } Thread.sleep(interval.millis()); for (long threadId : threadBean.getAllThreadIds()) { // ignore our own thread... if (Thread.currentThread().getId() == threadId) { continue; } long cpu = threadBean.getThreadCpuTime(threadId); if (cpu == -1) { threadInfos.remove(threadId); continue; } ThreadInfo info = threadBean.getThreadInfo(threadId, 0); if (info == null) { threadInfos.remove(threadId); continue; } MyThreadInfo data = threadInfos.get(threadId); if (data != null) { data.setDelta(cpu, info); } else { threadInfos.remove(threadId); } } // sort by delta CPU time on thread. List<MyThreadInfo> hotties = new ArrayList<>(threadInfos.values()); final int busiestThreads = Math.min(this.busiestThreads, hotties.size()); // skip that for now CollectionUtil.introSort( hotties, new Comparator<MyThreadInfo>() { @Override public int compare(MyThreadInfo o1, MyThreadInfo o2) { if ("cpu".equals(type)) { return (int) (o2.cpuTime - o1.cpuTime); } else if ("wait".equals(type)) { return (int) (o2.waitedTime - o1.waitedTime); } else if ("block".equals(type)) { return (int) (o2.blockedTime - o1.blockedTime); } throw new IllegalArgumentException(); } }); // analyse N stack traces for M busiest threads long[] ids = new long[busiestThreads]; for (int i = 0; i < busiestThreads; i++) { MyThreadInfo info = hotties.get(i); ids[i] = info.info.getThreadId(); } ThreadInfo[][] allInfos = new ThreadInfo[threadElementsSnapshotCount][]; for (int j = 0; j < threadElementsSnapshotCount; j++) { // NOTE, javadoc of getThreadInfo says: If a thread of the given ID is not alive or does not // exist, // null will be set in the corresponding element in the returned array. A thread is alive if // it has // been started and has not yet died. allInfos[j] = threadBean.getThreadInfo(ids, Integer.MAX_VALUE); Thread.sleep(threadElementsSnapshotDelay.millis()); } for (int t = 0; t < busiestThreads; t++) { long time = 0; if ("cpu".equals(type)) { time = hotties.get(t).cpuTime; } else if ("wait".equals(type)) { time = hotties.get(t).waitedTime; } else if ("block".equals(type)) { time = hotties.get(t).blockedTime; } String threadName = null; for (ThreadInfo[] info : allInfos) { if (info != null && info[t] != null) { if (ignoreIdleThreads && isIdleThread(info[t])) { info[t] = null; continue; } threadName = info[t].getThreadName(); break; } } if (threadName == null) { continue; // thread is not alive yet or died before the first snapshot - ignore it! } double percent = (((double) time) / interval.nanos()) * 100; sb.append( String.format( Locale.ROOT, "%n%4.1f%% (%s out of %s) %s usage by thread '%s'%n", percent, TimeValue.timeValueNanos(time), interval, type, threadName)); // for each snapshot (2nd array index) find later snapshot for same thread with max number // of // identical StackTraceElements (starting from end of each) boolean[] done = new boolean[threadElementsSnapshotCount]; for (int i = 0; i < threadElementsSnapshotCount; i++) { if (done[i]) continue; int maxSim = 1; boolean[] similars = new boolean[threadElementsSnapshotCount]; for (int j = i + 1; j < threadElementsSnapshotCount; j++) { if (done[j]) continue; int similarity = similarity(allInfos[i][t], allInfos[j][t]); if (similarity > maxSim) { maxSim = similarity; similars = new boolean[threadElementsSnapshotCount]; } if (similarity == maxSim) similars[j] = true; } // print out trace maxSim levels of i, and mark similar ones as done int count = 1; for (int j = i + 1; j < threadElementsSnapshotCount; j++) { if (similars[j]) { done[j] = true; count++; } } if (allInfos[i][t] != null) { final StackTraceElement[] show = allInfos[i][t].getStackTrace(); if (count == 1) { sb.append(String.format(Locale.ROOT, " unique snapshot%n")); for (int l = 0; l < show.length; l++) { sb.append(String.format(Locale.ROOT, " %s%n", show[l])); } } else { sb.append( String.format( Locale.ROOT, " %d/%d snapshots sharing following %d elements%n", count, threadElementsSnapshotCount, maxSim)); for (int l = show.length - maxSim; l < show.length; l++) { sb.append(String.format(Locale.ROOT, " %s%n", show[l])); } } } } } return sb.toString(); } finally { if (enabledCpu) { threadBean.setThreadCpuTimeEnabled(false); } } }