@Override
public TerminationHandle warmReader(
final IndexShard indexShard, final Engine.Searcher searcher) {
final MapperService mapperService = indexShard.mapperService();
final Map<String, MappedFieldType> warmUpGlobalOrdinals = new HashMap<>();
for (DocumentMapper docMapper : mapperService.docMappers(false)) {
for (FieldMapper fieldMapper : docMapper.mappers()) {
final MappedFieldType fieldType = fieldMapper.fieldType();
final String indexName = fieldType.name();
if (fieldType.eagerGlobalOrdinals() == false) {
continue;
}
warmUpGlobalOrdinals.put(indexName, fieldType);
}
}
final IndexFieldDataService indexFieldDataService = indexShard.indexFieldDataService();
final CountDownLatch latch = new CountDownLatch(warmUpGlobalOrdinals.size());
for (final MappedFieldType fieldType : warmUpGlobalOrdinals.values()) {
executor.execute(
() -> {
try {
final long start = System.nanoTime();
IndexFieldData.Global ifd = indexFieldDataService.getForField(fieldType);
DirectoryReader reader = searcher.getDirectoryReader();
IndexFieldData<?> global = ifd.loadGlobal(reader);
if (reader.leaves().isEmpty() == false) {
global.load(reader.leaves().get(0));
}
if (indexShard.warmerService().logger().isTraceEnabled()) {
indexShard
.warmerService()
.logger()
.trace(
"warmed global ordinals for [{}], took [{}]",
fieldType.name(),
TimeValue.timeValueNanos(System.nanoTime() - start));
}
} catch (Exception e) {
indexShard
.warmerService()
.logger()
.warn("failed to warm-up global ordinals for [{}]", e, fieldType.name());
} finally {
latch.countDown();
}
});
}
return () -> latch.await();
}
/**
* Check if the allocation of the replica is to be delayed. Compute the delay and if it is
* delayed, add it to the ignore unassigned list Note: we only care about replica in delayed
* allocation, since if we have an unassigned primary it will anyhow wait to find an existing copy
* of the shard to be allocated Note: the other side of the equation is scheduling a reroute in a
* timely manner, which happens in the RoutingService
*
* <p>PUBLIC FOR TESTS!
*
* @param unassignedIterator iterator over unassigned shards
* @param shard the shard which might be delayed
* @return true iff allocation is delayed for this shard
*/
public boolean ignoreUnassignedIfDelayed(
RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator, ShardRouting shard) {
// calculate delay and store it in UnassignedInfo to be used by RoutingService
long delay = shard.unassignedInfo().getLastComputedLeftDelayNanos();
if (delay > 0) {
logger.debug(
"[{}][{}]: delaying allocation of [{}] for [{}]",
shard.index(),
shard.id(),
shard,
TimeValue.timeValueNanos(delay));
/**
* mark it as changed, since we want to kick a publishing to schedule future allocation, see
* {@link
* org.elasticsearch.cluster.routing.RoutingService#clusterChanged(ClusterChangedEvent)}).
*/
unassignedIterator.removeAndIgnore();
return true;
}
return false;
}
private String innerDetect() throws Exception {
StringBuilder sb = new StringBuilder();
sb.append("Hot threads at ");
sb.append(DATE_TIME_FORMATTER.printer().print(System.currentTimeMillis()));
sb.append(", interval=");
sb.append(interval);
sb.append(", busiestThreads=");
sb.append(busiestThreads);
sb.append(", ignoreIdleThreads=");
sb.append(ignoreIdleThreads);
sb.append(":\n");
ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
boolean enabledCpu = false;
try {
if (threadBean.isThreadCpuTimeSupported()) {
if (!threadBean.isThreadCpuTimeEnabled()) {
enabledCpu = true;
threadBean.setThreadCpuTimeEnabled(true);
}
} else {
throw new IllegalStateException("MBean doesn't support thread CPU Time");
}
Map<Long, MyThreadInfo> threadInfos = new HashMap<>();
for (long threadId : threadBean.getAllThreadIds()) {
// ignore our own thread...
if (Thread.currentThread().getId() == threadId) {
continue;
}
long cpu = threadBean.getThreadCpuTime(threadId);
if (cpu == -1) {
continue;
}
ThreadInfo info = threadBean.getThreadInfo(threadId, 0);
if (info == null) {
continue;
}
threadInfos.put(threadId, new MyThreadInfo(cpu, info));
}
Thread.sleep(interval.millis());
for (long threadId : threadBean.getAllThreadIds()) {
// ignore our own thread...
if (Thread.currentThread().getId() == threadId) {
continue;
}
long cpu = threadBean.getThreadCpuTime(threadId);
if (cpu == -1) {
threadInfos.remove(threadId);
continue;
}
ThreadInfo info = threadBean.getThreadInfo(threadId, 0);
if (info == null) {
threadInfos.remove(threadId);
continue;
}
MyThreadInfo data = threadInfos.get(threadId);
if (data != null) {
data.setDelta(cpu, info);
} else {
threadInfos.remove(threadId);
}
}
// sort by delta CPU time on thread.
List<MyThreadInfo> hotties = new ArrayList<>(threadInfos.values());
final int busiestThreads = Math.min(this.busiestThreads, hotties.size());
// skip that for now
CollectionUtil.introSort(
hotties,
new Comparator<MyThreadInfo>() {
@Override
public int compare(MyThreadInfo o1, MyThreadInfo o2) {
if ("cpu".equals(type)) {
return (int) (o2.cpuTime - o1.cpuTime);
} else if ("wait".equals(type)) {
return (int) (o2.waitedTime - o1.waitedTime);
} else if ("block".equals(type)) {
return (int) (o2.blockedTime - o1.blockedTime);
}
throw new IllegalArgumentException();
}
});
// analyse N stack traces for M busiest threads
long[] ids = new long[busiestThreads];
for (int i = 0; i < busiestThreads; i++) {
MyThreadInfo info = hotties.get(i);
ids[i] = info.info.getThreadId();
}
ThreadInfo[][] allInfos = new ThreadInfo[threadElementsSnapshotCount][];
for (int j = 0; j < threadElementsSnapshotCount; j++) {
// NOTE, javadoc of getThreadInfo says: If a thread of the given ID is not alive or does not
// exist,
// null will be set in the corresponding element in the returned array. A thread is alive if
// it has
// been started and has not yet died.
allInfos[j] = threadBean.getThreadInfo(ids, Integer.MAX_VALUE);
Thread.sleep(threadElementsSnapshotDelay.millis());
}
for (int t = 0; t < busiestThreads; t++) {
long time = 0;
if ("cpu".equals(type)) {
time = hotties.get(t).cpuTime;
} else if ("wait".equals(type)) {
time = hotties.get(t).waitedTime;
} else if ("block".equals(type)) {
time = hotties.get(t).blockedTime;
}
String threadName = null;
for (ThreadInfo[] info : allInfos) {
if (info != null && info[t] != null) {
if (ignoreIdleThreads && isIdleThread(info[t])) {
info[t] = null;
continue;
}
threadName = info[t].getThreadName();
break;
}
}
if (threadName == null) {
continue; // thread is not alive yet or died before the first snapshot - ignore it!
}
double percent = (((double) time) / interval.nanos()) * 100;
sb.append(
String.format(
Locale.ROOT,
"%n%4.1f%% (%s out of %s) %s usage by thread '%s'%n",
percent,
TimeValue.timeValueNanos(time),
interval,
type,
threadName));
// for each snapshot (2nd array index) find later snapshot for same thread with max number
// of
// identical StackTraceElements (starting from end of each)
boolean[] done = new boolean[threadElementsSnapshotCount];
for (int i = 0; i < threadElementsSnapshotCount; i++) {
if (done[i]) continue;
int maxSim = 1;
boolean[] similars = new boolean[threadElementsSnapshotCount];
for (int j = i + 1; j < threadElementsSnapshotCount; j++) {
if (done[j]) continue;
int similarity = similarity(allInfos[i][t], allInfos[j][t]);
if (similarity > maxSim) {
maxSim = similarity;
similars = new boolean[threadElementsSnapshotCount];
}
if (similarity == maxSim) similars[j] = true;
}
// print out trace maxSim levels of i, and mark similar ones as done
int count = 1;
for (int j = i + 1; j < threadElementsSnapshotCount; j++) {
if (similars[j]) {
done[j] = true;
count++;
}
}
if (allInfos[i][t] != null) {
final StackTraceElement[] show = allInfos[i][t].getStackTrace();
if (count == 1) {
sb.append(String.format(Locale.ROOT, " unique snapshot%n"));
for (int l = 0; l < show.length; l++) {
sb.append(String.format(Locale.ROOT, " %s%n", show[l]));
}
} else {
sb.append(
String.format(
Locale.ROOT,
" %d/%d snapshots sharing following %d elements%n",
count,
threadElementsSnapshotCount,
maxSim));
for (int l = show.length - maxSim; l < show.length; l++) {
sb.append(String.format(Locale.ROOT, " %s%n", show[l]));
}
}
}
}
}
return sb.toString();
} finally {
if (enabledCpu) {
threadBean.setThreadCpuTimeEnabled(false);
}
}
}