@Override
protected void doStartNextScroll(
String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse) {
execute(
"POST",
scrollPath(),
scrollParams(
timeValueNanos(searchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos())),
scrollEntity(scrollId),
RESPONSE_PARSER,
onResponse);
}
@Override
protected void doStartNextScroll(
String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse) {
SearchScrollRequest request = new SearchScrollRequest();
// Add the wait time into the scroll timeout so it won't timeout while we wait for throttling
request
.scrollId(scrollId)
.scroll(
timeValueNanos(
firstSearchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos()));
searchWithRetry(
listener -> client.searchScroll(request, listener), r -> consume(r, onResponse));
}
private static String renderValue(RestRequest request, Object value) {
if (value == null) {
return null;
}
if (value instanceof ByteSizeValue) {
ByteSizeValue v = (ByteSizeValue) value;
String resolution = request.param("bytes");
if ("b".equals(resolution)) {
return Long.toString(v.getBytes());
} else if ("k".equals(resolution) || "kb".equals(resolution)) {
return Long.toString(v.getKb());
} else if ("m".equals(resolution) || "mb".equals(resolution)) {
return Long.toString(v.getMb());
} else if ("g".equals(resolution) || "gb".equals(resolution)) {
return Long.toString(v.getGb());
} else if ("t".equals(resolution) || "tb".equals(resolution)) {
return Long.toString(v.getTb());
} else if ("p".equals(resolution) || "pb".equals(resolution)) {
return Long.toString(v.getPb());
} else {
return v.toString();
}
}
if (value instanceof SizeValue) {
SizeValue v = (SizeValue) value;
String resolution = request.param("size");
if ("".equals(resolution)) {
return Long.toString(v.singles());
} else if ("k".equals(resolution)) {
return Long.toString(v.kilo());
} else if ("m".equals(resolution)) {
return Long.toString(v.mega());
} else if ("g".equals(resolution)) {
return Long.toString(v.giga());
} else if ("t".equals(resolution)) {
return Long.toString(v.tera());
} else if ("p".equals(resolution)) {
return Long.toString(v.peta());
} else {
return v.toString();
}
}
if (value instanceof TimeValue) {
TimeValue v = (TimeValue) value;
String resolution = request.param("time");
if ("nanos".equals(resolution)) {
return Long.toString(v.nanos());
} else if ("micros".equals(resolution)) {
return Long.toString(v.micros());
} else if ("ms".equals(resolution)) {
return Long.toString(v.millis());
} else if ("s".equals(resolution)) {
return Long.toString(v.seconds());
} else if ("m".equals(resolution)) {
return Long.toString(v.minutes());
} else if ("h".equals(resolution)) {
return Long.toString(v.hours());
} else if ("d".equals(resolution)) {
return Long.toString(v.days());
} else {
return v.toString();
}
}
// Add additional built in data points we can render based on request parameters?
return value.toString();
}
public void testMergeStatuses() {
BulkByScrollTask.StatusOrException[] statuses =
new BulkByScrollTask.StatusOrException[between(2, 100)];
boolean containsNullStatuses = randomBoolean();
int mergedTotal = 0;
int mergedUpdated = 0;
int mergedCreated = 0;
int mergedDeleted = 0;
int mergedBatches = 0;
int mergedVersionConflicts = 0;
int mergedNoops = 0;
int mergedBulkRetries = 0;
int mergedSearchRetries = 0;
TimeValue mergedThrottled = timeValueNanos(0);
float mergedRequestsPerSecond = 0;
TimeValue mergedThrottledUntil = timeValueNanos(Integer.MAX_VALUE);
for (int i = 0; i < statuses.length; i++) {
if (containsNullStatuses && rarely()) {
continue;
}
int total = between(0, 10000);
int updated = between(0, total);
int created = between(0, total - updated);
int deleted = between(0, total - updated - created);
int batches = between(0, 10);
int versionConflicts = between(0, 100);
int noops = total - updated - created - deleted;
int bulkRetries = between(0, 100);
int searchRetries = between(0, 100);
TimeValue throttled = timeValueNanos(between(0, 10000));
float requestsPerSecond = randomValueOtherThanMany(r -> r <= 0, () -> randomFloat());
String reasonCancelled = randomBoolean() ? null : "test";
TimeValue throttledUntil = timeValueNanos(between(0, 1000));
statuses[i] =
new BulkByScrollTask.StatusOrException(
new BulkByScrollTask.Status(
i,
total,
updated,
created,
deleted,
batches,
versionConflicts,
noops,
bulkRetries,
searchRetries,
throttled,
requestsPerSecond,
reasonCancelled,
throttledUntil));
mergedTotal += total;
mergedUpdated += updated;
mergedCreated += created;
mergedDeleted += deleted;
mergedBatches += batches;
mergedVersionConflicts += versionConflicts;
mergedNoops += noops;
mergedBulkRetries += bulkRetries;
mergedSearchRetries += searchRetries;
mergedThrottled = timeValueNanos(mergedThrottled.nanos() + throttled.nanos());
mergedRequestsPerSecond += requestsPerSecond;
mergedThrottledUntil =
timeValueNanos(min(mergedThrottledUntil.nanos(), throttledUntil.nanos()));
}
String reasonCancelled = randomBoolean() ? randomAsciiOfLength(10) : null;
BulkByScrollTask.Status merged =
new BulkByScrollTask.Status(Arrays.asList(statuses), reasonCancelled);
assertEquals(mergedTotal, merged.getTotal());
assertEquals(mergedUpdated, merged.getUpdated());
assertEquals(mergedCreated, merged.getCreated());
assertEquals(mergedDeleted, merged.getDeleted());
assertEquals(mergedBatches, merged.getBatches());
assertEquals(mergedVersionConflicts, merged.getVersionConflicts());
assertEquals(mergedNoops, merged.getNoops());
assertEquals(mergedBulkRetries, merged.getBulkRetries());
assertEquals(mergedSearchRetries, merged.getSearchRetries());
assertEquals(mergedThrottled, merged.getThrottled());
assertEquals(mergedRequestsPerSecond, merged.getRequestsPerSecond(), 0.0001f);
assertEquals(mergedThrottledUntil, merged.getThrottledUntil());
assertEquals(reasonCancelled, merged.getReasonCancelled());
}
private String innerDetect() throws Exception {
StringBuilder sb = new StringBuilder();
sb.append("Hot threads at ");
sb.append(DATE_TIME_FORMATTER.printer().print(System.currentTimeMillis()));
sb.append(", interval=");
sb.append(interval);
sb.append(", busiestThreads=");
sb.append(busiestThreads);
sb.append(", ignoreIdleThreads=");
sb.append(ignoreIdleThreads);
sb.append(":\n");
ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
boolean enabledCpu = false;
try {
if (threadBean.isThreadCpuTimeSupported()) {
if (!threadBean.isThreadCpuTimeEnabled()) {
enabledCpu = true;
threadBean.setThreadCpuTimeEnabled(true);
}
} else {
throw new IllegalStateException("MBean doesn't support thread CPU Time");
}
Map<Long, MyThreadInfo> threadInfos = new HashMap<>();
for (long threadId : threadBean.getAllThreadIds()) {
// ignore our own thread...
if (Thread.currentThread().getId() == threadId) {
continue;
}
long cpu = threadBean.getThreadCpuTime(threadId);
if (cpu == -1) {
continue;
}
ThreadInfo info = threadBean.getThreadInfo(threadId, 0);
if (info == null) {
continue;
}
threadInfos.put(threadId, new MyThreadInfo(cpu, info));
}
Thread.sleep(interval.millis());
for (long threadId : threadBean.getAllThreadIds()) {
// ignore our own thread...
if (Thread.currentThread().getId() == threadId) {
continue;
}
long cpu = threadBean.getThreadCpuTime(threadId);
if (cpu == -1) {
threadInfos.remove(threadId);
continue;
}
ThreadInfo info = threadBean.getThreadInfo(threadId, 0);
if (info == null) {
threadInfos.remove(threadId);
continue;
}
MyThreadInfo data = threadInfos.get(threadId);
if (data != null) {
data.setDelta(cpu, info);
} else {
threadInfos.remove(threadId);
}
}
// sort by delta CPU time on thread.
List<MyThreadInfo> hotties = new ArrayList<>(threadInfos.values());
final int busiestThreads = Math.min(this.busiestThreads, hotties.size());
// skip that for now
CollectionUtil.introSort(
hotties,
new Comparator<MyThreadInfo>() {
@Override
public int compare(MyThreadInfo o1, MyThreadInfo o2) {
if ("cpu".equals(type)) {
return (int) (o2.cpuTime - o1.cpuTime);
} else if ("wait".equals(type)) {
return (int) (o2.waitedTime - o1.waitedTime);
} else if ("block".equals(type)) {
return (int) (o2.blockedTime - o1.blockedTime);
}
throw new IllegalArgumentException();
}
});
// analyse N stack traces for M busiest threads
long[] ids = new long[busiestThreads];
for (int i = 0; i < busiestThreads; i++) {
MyThreadInfo info = hotties.get(i);
ids[i] = info.info.getThreadId();
}
ThreadInfo[][] allInfos = new ThreadInfo[threadElementsSnapshotCount][];
for (int j = 0; j < threadElementsSnapshotCount; j++) {
// NOTE, javadoc of getThreadInfo says: If a thread of the given ID is not alive or does not
// exist,
// null will be set in the corresponding element in the returned array. A thread is alive if
// it has
// been started and has not yet died.
allInfos[j] = threadBean.getThreadInfo(ids, Integer.MAX_VALUE);
Thread.sleep(threadElementsSnapshotDelay.millis());
}
for (int t = 0; t < busiestThreads; t++) {
long time = 0;
if ("cpu".equals(type)) {
time = hotties.get(t).cpuTime;
} else if ("wait".equals(type)) {
time = hotties.get(t).waitedTime;
} else if ("block".equals(type)) {
time = hotties.get(t).blockedTime;
}
String threadName = null;
for (ThreadInfo[] info : allInfos) {
if (info != null && info[t] != null) {
if (ignoreIdleThreads && isIdleThread(info[t])) {
info[t] = null;
continue;
}
threadName = info[t].getThreadName();
break;
}
}
if (threadName == null) {
continue; // thread is not alive yet or died before the first snapshot - ignore it!
}
double percent = (((double) time) / interval.nanos()) * 100;
sb.append(
String.format(
Locale.ROOT,
"%n%4.1f%% (%s out of %s) %s usage by thread '%s'%n",
percent,
TimeValue.timeValueNanos(time),
interval,
type,
threadName));
// for each snapshot (2nd array index) find later snapshot for same thread with max number
// of
// identical StackTraceElements (starting from end of each)
boolean[] done = new boolean[threadElementsSnapshotCount];
for (int i = 0; i < threadElementsSnapshotCount; i++) {
if (done[i]) continue;
int maxSim = 1;
boolean[] similars = new boolean[threadElementsSnapshotCount];
for (int j = i + 1; j < threadElementsSnapshotCount; j++) {
if (done[j]) continue;
int similarity = similarity(allInfos[i][t], allInfos[j][t]);
if (similarity > maxSim) {
maxSim = similarity;
similars = new boolean[threadElementsSnapshotCount];
}
if (similarity == maxSim) similars[j] = true;
}
// print out trace maxSim levels of i, and mark similar ones as done
int count = 1;
for (int j = i + 1; j < threadElementsSnapshotCount; j++) {
if (similars[j]) {
done[j] = true;
count++;
}
}
if (allInfos[i][t] != null) {
final StackTraceElement[] show = allInfos[i][t].getStackTrace();
if (count == 1) {
sb.append(String.format(Locale.ROOT, " unique snapshot%n"));
for (int l = 0; l < show.length; l++) {
sb.append(String.format(Locale.ROOT, " %s%n", show[l]));
}
} else {
sb.append(
String.format(
Locale.ROOT,
" %d/%d snapshots sharing following %d elements%n",
count,
threadElementsSnapshotCount,
maxSim));
for (int l = show.length - maxSim; l < show.length; l++) {
sb.append(String.format(Locale.ROOT, " %s%n", show[l]));
}
}
}
}
}
return sb.toString();
} finally {
if (enabledCpu) {
threadBean.setThreadCpuTimeEnabled(false);
}
}
}