void checkThreadInfo(ThreadInfo info) {
if (!getName().equals(info.getThreadName())) {
throw new RuntimeException(
"Name: " + info.getThreadName() + " not matched. Expected: " + getName());
}
MonitorInfo[] monitors = info.getLockedMonitors();
if (monitors.length != OWNED_MONITORS) {
throw new RuntimeException(
"Number of locked monitors = "
+ monitors.length
+ " not matched. Expected: "
+ OWNED_MONITORS);
}
MonitorInfo m = monitors[0];
StackTraceElement ste = m.getLockedStackFrame();
int depth = m.getLockedStackDepth();
StackTraceElement[] stacktrace = info.getStackTrace();
if (!ste.equals(stacktrace[depth])) {
System.out.println("LockedStackFrame:- " + ste);
System.out.println("StackTrace at " + depth + " :-" + stacktrace[depth]);
throw new RuntimeException(
"LockedStackFrame does not match " + "stack frame in ThreadInfo.getStackTrace");
}
String className = lock.getClass().getName();
int hcode = System.identityHashCode(lock);
if (!className.equals(m.getClassName())
|| hcode != m.getIdentityHashCode()
|| !m.getLockedStackFrame().getMethodName().equals("run")) {
System.out.println(info);
throw new RuntimeException("MonitorInfo " + m + " doesn't match.");
}
LockInfo[] syncs = info.getLockedSynchronizers();
if (syncs.length != OWNED_SYNCS) {
throw new RuntimeException(
"Number of locked syncs = " + syncs.length + " not matched. Expected: " + OWNED_SYNCS);
}
AbstractOwnableSynchronizer s = mutex.getSync();
String lockName = s.getClass().getName();
hcode = System.identityHashCode(s);
if (!lockName.equals(syncs[0].getClassName())) {
throw new RuntimeException(
"LockInfo : " + syncs[0] + " class name not matched. Expected: " + lockName);
}
if (hcode != syncs[0].getIdentityHashCode()) {
throw new RuntimeException(
"LockInfo: " + syncs[0] + " IdentityHashCode not matched. Expected: " + hcode);
}
LockInfo li = info.getLockInfo();
if (li == null) {
throw new RuntimeException("Expected non-null LockInfo");
}
}
public Map<String, String> call() {
Map<String, String> r = new LinkedHashMap<String, String>();
ThreadInfo[] data = Functions.getThreadInfos();
Functions.ThreadGroupMap map = Functions.sortThreadsAndGetGroupMap(data);
for (ThreadInfo ti : data) r.put(ti.getThreadName(), Functions.dumpThreadInfo(ti, map));
return r;
}
/**
* Print all of the thread's information and stack traces.
*
* @param sb
* @param info
* @param indent
*/
public static void appendThreadInfo(StringBuilder sb, ThreadInfo info, String indent) {
boolean contention = threadBean.isThreadContentionMonitoringEnabled();
if (info == null) {
sb.append(indent).append("Inactive (perhaps exited while monitoring was done)\n");
return;
}
String taskName = getTaskName(info.getThreadId(), info.getThreadName());
sb.append(indent).append("Thread ").append(taskName).append(":\n");
Thread.State state = info.getThreadState();
sb.append(indent).append(" State: ").append(state).append("\n");
sb.append(indent).append(" Blocked count: ").append(info.getBlockedCount()).append("\n");
sb.append(indent).append(" Waited count: ").append(info.getWaitedCount()).append("\n");
if (contention) {
sb.append(indent).append(" Blocked time: " + info.getBlockedTime()).append("\n");
sb.append(indent).append(" Waited time: " + info.getWaitedTime()).append("\n");
}
if (state == Thread.State.WAITING) {
sb.append(indent).append(" Waiting on ").append(info.getLockName()).append("\n");
} else if (state == Thread.State.BLOCKED) {
sb.append(indent).append(" Blocked on ").append(info.getLockName()).append("\n");
sb.append(indent)
.append(" Blocked by ")
.append(getTaskName(info.getLockOwnerId(), info.getLockOwnerName()))
.append("\n");
}
sb.append(indent).append(" Stack:").append("\n");
for (StackTraceElement frame : info.getStackTrace()) {
sb.append(indent).append(" ").append(frame.toString()).append("\n");
}
}
public static void main(String[] args) {
// 获取Java线程管理MXBean
ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
ThreadInfo[] threadInfos = threadMXBean.dumpAllThreads(false, false);
for (ThreadInfo threadInfo : threadInfos) {
System.out.println("[" + threadInfo.getThreadId() + " ] " + threadInfo.getThreadName());
}
}
public static CharSequence getThreadStats(
boolean lockedMonitors, boolean lockedSynchronizers, boolean stackTrace) {
StringBuilder list = new StringBuilder();
int threadCount = threadMXbean.getThreadCount();
int daemonCount = threadMXbean.getThreadCount();
int nonDaemonCount = threadCount - daemonCount;
int peakCount = threadMXbean.getPeakThreadCount();
long totalCount = threadMXbean.getTotalStartedThreadCount();
list.append("Live: .................... ").append(threadCount).append(" threads").append("\n");
list.append(" Non-Daemon: ......... ")
.append(nonDaemonCount)
.append(" threads")
.append("\n");
list.append(" Daemon: ............. ").append(daemonCount).append(" threads").append("\n");
list.append("Peak: .................... ").append(peakCount).append(" threads").append("\n");
list.append("Total started: ........... ").append(totalCount).append(" threads").append("\n");
list.append("=================================================").append("\n");
for (ThreadInfo info : threadMXbean.dumpAllThreads(lockedMonitors, lockedSynchronizers)) {
list.append("Thread #")
.append(info.getThreadId())
.append(" (")
.append(info.getThreadName())
.append(")")
.append("\n");
list.append("=================================================\n");
list.append("\tgetThreadState: ...... ").append(info.getThreadState()).append("\n");
list.append("\tgetWaitedTime: ....... ").append(info.getWaitedTime()).append("\n");
list.append("\tgetBlockedTime: ...... ").append(info.getBlockedTime()).append("\n");
for (MonitorInfo monitorInfo : info.getLockedMonitors()) {
list.append("\tLocked monitor: ....... ").append(monitorInfo).append("\n");
list.append("\t\t[")
.append(monitorInfo.getLockedStackDepth())
.append(".]: at ")
.append(monitorInfo.getLockedStackFrame())
.append("\n");
}
for (LockInfo lockInfo : info.getLockedSynchronizers()) {
list.append("\tLocked synchronizer: ...").append(lockInfo).append("\n");
}
if (stackTrace) {
list.append("\tgetStackTace: ..........\n");
for (StackTraceElement trace : info.getStackTrace())
list.append("\t\tat ").append(trace).append("\n");
}
list.append("=================================================\n");
}
return list;
}
@Before
public void setUp() throws Exception {
final StackTraceElement rLine1 = new StackTraceElement("Blah", "blee", "Blah.java", 100);
when(runnable.getThreadName()).thenReturn("runnable");
when(runnable.getThreadId()).thenReturn(100L);
when(runnable.getThreadState()).thenReturn(Thread.State.RUNNABLE);
when(runnable.getStackTrace()).thenReturn(new StackTraceElement[] {rLine1});
when(runnable.getLockedMonitors()).thenReturn(new MonitorInfo[] {});
when(runnable.getLockedSynchronizers()).thenReturn(new LockInfo[] {});
when(threadMXBean.dumpAllThreads(true, true)).thenReturn(new ThreadInfo[] {runnable});
}
private static void printThreadInfo(CompositeData cd) {
ThreadInfo info = ThreadInfo.from(cd);
if (info == null) {
throw new RuntimeException("TEST FAILED: " + " Null ThreadInfo");
}
System.out.print(info.getThreadName());
System.out.print(" id=" + info.getThreadId());
System.out.println(" " + info.getThreadState());
for (StackTraceElement s : info.getStackTrace()) {
System.out.println(s);
}
}
@DwrPermission(admin = true)
public ProcessResult getThreadInfo() {
synchronized (threadInfos) {
ProcessResult result = new ProcessResult();
// All of the last thread ids. Ids are removed from this set as they are processed. If ids
// remain,
// it means the thread is gone and should be removed from the map.
Set<Long> threadIds = new HashSet<>(threadInfos.keySet());
ThreadInfo[] threads = tmxb.getThreadInfo(tmxb.getAllThreadIds(), Integer.MAX_VALUE);
List<ThreadInfoBean> beans = new ArrayList<>();
for (ThreadInfo thread : threads) {
if (thread == null) continue;
ThreadInfoBean bean = threadInfos.get(thread.getThreadId());
if (bean == null) {
bean = new ThreadInfoBean();
bean.setId(thread.getThreadId());
bean.setName(thread.getThreadName());
threadInfos.put(bean.getId(), bean);
} else threadIds.remove(bean.getId());
bean.setCpuTime(tmxb.getThreadCpuTime(bean.getId()));
bean.setState(thread.getThreadState().name());
if (thread.getThreadState() == State.BLOCKED)
bean.setState(
bean.getState()
+ " by '"
+ thread.getLockOwnerName()
+ "' ("
+ thread.getLockOwnerId()
+ ")");
bean.setStackTrace(thread.getStackTrace());
beans.add(bean);
}
// Remove unreferenced threads
for (Long id : threadIds) threadInfos.remove(id);
result.addData("threads", beans);
return result;
}
}
private String threadDumpDescription() {
StringBuilder threadDumpDescription = new StringBuilder();
ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
ThreadInfo[] threadInfos = threadMXBean.dumpAllThreads(true, true);
for (ThreadInfo threadInfo : threadInfos) {
threadDumpDescription.append(
format(
"\"%s\"%n\tjava.lang.Thread.State: %s",
threadInfo.getThreadName(), threadInfo.getThreadState()));
for (StackTraceElement stackTraceElement : threadInfo.getStackTrace()) {
threadDumpDescription.append(LINE_SEPARATOR + "\t\tat " + stackTraceElement);
}
threadDumpDescription.append(LINE_SEPARATOR + LINE_SEPARATOR);
}
return threadDumpDescription.toString();
}
/**
* A total hack to see if there are any active Java3D threads running
*
* @return any java3d threads running
*/
private static boolean anyJava3dThreadsActive() {
ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
long[] ids = threadBean.getAllThreadIds();
for (int i = 0; i < ids.length; i++) {
ThreadInfo info = threadBean.getThreadInfo(ids[i], Integer.MAX_VALUE);
if (info == null) {
continue;
}
if (info.getThreadState() != Thread.State.RUNNABLE) {
continue;
}
if (info.getThreadName().indexOf("J3D") >= 0) {
return true;
}
}
return false;
}
/**
* Formats the thread dump header for one thread.
*
* @param ti the ThreadInfo describing the thread
* @return the formatted thread dump header
*/
private static String getThreadDumpHeader(ThreadInfo ti) {
StringBuilder sb = new StringBuilder("\"" + ti.getThreadName() + "\"");
sb.append(" Id=" + ti.getThreadId());
sb.append(" cpu=" + threadMXBean.getThreadCpuTime(ti.getThreadId()) + " ns");
sb.append(" usr="******" ns");
sb.append(" blocked " + ti.getBlockedCount() + " for " + ti.getBlockedTime() + " ms");
sb.append(" waited " + ti.getWaitedCount() + " for " + ti.getWaitedTime() + " ms");
if (ti.isSuspended()) {
sb.append(" (suspended)");
}
if (ti.isInNative()) {
sb.append(" (running in native)");
}
sb.append(CRLF);
sb.append(INDENT3 + "java.lang.Thread.State: " + ti.getThreadState());
sb.append(CRLF);
return sb.toString();
}
@RequestMapping("/loadThreadInfo")
@ResponseBody
public JSONObject doLoadThreadInfo(HttpServletRequest request) {
try {
String app = request.getParameter("app");
ThreadMXBean tBean = JMConnManager.getThreadMBean(app);
ThreadInfo[] allThreads = tBean.dumpAllThreads(false, false);
JSONObject root = new JSONObject();
JSONArray detail = new JSONArray();
HashMap<State, Integer> state = new HashMap<Thread.State, Integer>();
for (ThreadInfo info : allThreads) {
JSONObject th = new JSONObject();
long threadId = info.getThreadId();
long cpu = tBean.getThreadCpuTime(threadId);
State tState = info.getThreadState();
th.put("id", threadId);
th.put("state", tState);
th.put("name", info.getThreadName());
th.put("cpu", TimeUnit.NANOSECONDS.toMillis(cpu));
detail.add(th);
Integer vl = state.get(tState);
if (vl == null) {
state.put(tState, 0);
} else {
state.put(tState, vl + 1);
}
}
root.put("state", state);
root.put("detail", detail);
root.put("total", tBean.getThreadCount());
root.put("time", System.currentTimeMillis());
root.put("deamon", tBean.getDaemonThreadCount());
return root;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub
if (args.length != 4) {
System.err.println("Please provide process id zabbix-host zabbix-port host-guid");
System.exit(-1);
}
String processPid = args[0];
String zabbixHost = args[1];
String zabbixPort = args[2];
String hostGuid = args[3];
VirtualMachine vm = VirtualMachine.attach(processPid);
String connectorAddr =
vm.getAgentProperties().getProperty("com.sun.management.jmxremote.localConnectorAddress");
if (connectorAddr == null) {
String agent =
vm.getSystemProperties().getProperty("java.home")
+ File.separator
+ "lib"
+ File.separator
+ "management-agent.jar";
vm.loadAgent(agent);
connectorAddr =
vm.getAgentProperties().getProperty("com.sun.management.jmxremote.localConnectorAddress");
}
JMXServiceURL serviceURL = new JMXServiceURL(connectorAddr);
JMXConnector connector = JMXConnectorFactory.connect(serviceURL);
MBeanServerConnection mbsc = connector.getMBeanServerConnection();
ObjectName objName = new ObjectName(ManagementFactory.THREAD_MXBEAN_NAME);
Set<ObjectName> mbeans = mbsc.queryNames(objName, null);
for (ObjectName name : mbeans) {
ThreadMXBean threadBean;
threadBean =
ManagementFactory.newPlatformMXBeanProxy(mbsc, name.toString(), ThreadMXBean.class);
long threadIds[] = threadBean.getAllThreadIds();
for (long threadId : threadIds) {
ThreadInfo threadInfo = threadBean.getThreadInfo(threadId);
System.out.println(threadInfo.getThreadName() + " / " + threadInfo.getThreadState());
}
}
}
protected String generateThreadDump() {
StringBuilder dump = new StringBuilder();
ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
ThreadInfo[] threadInfos = threadMXBean.dumpAllThreads(true, true);
long currentThreadId = Thread.currentThread().getId();
for (ThreadInfo threadInfo : threadInfos) {
long threadId = threadInfo.getThreadId();
if (threadId == currentThreadId) {
continue;
}
dump.append('"');
dump.append(threadInfo.getThreadName());
dump.append("\" ");
Thread.State state = threadInfo.getThreadState();
dump.append("\n\tjava.lang.Thread.State: ");
dump.append(state);
if (threadInfo.getLockName() != null) {
dump.append(", on lock=").append(threadInfo.getLockName());
}
if (threadInfo.getLockOwnerName() != null) {
dump.append(", owned by ").append(threadInfo.getLockOwnerName());
dump.append(", id=").append(threadInfo.getLockOwnerId());
}
if (THREAD_CPU_TIME_INFO_AVAILABLE) {
dump.append(", cpu=").append(threadMXBean.getThreadCpuTime(threadId)).append(" nsecs");
dump.append(", usr="******" nsecs");
}
if (THREAD_CONTENTION_INFO_AVAILABLE) {
dump.append(", blocked=").append(threadInfo.getBlockedTime()).append(" msecs");
dump.append(", waited=").append(threadInfo.getWaitedTime()).append(" msecs");
}
StackTraceElement[] stackTraceElements = threadInfo.getStackTrace();
for (StackTraceElement stackTraceElement : stackTraceElements) {
dump.append("\n\t\tat ");
dump.append(stackTraceElement);
}
dump.append("\n\n");
}
return dump.toString();
}
/**
* Return current (raw) list of thread info objects wrapped in ThreadRankInfo type.
*
* @return list of threads
*/
protected List<ThreadRankInfo> rawList() {
// Platform MBean Server startup might be suspended (eg. for JBoss AS);
if (threadMXBean == null) {
if (mBeanServerRegistry.lookup("java") != null) {
threadMXBean = ManagementFactory.getThreadMXBean();
} else {
return new ArrayList<ThreadRankInfo>(1);
}
}
ThreadInfo[] ati = threadMXBean.getThreadInfo(threadMXBean.getAllThreadIds());
List<ThreadRankInfo> lst = new ArrayList<ThreadRankInfo>(ati.length);
for (ThreadInfo ti : ati) {
long tid = ti.getThreadId();
long cpuTime = threadMXBean.getThreadCpuTime(tid);
lst.add(new ThreadRankInfo(tid, ti.getThreadName(), cpuTime, ti.getBlockedTime()));
}
return lst;
}
public static String threadDump() {
final StringBuilder dump = new StringBuilder();
final java.lang.management.ThreadMXBean threadMXBean =
java.lang.management.ManagementFactory.getThreadMXBean();
final java.lang.management.ThreadInfo[] threadInfos =
threadMXBean.getThreadInfo(threadMXBean.getAllThreadIds(), 100);
for (java.lang.management.ThreadInfo threadInfo : threadInfos) {
dump.append('"');
dump.append(threadInfo.getThreadName());
dump.append("\" ");
final Thread.State state = threadInfo.getThreadState();
dump.append("\n java.lang.Thread.State: ");
dump.append(state);
final StackTraceElement[] stackTraceElements = threadInfo.getStackTrace();
for (final StackTraceElement stackTraceElement : stackTraceElements) {
dump.append("\n at ");
dump.append(stackTraceElement);
}
dump.append("\n\n");
}
return dump.toString();
}
private static SimpleOrderedMap<Object> getThreadInfo(ThreadInfo ti, ThreadMXBean tmbean) {
SimpleOrderedMap<Object> info = new SimpleOrderedMap<Object>();
long tid = ti.getThreadId();
info.add("id", tid);
info.add("name", ti.getThreadName());
info.add("state", ti.getThreadState().toString());
if (ti.getLockName() != null) {
info.add("lock", ti.getLockName());
}
if (ti.isSuspended()) {
info.add("suspended", true);
}
if (ti.isInNative()) {
info.add("native", true);
}
if (tmbean.isThreadCpuTimeSupported()) {
info.add("cpuTime", formatNanos(tmbean.getThreadCpuTime(tid)));
info.add("userTime", formatNanos(tmbean.getThreadUserTime(tid)));
}
if (ti.getLockOwnerName() != null) {
SimpleOrderedMap<Object> owner = new SimpleOrderedMap<Object>();
owner.add("name", ti.getLockOwnerName());
owner.add("id", ti.getLockOwnerId());
}
// Add the stack trace
int i = 0;
String[] trace = new String[ti.getStackTrace().length];
for (StackTraceElement ste : ti.getStackTrace()) {
trace[i++] = ste.toString();
}
info.add("stackTrace", trace);
return info;
}
private static boolean isIdleThread(ThreadInfo threadInfo) {
String threadName = threadInfo.getThreadName();
// NOTE: these are likely JVM dependent
if (threadName.equals("Signal Dispatcher")
|| threadName.equals("Finalizer")
|| threadName.equals("Reference Handler")) {
return true;
}
for (StackTraceElement frame : threadInfo.getStackTrace()) {
String className = frame.getClassName();
String methodName = frame.getMethodName();
if (className.equals("java.util.concurrent.ThreadPoolExecutor")
&& methodName.equals("getTask")) {
return true;
}
if (className.equals("sun.nio.ch.SelectorImpl") && methodName.equals("select")) {
return true;
}
if (className.equals("org.elasticsearch.threadpool.ThreadPool$EstimatedTimeThread")
&& methodName.equals("run")) {
return true;
}
if (className.equals("org.elasticsearch.indices.ttl.IndicesTTLService$Notifier")
&& methodName.equals("await")) {
return true;
}
if (className.equals("java.util.concurrent.LinkedTransferQueue")
&& methodName.equals("poll")) {
return true;
}
}
return false;
}
private static void printThread(ThreadInfo threadInfo, StringBuilder threadDump) {
StringBuilder sb =
new StringBuilder(
"\""
+ threadInfo.getThreadName()
+ "\""
+ " nid="
+ threadInfo.getThreadId()
+ " state="
+ threadInfo.getThreadState());
if (threadInfo.getLockName() != null && threadInfo.getThreadState() != Thread.State.BLOCKED) {
String[] lockInfo = threadInfo.getLockName().split("@");
sb.append("\n" + INDENT + "- waiting on <0x" + lockInfo[1] + "> (a " + lockInfo[0] + ")");
sb.append("\n" + INDENT + "- locked <0x" + lockInfo[1] + "> (a " + lockInfo[0] + ")");
} else if (threadInfo.getLockName() != null
&& threadInfo.getThreadState() == Thread.State.BLOCKED) {
String[] lockInfo = threadInfo.getLockName().split("@");
sb.append(
"\n" + INDENT + "- waiting to lock <0x" + lockInfo[1] + "> (a " + lockInfo[0] + ")");
}
if (threadInfo.isSuspended()) sb.append(" (suspended)");
if (threadInfo.isInNative()) sb.append(" (running in native)");
threadDump.append(sb.toString());
threadDump.append("\n");
if (threadInfo.getLockOwnerName() != null) {
threadDump.append(
INDENT
+ " owned by "
+ threadInfo.getLockOwnerName()
+ " id="
+ threadInfo.getLockOwnerId());
threadDump.append("\n");
}
}
/**
* スレッドの状態をStringBufferにして返します。<br>
*
* @param info {@link ThreadInfo}オブジェクト
* @return スレッドの状態をStringBufferにして返したもの。
*/
private static StringBuilder getThreadInfoBuffer(ThreadInfo info) {
StringBuilder sb =
new StringBuilder(
"\""
+ info.getThreadName()
+ "\""
+ " Id="
+ info.getThreadId()
+ " "
+ info.getThreadState());
if (info.getLockName() != null) {
sb.append(" on " + info.getLockName());
}
if (info.getLockOwnerName() != null) {
sb.append(" owned by \"" + info.getLockOwnerName() + "\" Id=" + info.getLockOwnerId());
}
if (info.isSuspended()) {
sb.append(" (suspended)");
}
if (info.isInNative()) {
sb.append(" (in native)");
}
return sb;
}
// ThreadInfo.toString() truncates the stack trace by first 8, so needed my own version
@IgnoreJRERequirement
public static String dumpThreadInfo(ThreadInfo ti) {
StringBuilder sb =
new StringBuilder(
"\""
+ ti.getThreadName()
+ "\""
+ " Id="
+ ti.getThreadId()
+ " "
+ ti.getThreadState());
if (ti.getLockName() != null) {
sb.append(" on " + ti.getLockName());
}
if (ti.getLockOwnerName() != null) {
sb.append(" owned by \"" + ti.getLockOwnerName() + "\" Id=" + ti.getLockOwnerId());
}
if (ti.isSuspended()) {
sb.append(" (suspended)");
}
if (ti.isInNative()) {
sb.append(" (in native)");
}
sb.append('\n');
StackTraceElement[] stackTrace = ti.getStackTrace();
for (int i = 0; i < stackTrace.length; i++) {
StackTraceElement ste = stackTrace[i];
sb.append("\tat " + ste.toString());
sb.append('\n');
if (i == 0 && ti.getLockInfo() != null) {
Thread.State ts = ti.getThreadState();
switch (ts) {
case BLOCKED:
sb.append("\t- blocked on " + ti.getLockInfo());
sb.append('\n');
break;
case WAITING:
sb.append("\t- waiting on " + ti.getLockInfo());
sb.append('\n');
break;
case TIMED_WAITING:
sb.append("\t- waiting on " + ti.getLockInfo());
sb.append('\n');
break;
default:
}
}
for (MonitorInfo mi : ti.getLockedMonitors()) {
if (mi.getLockedStackDepth() == i) {
sb.append("\t- locked " + mi);
sb.append('\n');
}
}
}
LockInfo[] locks = ti.getLockedSynchronizers();
if (locks.length > 0) {
sb.append("\n\tNumber of locked synchronizers = " + locks.length);
sb.append('\n');
for (LockInfo li : locks) {
sb.append("\t- " + li);
sb.append('\n');
}
}
sb.append('\n');
return sb.toString();
}
private void printThread() {
long now = System.currentTimeMillis();
long diffLastThreadDump = now - lastThreadDump;
logger.info("diffLastThreadDump:" + diffLastThreadDump);
if (diffLastThreadDump > 60000) {
logger.info("had not sent all threads for a while.. will do so now");
lastThreadDump = now;
try {
ThreadMXBean t = ManagementFactory.getThreadMXBean();
long threads[] = t.getAllThreadIds();
ThreadInfo[] tinfo = t.getThreadInfo(threads, 40);
StringBuilder sb = new StringBuilder("All Threads");
for (int i = 0; i < tinfo.length; i++) {
ThreadInfo e = tinfo[i];
try {
StackTraceElement[] el = e.getStackTrace();
sb.append(
"\n\n"
+ e.getThreadName()
+ "\n"
+ " "
+ " Thread id = "
+ e.getThreadId()
+ " "
+ e.getThreadState());
if (e.getThreadState().equals(State.BLOCKED)) {
sb.append(
"\n\nBlocked info: "
+ e.getBlockedCount()
+ ":"
+ e.getBlockedTime()
+ ":"
+ e.getLockName()
+ ":"
+ e.getLockOwnerId()
+ ":"
+ e.getLockOwnerName()
+ "\n"
+ " "
+ " Thread id = "
+ e.getThreadId()
+ " "
+ e.getThreadState());
ThreadInfo eBlockedThread = t.getThreadInfo(e.getLockOwnerId(), 40);
StackTraceElement[] elBlockedThread = eBlockedThread.getStackTrace();
sb.append(
"\n\n "
+ e.getThreadName()
+ "\n"
+ " "
+ " Thread id = "
+ eBlockedThread.getThreadId()
+ " "
+ eBlockedThread.getThreadState());
if (elBlockedThread == null || elBlockedThread.length == 0) {
sb.append(" no stack trace available");
} else {
for (int n = 0; n < elBlockedThread.length; n++) {
if (n != 0) sb.append("\n");
StackTraceElement frame = elBlockedThread[n];
if (frame == null) {
sb.append(" null stack frame");
continue;
}
sb.append(" ");
sb.append(frame.toString());
}
}
}
if (el == null || el.length == 0) {
sb.append(" no stack trace available");
continue;
}
for (int n = 0; n < el.length; n++) {
if (n != 0) sb.append("\n");
StackTraceElement frame = el[n];
if (frame == null) {
sb.append(" null stack frame");
continue;
}
sb.append(" ");
sb.append(frame.toString());
}
} catch (Exception e2) {
}
}
String warningEmailReceiver = CmsPropertyHandler.getWarningEmailReceiver();
if (warningEmailReceiver != null
&& !warningEmailReceiver.equals("")
&& warningEmailReceiver.indexOf("@warningEmailReceiver@") == -1) {
try {
logger.info("Mailing..");
MailServiceFactory.getService()
.sendEmail(
CmsPropertyHandler.getMailContentType(),
warningEmailReceiver,
warningEmailReceiver,
null,
null,
null,
null,
message,
sb.toString().replaceAll("\n", "<br/>"),
"utf-8");
} catch (Exception e) {
logger.error("Could not send mail:" + e.getMessage(), e);
}
}
} catch (Throwable e) {
logger.error("Error generating message:" + e.getMessage(), e);
}
}
// Only sends if the last stack was sent more than 3 seconds ago.
if ((now - lastSentTimer) > 10000) {
lastSentTimer = System.currentTimeMillis();
StackTraceElement[] el = targetThread.getStackTrace();
ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
long stopTimeInNs = threadMXBean.getThreadUserTime(targetThread.getId());
long diff = (stopTimeInNs - startTimeInNs) / 1000000000;
StringBuffer stackString = new StringBuffer("\n\n" + message + "\n\n");
stackString.append("ServerName: " + getServerName() + "\n");
stackString.append("Maximum memory (MB): " + (maxMemory / 1024 / 1024) + "\n");
stackString.append("Used memory (MB): " + ((totalMemory - freeMemory) / 1024 / 1024) + "\n");
stackString.append("Free memory (MB): " + (freeMemory / 1024 / 1024) + "\n");
stackString.append("Total memory (MB): " + (totalMemory / 1024 / 1024) + "\n");
stackString.append("Number of current requests: " + numberOfCurrentRequests + "\n");
stackString.append("Number of active requests: " + numberOfActiveRequests + "\n");
stackString.append("Number of long requests: " + longThreadMonitorsSize + "\n");
stackString.append("Current thread time: " + diff + " seconds\n");
stackString.append(
"Average time: " + RequestAnalyser.getRequestAnalyser().getAverageElapsedTime() + "\n");
stackString.append(
"Longest time: " + RequestAnalyser.getRequestAnalyser().getMaxElapsedTime() + "\n");
stackString.append("Original url: " + getOriginalFullURL() + "\n");
stackString.append("UserInfo: " + getUserInfo() + "\n");
stackString.append("--------------------------------------------\n\n");
stackString.append("Thread with id [" + targetThread.getId() + "] at report time:\n");
if (el != null && el.length != 0) {
for (int j = 0; j < el.length; j++) {
StackTraceElement frame = el[j];
if (frame == null) stackString.append(" null stack frame" + "\n");
else stackString.append(" ").append(frame.toString()).append("\n");
// if((stackString.indexOf("infoglue") > -1 && j > 20) || j > 35)
// break;
}
}
if (targetThread.getState().equals(State.BLOCKED)) {
ThreadMXBean t = ManagementFactory.getThreadMXBean();
ThreadInfo e = t.getThreadInfo(targetThread.getId(), 40);
stackString.append(
"\n\nBlocked info: "
+ e.getBlockedCount()
+ ":"
+ e.getBlockedTime()
+ ":"
+ e.getLockName()
+ ":"
+ e.getLockOwnerId()
+ ":"
+ e.getLockOwnerName()
+ "\n"
+ " "
+ " Thread id = "
+ e.getThreadId()
+ " "
+ e.getThreadState());
ThreadInfo eBlockedThread = t.getThreadInfo(e.getLockOwnerId(), 40);
StackTraceElement[] elBlockedThread = eBlockedThread.getStackTrace();
stackString.append(
"\n\nBlocked thread: "
+ e.getThreadName()
+ "\n"
+ " "
+ " Thread id = "
+ eBlockedThread.getThreadId()
+ " "
+ eBlockedThread.getThreadState());
if (elBlockedThread == null || elBlockedThread.length == 0) {
stackString.append(" no stack trace available");
} else {
for (int n = 0; n < elBlockedThread.length; n++) {
if (n != 0) stackString.append("\n");
StackTraceElement frame = elBlockedThread[n];
if (frame == null) {
stackString.append(" null stack frame");
continue;
}
stackString.append(" ");
stackString.append(frame.toString());
}
}
}
stackString.append(
"\n\n**********************************\nConcurrent long threads (Only an excerpt of all)\n**********************************");
ThreadMXBean t = ManagementFactory.getThreadMXBean();
List threadMonitors = RequestAnalyser.getLongThreadMonitors();
Iterator threadMonitorsIterator = threadMonitors.iterator();
int threadCount = 0;
while (threadMonitorsIterator.hasNext() && threadCount < 5) {
SimpleThreadMonitor tm = (SimpleThreadMonitor) threadMonitorsIterator.next();
if (targetThread.getId() == tm.getThreadId()) continue;
long threads[] = {tm.getThreadId()};
ThreadInfo[] tinfo = t.getThreadInfo(threads, 40);
stackString
.append("\n\n---------------------------------\nConcurrent long thread [")
.append(tm.getThreadId())
.append("]:\n");
stackString
.append("Elapsed time:")
.append(tm.getElapsedTime())
.append("\n Thread id: ")
.append(tm.getThreadId())
.append("\n Original url: ")
.append(tm.getOriginalFullURL())
.append(")");
for (int i = 0; i < tinfo.length; i++) {
ThreadInfo e = tinfo[i];
el = e.getStackTrace();
if (el != null && el.length != 0) {
for (int n = 0; n < el.length; n++) {
StackTraceElement frame = el[n];
if (frame == null) stackString.append(" null stack frame\n");
else stackString.append(" null stack frame").append(frame.toString()).append("\n");
}
}
}
threadCount++;
}
logger.warn(stackString);
} else {
logger.warn(
"A thread took to long but the system seems to be really clogged so we don't send this one.");
}
}
@Override
public void doTest() throws Throwable {
ThreadMXBean tbean;
tbean = ManagementFactory.getThreadMXBean();
int nonDaemonThreadCountA = tbean.getThreadCount() - tbean.getDaemonThreadCount();
int daemonThreadCountA = tbean.getDaemonThreadCount();
long[] listA = tbean.getAllThreadIds();
for (int loopNumber = 0; loopNumber < 4; loopNumber++) {
cacheManager =
new CacheManager(
DaemonThreadsWriteBehindTestClient.class.getResourceAsStream("/ehcache-config.xml"));
int daemonThreadCountB = tbean.getDaemonThreadCount();
Assert.assertTrue(daemonThreadCountA < daemonThreadCountB);
Cache cache = cacheManager.getCache("test");
cache.registerCacheWriter(new WriteBehindCacheWriter(this));
Assert.assertNotNull(cache.getWriterManager());
Assert.assertTrue(cache.getWriterManager() instanceof WriteBehindManager);
for (int i = 0; i < 10; i++) {
cache.putWithWriter(new Element(i, i));
}
while (getWriteCount() < 10) {
Thread.sleep(200);
}
resetWriteCount();
cacheManager.shutdown();
System.out.println("done with iteration " + loopNumber);
}
TimeUnit.MINUTES.sleep(1L);
long[] listC = tbean.getAllThreadIds();
int daemonThreadCountC = tbean.getDaemonThreadCount();
int nonDaemonThreadCountC = tbean.getThreadCount() - tbean.getDaemonThreadCount();
List<Long> listIntA = new ArrayList<Long>();
for (long listAItrator : listA) {
listIntA.add(new Long(listAItrator));
}
List<Long> listIntC = new ArrayList<Long>();
for (long listAItrator : listC) {
listIntC.add(new Long(listAItrator));
}
listIntC.removeAll(listIntA);
Set<String> knownThreads = getKnownThreads();
int skipThreadCount = 0;
StringBuffer threadsInfo = new StringBuffer();
System.out.println(
"\n\n" + listIntC.size() + " Start Printing Stack Trace\n--------------------");
for (int i = 0; i < listIntC.size(); i++) {
ThreadInfo tinfo = tbean.getThreadInfo(listIntC.get(i));
if (knownThreads.contains(tinfo.getThreadName().trim())) {
++skipThreadCount;
continue;
}
String info = "Thread name: " + tinfo.getThreadName() + " | " + tinfo.getThreadId();
threadsInfo.append(info);
for (StackTraceElement e : tinfo.getStackTrace()) {
threadsInfo.append(e + "\n\n");
}
}
System.out.println(threadsInfo + "\n\n-----------------------\n\n");
Assert.assertEquals(
threadsInfo.toString(), daemonThreadCountA, daemonThreadCountC - skipThreadCount);
Assert.assertEquals(nonDaemonThreadCountA, nonDaemonThreadCountC);
}
@Test
public void verifyThreadNames() throws Exception {
ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
Set<String> preNodeStartThreadNames = Sets.newHashSet();
for (long l : threadBean.getAllThreadIds()) {
ThreadInfo threadInfo = threadBean.getThreadInfo(l);
if (threadInfo != null) {
preNodeStartThreadNames.add(threadInfo.getThreadName());
}
}
logger.info("pre node threads are {}", preNodeStartThreadNames);
String node = internalCluster().startNode();
logger.info("do some indexing, flushing, optimize, and searches");
int numDocs = randomIntBetween(2, 100);
IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
for (int i = 0; i < numDocs; ++i) {
builders[i] =
client()
.prepareIndex("idx", "type")
.setSource(
jsonBuilder()
.startObject()
.field("str_value", "s" + i)
.field("str_values", new String[] {"s" + (i * 2), "s" + (i * 2 + 1)})
.field("l_value", i)
.field("l_values", new int[] {i * 2, i * 2 + 1})
.field("d_value", i)
.field("d_values", new double[] {i * 2, i * 2 + 1})
.endObject());
}
indexRandom(true, builders);
int numSearches = randomIntBetween(2, 100);
for (int i = 0; i < numSearches; i++) {
assertNoFailures(
client()
.prepareSearch("idx")
.setQuery(QueryBuilders.termQuery("str_value", "s" + i))
.get());
assertNoFailures(
client().prepareSearch("idx").setQuery(QueryBuilders.termQuery("l_value", i)).get());
}
Set<String> threadNames = Sets.newHashSet();
for (long l : threadBean.getAllThreadIds()) {
ThreadInfo threadInfo = threadBean.getThreadInfo(l);
if (threadInfo != null) {
threadNames.add(threadInfo.getThreadName());
}
}
logger.info("post node threads are {}", threadNames);
threadNames.removeAll(preNodeStartThreadNames);
logger.info("post node *new* threads are {}", threadNames);
for (String threadName : threadNames) {
// ignore some shared threads we know that are created within the same VM, like the shared
// discovery one
// or the ones that are occasionally come up from ElasticsearchSingleNodeTest
if (threadName.contains("[" + MulticastChannel.SHARED_CHANNEL_NAME + "]")
|| threadName.contains("[" + ElasticsearchSingleNodeTest.nodeName() + "]")
|| threadName.contains("Keep-Alive-Timer")) {
continue;
}
String nodePrefix =
"("
+ Pattern.quote(InternalTestCluster.TRANSPORT_CLIENT_PREFIX)
+ ")?("
+ Pattern.quote(ElasticsearchIntegrationTest.SUITE_CLUSTER_NODE_PREFIX)
+ "|"
+ Pattern.quote(ElasticsearchIntegrationTest.TEST_CLUSTER_NODE_PREFIX)
+ "|"
+ Pattern.quote(TribeTests.SECOND_CLUSTER_NODE_PREFIX)
+ ")";
assertThat(threadName, RegexMatcher.matches("\\[" + nodePrefix + "\\d+\\]"));
}
}