private static void showThreads(PrintStream pw, ThreadGroup g, Thread current) {
int nthreads = g.activeCount();
pw.println("\nThread Group = " + g.getName() + " activeCount= " + nthreads);
Thread[] tarray = new Thread[nthreads];
int n = g.enumerate(tarray, false);
for (int i = 0; i < n; i++) {
Thread thread = tarray[i];
ClassLoader loader = thread.getContextClassLoader();
String loaderName = (loader == null) ? "Default" : loader.getClass().getName();
Thread.State state = thread.getState();
long id = thread.getId();
pw.print(" " + id + " " + thread.getName() + " " + state + " " + loaderName);
if (thread == current) pw.println(" **** CURRENT ***");
else pw.println();
}
int ngroups = g.activeGroupCount();
ThreadGroup[] garray = new ThreadGroup[ngroups];
int ng = g.enumerate(garray, false);
for (int i = 0; i < ng; i++) {
ThreadGroup nested = garray[i];
showThreads(pw, nested, current);
}
}
private static void runExecutionBoundedTest(int nthreads, List<Counter> counters, int nexecutions)
throws InterruptedException {
final List<CounterThread> threads = new ArrayList<CounterThread>();
for (int i = 0; i < nthreads; i++) {
CounterThread t;
t = new CounterThread(counters.get(i % counters.size()), nexecutions);
t.start();
threads.add(t);
}
CounterThread.shoot(); // let all the threads go crazy at the same time
for (int i = 0; i < nthreads; i++) {
CounterThread t = threads.get(i);
t.join(30000);
if (t.isAlive()) {
System.out.println("stuck thread name: " + t.toString());
System.out.println("stuck thread state: " + t.getState());
safe.ReentrantLock l = (safe.ReentrantLock) t.getLock();
System.out.println("thread waiting lock: " + l);
System.out.println("stuck thread increments: " + t.getExecutions());
System.out.println("stuck thread counter value: " + t.getCounterCount());
Thread other = l.getOwner();
System.out.println("lock owner: " + other);
if (other != null) {
System.out.println("owner name: " + other.toString());
System.out.println("state owner: " + other.getState());
}
// Keep program alive to dump thread stacks with: jstack -l $(pidof java)
System.out.println(
java.lang.management.ManagementFactory.getThreadMXBean().getPeakThreadCount());
while (true) {}
}
}
long sum = 0;
for (int i = 0; i < counters.size(); ++i) {
sum += counters.get(i).getCount();
}
System.out.println(sum);
System.out.println(
java.lang.management.ManagementFactory.getThreadMXBean().getPeakThreadCount());
}
private void printThread() {
long now = System.currentTimeMillis();
long diffLastThreadDump = now - lastThreadDump;
logger.info("diffLastThreadDump:" + diffLastThreadDump);
if (diffLastThreadDump > 60000) {
logger.info("had not sent all threads for a while.. will do so now");
lastThreadDump = now;
try {
ThreadMXBean t = ManagementFactory.getThreadMXBean();
long threads[] = t.getAllThreadIds();
ThreadInfo[] tinfo = t.getThreadInfo(threads, 40);
StringBuilder sb = new StringBuilder("All Threads");
for (int i = 0; i < tinfo.length; i++) {
ThreadInfo e = tinfo[i];
try {
StackTraceElement[] el = e.getStackTrace();
sb.append(
"\n\n"
+ e.getThreadName()
+ "\n"
+ " "
+ " Thread id = "
+ e.getThreadId()
+ " "
+ e.getThreadState());
if (e.getThreadState().equals(State.BLOCKED)) {
sb.append(
"\n\nBlocked info: "
+ e.getBlockedCount()
+ ":"
+ e.getBlockedTime()
+ ":"
+ e.getLockName()
+ ":"
+ e.getLockOwnerId()
+ ":"
+ e.getLockOwnerName()
+ "\n"
+ " "
+ " Thread id = "
+ e.getThreadId()
+ " "
+ e.getThreadState());
ThreadInfo eBlockedThread = t.getThreadInfo(e.getLockOwnerId(), 40);
StackTraceElement[] elBlockedThread = eBlockedThread.getStackTrace();
sb.append(
"\n\n "
+ e.getThreadName()
+ "\n"
+ " "
+ " Thread id = "
+ eBlockedThread.getThreadId()
+ " "
+ eBlockedThread.getThreadState());
if (elBlockedThread == null || elBlockedThread.length == 0) {
sb.append(" no stack trace available");
} else {
for (int n = 0; n < elBlockedThread.length; n++) {
if (n != 0) sb.append("\n");
StackTraceElement frame = elBlockedThread[n];
if (frame == null) {
sb.append(" null stack frame");
continue;
}
sb.append(" ");
sb.append(frame.toString());
}
}
}
if (el == null || el.length == 0) {
sb.append(" no stack trace available");
continue;
}
for (int n = 0; n < el.length; n++) {
if (n != 0) sb.append("\n");
StackTraceElement frame = el[n];
if (frame == null) {
sb.append(" null stack frame");
continue;
}
sb.append(" ");
sb.append(frame.toString());
}
} catch (Exception e2) {
}
}
String warningEmailReceiver = CmsPropertyHandler.getWarningEmailReceiver();
if (warningEmailReceiver != null
&& !warningEmailReceiver.equals("")
&& warningEmailReceiver.indexOf("@warningEmailReceiver@") == -1) {
try {
logger.info("Mailing..");
MailServiceFactory.getService()
.sendEmail(
CmsPropertyHandler.getMailContentType(),
warningEmailReceiver,
warningEmailReceiver,
null,
null,
null,
null,
message,
sb.toString().replaceAll("\n", "<br/>"),
"utf-8");
} catch (Exception e) {
logger.error("Could not send mail:" + e.getMessage(), e);
}
}
} catch (Throwable e) {
logger.error("Error generating message:" + e.getMessage(), e);
}
}
// Only sends if the last stack was sent more than 3 seconds ago.
if ((now - lastSentTimer) > 10000) {
lastSentTimer = System.currentTimeMillis();
StackTraceElement[] el = targetThread.getStackTrace();
ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
long stopTimeInNs = threadMXBean.getThreadUserTime(targetThread.getId());
long diff = (stopTimeInNs - startTimeInNs) / 1000000000;
StringBuffer stackString = new StringBuffer("\n\n" + message + "\n\n");
stackString.append("ServerName: " + getServerName() + "\n");
stackString.append("Maximum memory (MB): " + (maxMemory / 1024 / 1024) + "\n");
stackString.append("Used memory (MB): " + ((totalMemory - freeMemory) / 1024 / 1024) + "\n");
stackString.append("Free memory (MB): " + (freeMemory / 1024 / 1024) + "\n");
stackString.append("Total memory (MB): " + (totalMemory / 1024 / 1024) + "\n");
stackString.append("Number of current requests: " + numberOfCurrentRequests + "\n");
stackString.append("Number of active requests: " + numberOfActiveRequests + "\n");
stackString.append("Number of long requests: " + longThreadMonitorsSize + "\n");
stackString.append("Current thread time: " + diff + " seconds\n");
stackString.append(
"Average time: " + RequestAnalyser.getRequestAnalyser().getAverageElapsedTime() + "\n");
stackString.append(
"Longest time: " + RequestAnalyser.getRequestAnalyser().getMaxElapsedTime() + "\n");
stackString.append("Original url: " + getOriginalFullURL() + "\n");
stackString.append("UserInfo: " + getUserInfo() + "\n");
stackString.append("--------------------------------------------\n\n");
stackString.append("Thread with id [" + targetThread.getId() + "] at report time:\n");
if (el != null && el.length != 0) {
for (int j = 0; j < el.length; j++) {
StackTraceElement frame = el[j];
if (frame == null) stackString.append(" null stack frame" + "\n");
else stackString.append(" ").append(frame.toString()).append("\n");
// if((stackString.indexOf("infoglue") > -1 && j > 20) || j > 35)
// break;
}
}
if (targetThread.getState().equals(State.BLOCKED)) {
ThreadMXBean t = ManagementFactory.getThreadMXBean();
ThreadInfo e = t.getThreadInfo(targetThread.getId(), 40);
stackString.append(
"\n\nBlocked info: "
+ e.getBlockedCount()
+ ":"
+ e.getBlockedTime()
+ ":"
+ e.getLockName()
+ ":"
+ e.getLockOwnerId()
+ ":"
+ e.getLockOwnerName()
+ "\n"
+ " "
+ " Thread id = "
+ e.getThreadId()
+ " "
+ e.getThreadState());
ThreadInfo eBlockedThread = t.getThreadInfo(e.getLockOwnerId(), 40);
StackTraceElement[] elBlockedThread = eBlockedThread.getStackTrace();
stackString.append(
"\n\nBlocked thread: "
+ e.getThreadName()
+ "\n"
+ " "
+ " Thread id = "
+ eBlockedThread.getThreadId()
+ " "
+ eBlockedThread.getThreadState());
if (elBlockedThread == null || elBlockedThread.length == 0) {
stackString.append(" no stack trace available");
} else {
for (int n = 0; n < elBlockedThread.length; n++) {
if (n != 0) stackString.append("\n");
StackTraceElement frame = elBlockedThread[n];
if (frame == null) {
stackString.append(" null stack frame");
continue;
}
stackString.append(" ");
stackString.append(frame.toString());
}
}
}
stackString.append(
"\n\n**********************************\nConcurrent long threads (Only an excerpt of all)\n**********************************");
ThreadMXBean t = ManagementFactory.getThreadMXBean();
List threadMonitors = RequestAnalyser.getLongThreadMonitors();
Iterator threadMonitorsIterator = threadMonitors.iterator();
int threadCount = 0;
while (threadMonitorsIterator.hasNext() && threadCount < 5) {
SimpleThreadMonitor tm = (SimpleThreadMonitor) threadMonitorsIterator.next();
if (targetThread.getId() == tm.getThreadId()) continue;
long threads[] = {tm.getThreadId()};
ThreadInfo[] tinfo = t.getThreadInfo(threads, 40);
stackString
.append("\n\n---------------------------------\nConcurrent long thread [")
.append(tm.getThreadId())
.append("]:\n");
stackString
.append("Elapsed time:")
.append(tm.getElapsedTime())
.append("\n Thread id: ")
.append(tm.getThreadId())
.append("\n Original url: ")
.append(tm.getOriginalFullURL())
.append(")");
for (int i = 0; i < tinfo.length; i++) {
ThreadInfo e = tinfo[i];
el = e.getStackTrace();
if (el != null && el.length != 0) {
for (int n = 0; n < el.length; n++) {
StackTraceElement frame = el[n];
if (frame == null) stackString.append(" null stack frame\n");
else stackString.append(" null stack frame").append(frame.toString()).append("\n");
}
}
}
threadCount++;
}
logger.warn(stackString);
} else {
logger.warn(
"A thread took to long but the system seems to be really clogged so we don't send this one.");
}
}
public Thread.State getState() {
return th.getState();
}
/** Run and periodically report status. */
public void run() {
long st = System.currentTimeMillis();
long lasten = st;
long lasttotalops = 0;
boolean alldone;
do {
alldone = true;
int totalops = 0;
// terminate this thread when all the worker threads are done
for (Thread t : _threads) {
if (t.getState() != Thread.State.TERMINATED) {
alldone = false;
}
ClientThread ct = (ClientThread) t;
totalops += ct.getOpsDone();
}
long en = System.currentTimeMillis();
long interval = en - st;
// double throughput=1000.0*((double)totalops)/((double)interval);
double curthroughput =
1000.0 * (((double) (totalops - lasttotalops)) / ((double) (en - lasten)));
lasttotalops = totalops;
lasten = en;
DecimalFormat d = new DecimalFormat("#.##");
if (totalops == 0) {
System.err.println(
_label
+ " "
+ (interval / 1000)
+ " sec: "
+ totalops
+ " operations; "
+ Measurements.getMeasurements().getSummary());
} else {
System.err.println(
_label
+ " "
+ (interval / 1000)
+ " sec: "
+ totalops
+ " operations; "
+ d.format(curthroughput)
+ " current ops/sec; "
+ Measurements.getMeasurements().getSummary());
}
if (_standardstatus) {
if (totalops == 0) {
System.out.println(
_label
+ " "
+ (interval / 1000)
+ " sec: "
+ totalops
+ " operations; "
+ Measurements.getMeasurements().getSummary());
} else {
System.out.println(
_label
+ " "
+ (interval / 1000)
+ " sec: "
+ totalops
+ " operations; "
+ d.format(curthroughput)
+ " current ops/sec; "
+ Measurements.getMeasurements().getSummary());
}
}
try {
sleep(sleeptime);
} catch (InterruptedException e) {
// do nothing
}
} while (!alldone);
}