private static void showThreads(PrintStream pw, ThreadGroup g, Thread current) { int nthreads = g.activeCount(); pw.println("\nThread Group = " + g.getName() + " activeCount= " + nthreads); Thread[] tarray = new Thread[nthreads]; int n = g.enumerate(tarray, false); for (int i = 0; i < n; i++) { Thread thread = tarray[i]; ClassLoader loader = thread.getContextClassLoader(); String loaderName = (loader == null) ? "Default" : loader.getClass().getName(); Thread.State state = thread.getState(); long id = thread.getId(); pw.print(" " + id + " " + thread.getName() + " " + state + " " + loaderName); if (thread == current) pw.println(" **** CURRENT ***"); else pw.println(); } int ngroups = g.activeGroupCount(); ThreadGroup[] garray = new ThreadGroup[ngroups]; int ng = g.enumerate(garray, false); for (int i = 0; i < ng; i++) { ThreadGroup nested = garray[i]; showThreads(pw, nested, current); } }
/// The watcher thread - from the Runnable interface. // This has to be pretty anal to avoid monitor lockup, lost // threads, etc. public synchronized void run() { Thread me = Thread.currentThread(); me.setPriority(Thread.MAX_PRIORITY); ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); startTimeInNs = threadMXBean.getCurrentThreadCpuTime(); if (enabled.get()) { do { loop.set(false); try { wait(millis); } catch (InterruptedException e) { } } while (enabled.get() && loop.get()); } if (enabled.get() && targetThread.isAlive()) { isDoneRunning.set(true); printThread(); if (kill) { logger.warn( "Trying to kill thread with id:" + targetThread.getId() + " but did not as it can cause deadlocks etc."); // targetThread.interrupt(); // targetThread.stop(); //Never kill thread - it can cause other problems } done(); isDoneRunning.set(false); } }
@Override public void run() { byte[] buf = new byte[100]; try { int len; while ((len = in.read(buf)) > 0) { String output = new String(buf, 0, len); Thread t = Thread.currentThread(); System.out.println( "thread " + t.getName() + " " + t.getId() + ", read " + len + " bytes: " + output); } } catch (IOException e) { logger.log(Level.SEVERE, "Failed to read", e); } finally { try { in.close(); } catch (IOException e) { logger.log(Level.SEVERE, "Failed to close", e); } } }
public Object down(Event evt) { switch (evt.getType()) { case ExecutorEvent.TASK_SUBMIT: Runnable runnable = evt.getArg(); // We are limited to a number of concurrent request id's // equal to 2^63-1. This is quite large and if it // overflows it will still be positive long requestId = Math.abs(counter.getAndIncrement()); if (requestId == Long.MIN_VALUE) { // TODO: need to fix this it isn't safe for concurrent modifications counter.set(0); requestId = Math.abs(counter.getAndIncrement()); } // Need to make sure to put the requestId in our map before // adding the runnable to awaiting consumer in case if // coordinator sent a consumer found and their original task // is no longer around // see https://issues.jboss.org/browse/JGRP-1744 _requestId.put(runnable, requestId); _awaitingConsumer.add(runnable); sendToCoordinator(RUN_REQUEST, requestId, local_addr); break; case ExecutorEvent.CONSUMER_READY: Thread currentThread = Thread.currentThread(); long threadId = currentThread.getId(); _consumerId.put(threadId, PRESENT); try { for (; ; ) { CyclicBarrier barrier = new CyclicBarrier(2); _taskBarriers.put(threadId, barrier); // We only send to the coordinator that we are ready after // making the barrier, wait for request to come and let // us free sendToCoordinator(Type.CONSUMER_READY, threadId, local_addr); try { barrier.await(); break; } catch (BrokenBarrierException e) { if (log.isDebugEnabled()) log.debug( "Producer timed out before we picked up" + " the task, have to tell coordinator" + " we are still good."); } } // This should always be non nullable since the latch // was freed runnable = _tasks.remove(threadId); _runnableThreads.put(runnable, currentThread); return runnable; } catch (InterruptedException e) { if (log.isDebugEnabled()) log.debug("Consumer " + threadId + " stopped via interrupt"); sendToCoordinator(Type.CONSUMER_UNREADY, threadId, local_addr); Thread.currentThread().interrupt(); } finally { // Make sure the barriers are cleaned up as well _taskBarriers.remove(threadId); _consumerId.remove(threadId); } break; case ExecutorEvent.TASK_COMPLETE: Object arg = evt.getArg(); Throwable throwable = null; if (arg instanceof Object[]) { Object[] array = (Object[]) arg; runnable = (Runnable) array[0]; throwable = (Throwable) array[1]; } else { runnable = (Runnable) arg; } Owner owner = _running.remove(runnable); // This won't remove anything if owner doesn't come back _runnableThreads.remove(runnable); Object value = null; boolean exception = false; if (throwable != null) { // InterruptedException is special telling us that // we interrupted the thread while waiting but still got // a task therefore we have to reject it. if (throwable instanceof InterruptedException) { if (log.isDebugEnabled()) log.debug("Run rejected due to interrupted exception returned"); sendRequest(owner.address, Type.RUN_REJECTED, owner.requestId, null); break; } value = throwable; exception = true; } else if (runnable instanceof RunnableFuture<?>) { RunnableFuture<?> future = (RunnableFuture<?>) runnable; boolean interrupted = false; boolean gotValue = false; // We have the value, before we interrupt at least get it! while (!gotValue) { try { value = future.get(); gotValue = true; } catch (InterruptedException e) { interrupted = true; } catch (ExecutionException e) { value = e.getCause(); exception = true; gotValue = true; } } if (interrupted) { Thread.currentThread().interrupt(); } } if (owner != null) { final Type type; final Object valueToSend; if (value == null) { type = Type.RESULT_SUCCESS; valueToSend = value; } // Both serializable values and exceptions would go in here else if (value instanceof Serializable || value instanceof Externalizable || value instanceof Streamable) { type = exception ? Type.RESULT_EXCEPTION : Type.RESULT_SUCCESS; valueToSend = value; } // This would happen if the value wasn't serializable, // so we have to send back to the client that the class // wasn't serializable else { type = Type.RESULT_EXCEPTION; valueToSend = new NotSerializableException(value.getClass().getName()); } if (local_addr.equals(owner.getAddress())) { if (log.isTraceEnabled()) log.trace( "[redirect] <--> [" + local_addr + "] " + type.name() + " [" + value + (owner.requestId != -1 ? " request id: " + owner.requestId : "") + "]"); if (type == Type.RESULT_SUCCESS) { handleValueResponse(local_addr, owner.requestId, valueToSend); } else if (type == Type.RESULT_EXCEPTION) { handleExceptionResponse(local_addr, owner.requestId, (Throwable) valueToSend); } } else { sendRequest(owner.getAddress(), type, owner.requestId, valueToSend); } } else { if (log.isTraceEnabled()) { log.trace("Could not return result - most likely because it was interrupted"); } } break; case ExecutorEvent.TASK_CANCEL: Object[] array = evt.getArg(); runnable = (Runnable) array[0]; if (_awaitingConsumer.remove(runnable)) { _requestId.remove(runnable); ExecutorNotification notification = notifiers.remove(runnable); if (notification != null) { notification.interrupted(runnable); } if (log.isTraceEnabled()) log.trace("Cancelled task " + runnable + " before it was picked up"); return Boolean.TRUE; } // This is guaranteed to not be null so don't take cost of auto unboxing else if (array[1] == Boolean.TRUE) { owner = removeKeyForValue(_awaitingReturn, runnable); if (owner != null) { Long requestIdValue = _requestId.remove(runnable); // We only cancel if the requestId is still available // this means the result hasn't been returned yet and // we still have a chance to interrupt if (requestIdValue != null) { if (requestIdValue != owner.getRequestId()) { log.warn("Cancelling requestId didn't match waiting"); } sendRequest(owner.getAddress(), Type.INTERRUPT_RUN, owner.getRequestId(), null); } } else { if (log.isTraceEnabled()) log.warn("Couldn't interrupt server task: " + runnable); } ExecutorNotification notification = notifiers.remove(runnable); if (notification != null) { notification.interrupted(runnable); } return Boolean.TRUE; } else { return Boolean.FALSE; } case ExecutorEvent.ALL_TASK_CANCEL: array = evt.getArg(); // This is a RunnableFuture<?> so this cast is okay @SuppressWarnings("unchecked") Set<Runnable> runnables = (Set<Runnable>) array[0]; Boolean booleanValue = (Boolean) array[1]; List<Runnable> notRan = new ArrayList<>(); for (Runnable cancelRunnable : runnables) { // Removed from the consumer if (!_awaitingConsumer.remove(cancelRunnable) && booleanValue == Boolean.TRUE) { synchronized (_awaitingReturn) { owner = removeKeyForValue(_awaitingReturn, cancelRunnable); if (owner != null) { Long requestIdValue = _requestId.remove(cancelRunnable); if (requestIdValue != owner.getRequestId()) { log.warn("Cancelling requestId didn't match waiting"); } sendRequest(owner.getAddress(), Type.INTERRUPT_RUN, owner.getRequestId(), null); } ExecutorNotification notification = notifiers.remove(cancelRunnable); if (notification != null) { log.trace("Notifying listener"); notification.interrupted(cancelRunnable); } } } else { _requestId.remove(cancelRunnable); notRan.add(cancelRunnable); } } return notRan; case Event.SET_LOCAL_ADDRESS: local_addr = evt.getArg(); break; case Event.VIEW_CHANGE: handleView(evt.getArg()); break; } return down_prot.down(evt); }
private void printThread() { long now = System.currentTimeMillis(); long diffLastThreadDump = now - lastThreadDump; logger.info("diffLastThreadDump:" + diffLastThreadDump); if (diffLastThreadDump > 60000) { logger.info("had not sent all threads for a while.. will do so now"); lastThreadDump = now; try { ThreadMXBean t = ManagementFactory.getThreadMXBean(); long threads[] = t.getAllThreadIds(); ThreadInfo[] tinfo = t.getThreadInfo(threads, 40); StringBuilder sb = new StringBuilder("All Threads"); for (int i = 0; i < tinfo.length; i++) { ThreadInfo e = tinfo[i]; try { StackTraceElement[] el = e.getStackTrace(); sb.append( "\n\n" + e.getThreadName() + "\n" + " " + " Thread id = " + e.getThreadId() + " " + e.getThreadState()); if (e.getThreadState().equals(State.BLOCKED)) { sb.append( "\n\nBlocked info: " + e.getBlockedCount() + ":" + e.getBlockedTime() + ":" + e.getLockName() + ":" + e.getLockOwnerId() + ":" + e.getLockOwnerName() + "\n" + " " + " Thread id = " + e.getThreadId() + " " + e.getThreadState()); ThreadInfo eBlockedThread = t.getThreadInfo(e.getLockOwnerId(), 40); StackTraceElement[] elBlockedThread = eBlockedThread.getStackTrace(); sb.append( "\n\n " + e.getThreadName() + "\n" + " " + " Thread id = " + eBlockedThread.getThreadId() + " " + eBlockedThread.getThreadState()); if (elBlockedThread == null || elBlockedThread.length == 0) { sb.append(" no stack trace available"); } else { for (int n = 0; n < elBlockedThread.length; n++) { if (n != 0) sb.append("\n"); StackTraceElement frame = elBlockedThread[n]; if (frame == null) { sb.append(" null stack frame"); continue; } sb.append(" "); sb.append(frame.toString()); } } } if (el == null || el.length == 0) { sb.append(" no stack trace available"); continue; } for (int n = 0; n < el.length; n++) { if (n != 0) sb.append("\n"); StackTraceElement frame = el[n]; if (frame == null) { sb.append(" null stack frame"); continue; } sb.append(" "); sb.append(frame.toString()); } } catch (Exception e2) { } } String warningEmailReceiver = CmsPropertyHandler.getWarningEmailReceiver(); if (warningEmailReceiver != null && !warningEmailReceiver.equals("") && warningEmailReceiver.indexOf("@warningEmailReceiver@") == -1) { try { logger.info("Mailing.."); MailServiceFactory.getService() .sendEmail( CmsPropertyHandler.getMailContentType(), warningEmailReceiver, warningEmailReceiver, null, null, null, null, message, sb.toString().replaceAll("\n", "<br/>"), "utf-8"); } catch (Exception e) { logger.error("Could not send mail:" + e.getMessage(), e); } } } catch (Throwable e) { logger.error("Error generating message:" + e.getMessage(), e); } } // Only sends if the last stack was sent more than 3 seconds ago. if ((now - lastSentTimer) > 10000) { lastSentTimer = System.currentTimeMillis(); StackTraceElement[] el = targetThread.getStackTrace(); ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); long stopTimeInNs = threadMXBean.getThreadUserTime(targetThread.getId()); long diff = (stopTimeInNs - startTimeInNs) / 1000000000; StringBuffer stackString = new StringBuffer("\n\n" + message + "\n\n"); stackString.append("ServerName: " + getServerName() + "\n"); stackString.append("Maximum memory (MB): " + (maxMemory / 1024 / 1024) + "\n"); stackString.append("Used memory (MB): " + ((totalMemory - freeMemory) / 1024 / 1024) + "\n"); stackString.append("Free memory (MB): " + (freeMemory / 1024 / 1024) + "\n"); stackString.append("Total memory (MB): " + (totalMemory / 1024 / 1024) + "\n"); stackString.append("Number of current requests: " + numberOfCurrentRequests + "\n"); stackString.append("Number of active requests: " + numberOfActiveRequests + "\n"); stackString.append("Number of long requests: " + longThreadMonitorsSize + "\n"); stackString.append("Current thread time: " + diff + " seconds\n"); stackString.append( "Average time: " + RequestAnalyser.getRequestAnalyser().getAverageElapsedTime() + "\n"); stackString.append( "Longest time: " + RequestAnalyser.getRequestAnalyser().getMaxElapsedTime() + "\n"); stackString.append("Original url: " + getOriginalFullURL() + "\n"); stackString.append("UserInfo: " + getUserInfo() + "\n"); stackString.append("--------------------------------------------\n\n"); stackString.append("Thread with id [" + targetThread.getId() + "] at report time:\n"); if (el != null && el.length != 0) { for (int j = 0; j < el.length; j++) { StackTraceElement frame = el[j]; if (frame == null) stackString.append(" null stack frame" + "\n"); else stackString.append(" ").append(frame.toString()).append("\n"); // if((stackString.indexOf("infoglue") > -1 && j > 20) || j > 35) // break; } } if (targetThread.getState().equals(State.BLOCKED)) { ThreadMXBean t = ManagementFactory.getThreadMXBean(); ThreadInfo e = t.getThreadInfo(targetThread.getId(), 40); stackString.append( "\n\nBlocked info: " + e.getBlockedCount() + ":" + e.getBlockedTime() + ":" + e.getLockName() + ":" + e.getLockOwnerId() + ":" + e.getLockOwnerName() + "\n" + " " + " Thread id = " + e.getThreadId() + " " + e.getThreadState()); ThreadInfo eBlockedThread = t.getThreadInfo(e.getLockOwnerId(), 40); StackTraceElement[] elBlockedThread = eBlockedThread.getStackTrace(); stackString.append( "\n\nBlocked thread: " + e.getThreadName() + "\n" + " " + " Thread id = " + eBlockedThread.getThreadId() + " " + eBlockedThread.getThreadState()); if (elBlockedThread == null || elBlockedThread.length == 0) { stackString.append(" no stack trace available"); } else { for (int n = 0; n < elBlockedThread.length; n++) { if (n != 0) stackString.append("\n"); StackTraceElement frame = elBlockedThread[n]; if (frame == null) { stackString.append(" null stack frame"); continue; } stackString.append(" "); stackString.append(frame.toString()); } } } stackString.append( "\n\n**********************************\nConcurrent long threads (Only an excerpt of all)\n**********************************"); ThreadMXBean t = ManagementFactory.getThreadMXBean(); List threadMonitors = RequestAnalyser.getLongThreadMonitors(); Iterator threadMonitorsIterator = threadMonitors.iterator(); int threadCount = 0; while (threadMonitorsIterator.hasNext() && threadCount < 5) { SimpleThreadMonitor tm = (SimpleThreadMonitor) threadMonitorsIterator.next(); if (targetThread.getId() == tm.getThreadId()) continue; long threads[] = {tm.getThreadId()}; ThreadInfo[] tinfo = t.getThreadInfo(threads, 40); stackString .append("\n\n---------------------------------\nConcurrent long thread [") .append(tm.getThreadId()) .append("]:\n"); stackString .append("Elapsed time:") .append(tm.getElapsedTime()) .append("\n Thread id: ") .append(tm.getThreadId()) .append("\n Original url: ") .append(tm.getOriginalFullURL()) .append(")"); for (int i = 0; i < tinfo.length; i++) { ThreadInfo e = tinfo[i]; el = e.getStackTrace(); if (el != null && el.length != 0) { for (int n = 0; n < el.length; n++) { StackTraceElement frame = el[n]; if (frame == null) stackString.append(" null stack frame\n"); else stackString.append(" null stack frame").append(frame.toString()).append("\n"); } } } threadCount++; } logger.warn(stackString); } else { logger.warn( "A thread took to long but the system seems to be really clogged so we don't send this one."); } }