private static void showThreads(PrintStream pw, ThreadGroup g, Thread current) {
    int nthreads = g.activeCount();
    pw.println("\nThread Group = " + g.getName() + " activeCount= " + nthreads);
    Thread[] tarray = new Thread[nthreads];
    int n = g.enumerate(tarray, false);

    for (int i = 0; i < n; i++) {
      Thread thread = tarray[i];
      ClassLoader loader = thread.getContextClassLoader();
      String loaderName = (loader == null) ? "Default" : loader.getClass().getName();
      Thread.State state = thread.getState();
      long id = thread.getId();
      pw.print("   " + id + " " + thread.getName() + " " + state + " " + loaderName);
      if (thread == current) pw.println(" **** CURRENT ***");
      else pw.println();
    }

    int ngroups = g.activeGroupCount();
    ThreadGroup[] garray = new ThreadGroup[ngroups];
    int ng = g.enumerate(garray, false);
    for (int i = 0; i < ng; i++) {
      ThreadGroup nested = garray[i];
      showThreads(pw, nested, current);
    }
  }
  /// The watcher thread - from the Runnable interface.
  // This has to be pretty anal to avoid monitor lockup, lost
  // threads, etc.
  public synchronized void run() {
    Thread me = Thread.currentThread();
    me.setPriority(Thread.MAX_PRIORITY);

    ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
    startTimeInNs = threadMXBean.getCurrentThreadCpuTime();

    if (enabled.get()) {
      do {
        loop.set(false);
        try {
          wait(millis);
        } catch (InterruptedException e) {
        }
      } while (enabled.get() && loop.get());
    }

    if (enabled.get() && targetThread.isAlive()) {
      isDoneRunning.set(true);

      printThread();
      if (kill) {
        logger.warn(
            "Trying to kill thread with id:"
                + targetThread.getId()
                + " but did not as it can cause deadlocks etc.");
        // targetThread.interrupt();
        // targetThread.stop(); //Never kill thread - it can cause other problems
      }
      done();
      isDoneRunning.set(false);
    }
  }
Beispiel #3
0
  public Object down(Event evt) {
    switch (evt.getType()) {
      case ExecutorEvent.TASK_SUBMIT:
        Runnable runnable = evt.getArg();
        // We are limited to a number of concurrent request id's
        // equal to 2^63-1.  This is quite large and if it
        // overflows it will still be positive
        long requestId = Math.abs(counter.getAndIncrement());
        if (requestId == Long.MIN_VALUE) {
          // TODO: need to fix this it isn't safe for concurrent modifications
          counter.set(0);
          requestId = Math.abs(counter.getAndIncrement());
        }

        // Need to make sure to put the requestId in our map before
        // adding the runnable to awaiting consumer in case if
        // coordinator sent a consumer found and their original task
        // is no longer around
        // see https://issues.jboss.org/browse/JGRP-1744
        _requestId.put(runnable, requestId);

        _awaitingConsumer.add(runnable);

        sendToCoordinator(RUN_REQUEST, requestId, local_addr);
        break;
      case ExecutorEvent.CONSUMER_READY:
        Thread currentThread = Thread.currentThread();
        long threadId = currentThread.getId();
        _consumerId.put(threadId, PRESENT);
        try {
          for (; ; ) {
            CyclicBarrier barrier = new CyclicBarrier(2);
            _taskBarriers.put(threadId, barrier);

            // We only send to the coordinator that we are ready after
            // making the barrier, wait for request to come and let
            // us free
            sendToCoordinator(Type.CONSUMER_READY, threadId, local_addr);

            try {
              barrier.await();
              break;
            } catch (BrokenBarrierException e) {
              if (log.isDebugEnabled())
                log.debug(
                    "Producer timed out before we picked up"
                        + " the task, have to tell coordinator"
                        + " we are still good.");
            }
          }
          // This should always be non nullable since the latch
          // was freed
          runnable = _tasks.remove(threadId);
          _runnableThreads.put(runnable, currentThread);
          return runnable;
        } catch (InterruptedException e) {
          if (log.isDebugEnabled()) log.debug("Consumer " + threadId + " stopped via interrupt");
          sendToCoordinator(Type.CONSUMER_UNREADY, threadId, local_addr);
          Thread.currentThread().interrupt();
        } finally {
          // Make sure the barriers are cleaned up as well
          _taskBarriers.remove(threadId);
          _consumerId.remove(threadId);
        }
        break;
      case ExecutorEvent.TASK_COMPLETE:
        Object arg = evt.getArg();
        Throwable throwable = null;
        if (arg instanceof Object[]) {
          Object[] array = (Object[]) arg;
          runnable = (Runnable) array[0];
          throwable = (Throwable) array[1];
        } else {
          runnable = (Runnable) arg;
        }
        Owner owner = _running.remove(runnable);
        // This won't remove anything if owner doesn't come back
        _runnableThreads.remove(runnable);

        Object value = null;
        boolean exception = false;
        if (throwable != null) {
          // InterruptedException is special telling us that
          // we interrupted the thread while waiting but still got
          // a task therefore we have to reject it.
          if (throwable instanceof InterruptedException) {
            if (log.isDebugEnabled())
              log.debug("Run rejected due to interrupted exception returned");
            sendRequest(owner.address, Type.RUN_REJECTED, owner.requestId, null);
            break;
          }
          value = throwable;
          exception = true;
        } else if (runnable instanceof RunnableFuture<?>) {
          RunnableFuture<?> future = (RunnableFuture<?>) runnable;

          boolean interrupted = false;
          boolean gotValue = false;

          // We have the value, before we interrupt at least get it!
          while (!gotValue) {
            try {
              value = future.get();
              gotValue = true;
            } catch (InterruptedException e) {
              interrupted = true;
            } catch (ExecutionException e) {
              value = e.getCause();
              exception = true;
              gotValue = true;
            }
          }

          if (interrupted) {
            Thread.currentThread().interrupt();
          }
        }

        if (owner != null) {
          final Type type;
          final Object valueToSend;
          if (value == null) {
            type = Type.RESULT_SUCCESS;
            valueToSend = value;
          }
          // Both serializable values and exceptions would go in here
          else if (value instanceof Serializable
              || value instanceof Externalizable
              || value instanceof Streamable) {
            type = exception ? Type.RESULT_EXCEPTION : Type.RESULT_SUCCESS;
            valueToSend = value;
          }
          // This would happen if the value wasn't serializable,
          // so we have to send back to the client that the class
          // wasn't serializable
          else {
            type = Type.RESULT_EXCEPTION;
            valueToSend = new NotSerializableException(value.getClass().getName());
          }

          if (local_addr.equals(owner.getAddress())) {
            if (log.isTraceEnabled())
              log.trace(
                  "[redirect] <--> ["
                      + local_addr
                      + "] "
                      + type.name()
                      + " ["
                      + value
                      + (owner.requestId != -1 ? " request id: " + owner.requestId : "")
                      + "]");
            if (type == Type.RESULT_SUCCESS) {
              handleValueResponse(local_addr, owner.requestId, valueToSend);
            } else if (type == Type.RESULT_EXCEPTION) {
              handleExceptionResponse(local_addr, owner.requestId, (Throwable) valueToSend);
            }
          } else {
            sendRequest(owner.getAddress(), type, owner.requestId, valueToSend);
          }
        } else {
          if (log.isTraceEnabled()) {
            log.trace("Could not return result - most likely because it was interrupted");
          }
        }
        break;
      case ExecutorEvent.TASK_CANCEL:
        Object[] array = evt.getArg();
        runnable = (Runnable) array[0];

        if (_awaitingConsumer.remove(runnable)) {
          _requestId.remove(runnable);
          ExecutorNotification notification = notifiers.remove(runnable);
          if (notification != null) {
            notification.interrupted(runnable);
          }
          if (log.isTraceEnabled())
            log.trace("Cancelled task " + runnable + " before it was picked up");
          return Boolean.TRUE;
        }
        // This is guaranteed to not be null so don't take cost of auto unboxing
        else if (array[1] == Boolean.TRUE) {
          owner = removeKeyForValue(_awaitingReturn, runnable);
          if (owner != null) {
            Long requestIdValue = _requestId.remove(runnable);
            // We only cancel if the requestId is still available
            // this means the result hasn't been returned yet and
            // we still have a chance to interrupt
            if (requestIdValue != null) {
              if (requestIdValue != owner.getRequestId()) {
                log.warn("Cancelling requestId didn't match waiting");
              }
              sendRequest(owner.getAddress(), Type.INTERRUPT_RUN, owner.getRequestId(), null);
            }
          } else {
            if (log.isTraceEnabled()) log.warn("Couldn't interrupt server task: " + runnable);
          }
          ExecutorNotification notification = notifiers.remove(runnable);
          if (notification != null) {
            notification.interrupted(runnable);
          }
          return Boolean.TRUE;
        } else {
          return Boolean.FALSE;
        }
      case ExecutorEvent.ALL_TASK_CANCEL:
        array = evt.getArg();

        // This is a RunnableFuture<?> so this cast is okay
        @SuppressWarnings("unchecked")
        Set<Runnable> runnables = (Set<Runnable>) array[0];
        Boolean booleanValue = (Boolean) array[1];

        List<Runnable> notRan = new ArrayList<>();

        for (Runnable cancelRunnable : runnables) {
          // Removed from the consumer
          if (!_awaitingConsumer.remove(cancelRunnable) && booleanValue == Boolean.TRUE) {
            synchronized (_awaitingReturn) {
              owner = removeKeyForValue(_awaitingReturn, cancelRunnable);
              if (owner != null) {
                Long requestIdValue = _requestId.remove(cancelRunnable);
                if (requestIdValue != owner.getRequestId()) {
                  log.warn("Cancelling requestId didn't match waiting");
                }
                sendRequest(owner.getAddress(), Type.INTERRUPT_RUN, owner.getRequestId(), null);
              }
              ExecutorNotification notification = notifiers.remove(cancelRunnable);
              if (notification != null) {
                log.trace("Notifying listener");
                notification.interrupted(cancelRunnable);
              }
            }
          } else {
            _requestId.remove(cancelRunnable);
            notRan.add(cancelRunnable);
          }
        }
        return notRan;
      case Event.SET_LOCAL_ADDRESS:
        local_addr = evt.getArg();
        break;

      case Event.VIEW_CHANGE:
        handleView(evt.getArg());
        break;
    }
    return down_prot.down(evt);
  }
 public boolean runningOnClientThread() {
   return clientThread != null && Thread.currentThread().getId() == clientThread.getId();
 }
  private void printThread() {
    long now = System.currentTimeMillis();

    long diffLastThreadDump = now - lastThreadDump;
    logger.info("diffLastThreadDump:" + diffLastThreadDump);
    if (diffLastThreadDump > 60000) {
      logger.info("had not sent all threads for a while.. will do so now");
      lastThreadDump = now;

      try {
        ThreadMXBean t = ManagementFactory.getThreadMXBean();
        long threads[] = t.getAllThreadIds();
        ThreadInfo[] tinfo = t.getThreadInfo(threads, 40);

        StringBuilder sb = new StringBuilder("All Threads");

        for (int i = 0; i < tinfo.length; i++) {
          ThreadInfo e = tinfo[i];

          try {
            StackTraceElement[] el = e.getStackTrace();
            sb.append(
                "\n\n"
                    + e.getThreadName()
                    + "\n"
                    + " "
                    + " Thread id = "
                    + e.getThreadId()
                    + " "
                    + e.getThreadState());
            if (e.getThreadState().equals(State.BLOCKED)) {
              sb.append(
                  "\n\nBlocked info: "
                      + e.getBlockedCount()
                      + ":"
                      + e.getBlockedTime()
                      + ":"
                      + e.getLockName()
                      + ":"
                      + e.getLockOwnerId()
                      + ":"
                      + e.getLockOwnerName()
                      + "\n"
                      + " "
                      + " Thread id = "
                      + e.getThreadId()
                      + " "
                      + e.getThreadState());

              ThreadInfo eBlockedThread = t.getThreadInfo(e.getLockOwnerId(), 40);
              StackTraceElement[] elBlockedThread = eBlockedThread.getStackTrace();
              sb.append(
                  "\n\n    "
                      + e.getThreadName()
                      + "\n"
                      + " "
                      + " Thread id = "
                      + eBlockedThread.getThreadId()
                      + " "
                      + eBlockedThread.getThreadState());

              if (elBlockedThread == null || elBlockedThread.length == 0) {
                sb.append("        no stack trace available");
              } else {
                for (int n = 0; n < elBlockedThread.length; n++) {
                  if (n != 0) sb.append("\n");

                  StackTraceElement frame = elBlockedThread[n];

                  if (frame == null) {
                    sb.append("        null stack frame");
                    continue;
                  }

                  sb.append("        ");
                  sb.append(frame.toString());
                }
              }
            }

            if (el == null || el.length == 0) {
              sb.append("    no stack trace available");
              continue;
            }

            for (int n = 0; n < el.length; n++) {
              if (n != 0) sb.append("\n");

              StackTraceElement frame = el[n];

              if (frame == null) {
                sb.append("    null stack frame");
                continue;
              }

              sb.append("    ");
              sb.append(frame.toString());
            }
          } catch (Exception e2) {
          }
        }

        String warningEmailReceiver = CmsPropertyHandler.getWarningEmailReceiver();
        if (warningEmailReceiver != null
            && !warningEmailReceiver.equals("")
            && warningEmailReceiver.indexOf("@warningEmailReceiver@") == -1) {
          try {
            logger.info("Mailing..");
            MailServiceFactory.getService()
                .sendEmail(
                    CmsPropertyHandler.getMailContentType(),
                    warningEmailReceiver,
                    warningEmailReceiver,
                    null,
                    null,
                    null,
                    null,
                    message,
                    sb.toString().replaceAll("\n", "<br/>"),
                    "utf-8");
          } catch (Exception e) {
            logger.error("Could not send mail:" + e.getMessage(), e);
          }
        }
      } catch (Throwable e) {
        logger.error("Error generating message:" + e.getMessage(), e);
      }
    }

    // Only sends if the last stack was sent more than 3 seconds ago.
    if ((now - lastSentTimer) > 10000) {
      lastSentTimer = System.currentTimeMillis();

      StackTraceElement[] el = targetThread.getStackTrace();

      ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
      long stopTimeInNs = threadMXBean.getThreadUserTime(targetThread.getId());
      long diff = (stopTimeInNs - startTimeInNs) / 1000000000;

      StringBuffer stackString = new StringBuffer("\n\n" + message + "\n\n");
      stackString.append("ServerName: " + getServerName() + "\n");
      stackString.append("Maximum memory (MB): " + (maxMemory / 1024 / 1024) + "\n");
      stackString.append("Used memory (MB): " + ((totalMemory - freeMemory) / 1024 / 1024) + "\n");
      stackString.append("Free memory (MB): " + (freeMemory / 1024 / 1024) + "\n");
      stackString.append("Total memory (MB): " + (totalMemory / 1024 / 1024) + "\n");
      stackString.append("Number of current requests: " + numberOfCurrentRequests + "\n");
      stackString.append("Number of active requests: " + numberOfActiveRequests + "\n");
      stackString.append("Number of long requests: " + longThreadMonitorsSize + "\n");
      stackString.append("Current thread time: " + diff + " seconds\n");
      stackString.append(
          "Average time: " + RequestAnalyser.getRequestAnalyser().getAverageElapsedTime() + "\n");
      stackString.append(
          "Longest time: " + RequestAnalyser.getRequestAnalyser().getMaxElapsedTime() + "\n");
      stackString.append("Original url: " + getOriginalFullURL() + "\n");
      stackString.append("UserInfo: " + getUserInfo() + "\n");
      stackString.append("--------------------------------------------\n\n");
      stackString.append("Thread with id [" + targetThread.getId() + "] at report time:\n");

      if (el != null && el.length != 0) {
        for (int j = 0; j < el.length; j++) {
          StackTraceElement frame = el[j];
          if (frame == null) stackString.append("    null stack frame" + "\n");
          else stackString.append("    ").append(frame.toString()).append("\n");

          // if((stackString.indexOf("infoglue") > -1 && j > 20) || j > 35)
          //	break;
        }
      }

      if (targetThread.getState().equals(State.BLOCKED)) {
        ThreadMXBean t = ManagementFactory.getThreadMXBean();

        ThreadInfo e = t.getThreadInfo(targetThread.getId(), 40);
        stackString.append(
            "\n\nBlocked info: "
                + e.getBlockedCount()
                + ":"
                + e.getBlockedTime()
                + ":"
                + e.getLockName()
                + ":"
                + e.getLockOwnerId()
                + ":"
                + e.getLockOwnerName()
                + "\n"
                + " "
                + " Thread id = "
                + e.getThreadId()
                + " "
                + e.getThreadState());

        ThreadInfo eBlockedThread = t.getThreadInfo(e.getLockOwnerId(), 40);
        StackTraceElement[] elBlockedThread = eBlockedThread.getStackTrace();
        stackString.append(
            "\n\nBlocked thread: "
                + e.getThreadName()
                + "\n"
                + " "
                + " Thread id = "
                + eBlockedThread.getThreadId()
                + " "
                + eBlockedThread.getThreadState());

        if (elBlockedThread == null || elBlockedThread.length == 0) {
          stackString.append("        no stack trace available");
        } else {
          for (int n = 0; n < elBlockedThread.length; n++) {
            if (n != 0) stackString.append("\n");

            StackTraceElement frame = elBlockedThread[n];

            if (frame == null) {
              stackString.append("        null stack frame");
              continue;
            }

            stackString.append("        ");
            stackString.append(frame.toString());
          }
        }
      }

      stackString.append(
          "\n\n**********************************\nConcurrent long threads (Only an excerpt of all)\n**********************************");

      ThreadMXBean t = ManagementFactory.getThreadMXBean();

      List threadMonitors = RequestAnalyser.getLongThreadMonitors();
      Iterator threadMonitorsIterator = threadMonitors.iterator();
      int threadCount = 0;
      while (threadMonitorsIterator.hasNext() && threadCount < 5) {
        SimpleThreadMonitor tm = (SimpleThreadMonitor) threadMonitorsIterator.next();

        if (targetThread.getId() == tm.getThreadId()) continue;

        long threads[] = {tm.getThreadId()};
        ThreadInfo[] tinfo = t.getThreadInfo(threads, 40);

        stackString
            .append("\n\n---------------------------------\nConcurrent long thread [")
            .append(tm.getThreadId())
            .append("]:\n");
        stackString
            .append("Elapsed time:")
            .append(tm.getElapsedTime())
            .append("\n Thread id: ")
            .append(tm.getThreadId())
            .append("\n Original url: ")
            .append(tm.getOriginalFullURL())
            .append(")");

        for (int i = 0; i < tinfo.length; i++) {
          ThreadInfo e = tinfo[i];

          el = e.getStackTrace();

          if (el != null && el.length != 0) {
            for (int n = 0; n < el.length; n++) {
              StackTraceElement frame = el[n];
              if (frame == null) stackString.append("    null stack frame\n");
              else stackString.append("    null stack frame").append(frame.toString()).append("\n");
            }
          }
        }

        threadCount++;
      }

      logger.warn(stackString);
    } else {
      logger.warn(
          "A thread took to long but the system seems to be really clogged so we don't send this one.");
    }
  }