/**
* Factory method for "JvmCompilation" group metadata class.
*
* <p>You can redefine this method if you need to replace the default generated metadata class
* with your own customized class.
*
* @param groupName Name of the group ("JvmCompilation")
* @param groupOid OID of this group
* @param groupObjname ObjectName for this group (may be null)
* @param server MBeanServer for this group (may be null)
* @return An instance of the metadata class generated for the "JvmCompilation" group
* (JvmCompilationMeta)
*/
protected JvmCompilationMeta createJvmCompilationMetaNode(
String groupName, String groupOid, ObjectName groupObjname, MBeanServer server) {
// If there is no compilation system, the jvmCompilation will not
// be instantiated.
//
if (ManagementFactory.getCompilationMXBean() == null) return null;
return super.createJvmCompilationMetaNode(
groupName, groupOid,
groupObjname, server);
}
public synchronized void addMutation(String table, Mutation m) throws MutationsRejectedException {
if (closed) throw new IllegalStateException("Closed");
if (m.size() == 0) throw new IllegalArgumentException("Can not add empty mutations");
checkForFailures();
while ((totalMemUsed >= maxMem || flushing) && !somethingFailed) {
waitRTE();
}
// do checks again since things could have changed while waiting and not holding lock
if (closed) throw new IllegalStateException("Closed");
checkForFailures();
if (startTime == 0) {
startTime = System.currentTimeMillis();
List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
for (GarbageCollectorMXBean garbageCollectorMXBean : gcmBeans) {
initialGCTimes += garbageCollectorMXBean.getCollectionTime();
}
CompilationMXBean compMxBean = ManagementFactory.getCompilationMXBean();
if (compMxBean.isCompilationTimeMonitoringSupported()) {
initialCompileTimes = compMxBean.getTotalCompilationTime();
}
initialSystemLoad = ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();
}
// create a copy of mutation so that after this method returns the user
// is free to reuse the mutation object, like calling readFields... this
// is important for the case where a mutation is passed from map to reduce
// to batch writer... the map reduce code will keep passing the same mutation
// object into the reduce method
m = new Mutation(m);
totalMemUsed += m.estimatedMemoryUsed();
mutations.addMutation(table, m);
totalAdded++;
if (mutations.getMemoryUsed() >= maxMem / 2) {
startProcessing();
checkForFailures();
}
}
public class Diagnostics {
private static final String PACKAGE = "org.apache.tomcat.util";
private static final String INDENT1 = " ";
private static final String INDENT2 = "\t";
private static final String INDENT3 = " ";
private static final String CRLF = "\r\n";
private static final String vminfoSystemProperty = "java.vm.info";
private static final SimpleDateFormat timeformat =
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
/* Some platform MBeans */
private static final ClassLoadingMXBean classLoadingMXBean =
ManagementFactory.getClassLoadingMXBean();
private static final CompilationMXBean compilationMXBean =
ManagementFactory.getCompilationMXBean();
private static final OperatingSystemMXBean operatingSystemMXBean =
ManagementFactory.getOperatingSystemMXBean();
private static final RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
private static final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
// XXX Not sure whether the following MBeans should better
// be retrieved on demand, i.e. whether they can change
// dynamically in the MBeanServer.
private static final LoggingMXBean loggingMXBean = LogManager.getLoggingMXBean();
private static final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
private static final List<GarbageCollectorMXBean> garbageCollectorMXBeans =
ManagementFactory.getGarbageCollectorMXBeans();
private static final List<MemoryManagerMXBean> memoryManagerMXBeans =
ManagementFactory.getMemoryManagerMXBeans();
private static final List<MemoryPoolMXBean> memoryPoolMXBeans =
ManagementFactory.getMemoryPoolMXBeans();
/**
* Formats the thread dump header for one thread.
*
* @param ti the ThreadInfo describing the thread
* @return the formatted thread dump header
*/
private static String getThreadDumpHeader(ThreadInfo ti) {
StringBuilder sb = new StringBuilder("\"" + ti.getThreadName() + "\"");
sb.append(" Id=" + ti.getThreadId());
sb.append(" cpu=" + threadMXBean.getThreadCpuTime(ti.getThreadId()) + " ns");
sb.append(" usr="******" ns");
sb.append(" blocked " + ti.getBlockedCount() + " for " + ti.getBlockedTime() + " ms");
sb.append(" waited " + ti.getWaitedCount() + " for " + ti.getWaitedTime() + " ms");
if (ti.isSuspended()) {
sb.append(" (suspended)");
}
if (ti.isInNative()) {
sb.append(" (running in native)");
}
sb.append(CRLF);
sb.append(INDENT3 + "java.lang.Thread.State: " + ti.getThreadState());
sb.append(CRLF);
return sb.toString();
}
/**
* Formats the thread dump for one thread.
*
* @param ti the ThreadInfo describing the thread
* @return the formatted thread dump
*/
private static String getThreadDump(ThreadInfo ti) {
StringBuilder sb = new StringBuilder(getThreadDumpHeader(ti));
for (LockInfo li : ti.getLockedSynchronizers()) {
sb.append(INDENT2 + "locks " + li.toString() + CRLF);
}
boolean start = true;
StackTraceElement[] stes = ti.getStackTrace();
Object[] monitorDepths = new Object[stes.length];
MonitorInfo[] mis = ti.getLockedMonitors();
for (int i = 0; i < mis.length; i++) {
monitorDepths[mis[i].getLockedStackDepth()] = mis[i];
}
for (int i = 0; i < stes.length; i++) {
StackTraceElement ste = stes[i];
sb.append(INDENT2 + "at " + ste.toString() + CRLF);
if (start) {
if (ti.getLockName() != null) {
sb.append(INDENT2 + "- waiting on (a " + ti.getLockName() + ")");
if (ti.getLockOwnerName() != null) {
sb.append(" owned by " + ti.getLockOwnerName() + " Id=" + ti.getLockOwnerId());
}
sb.append(CRLF);
}
start = false;
}
if (monitorDepths[i] != null) {
MonitorInfo mi = (MonitorInfo) monitorDepths[i];
sb.append(
INDENT2
+ "- locked (a "
+ mi.toString()
+ ")"
+ " index "
+ mi.getLockedStackDepth()
+ " frame "
+ mi.getLockedStackFrame().toString());
sb.append(CRLF);
}
}
return sb.toString();
}
/**
* Formats the thread dump for a list of threads.
*
* @param tinfos the ThreadInfo array describing the thread list
* @return the formatted thread dump
*/
private static String getThreadDump(ThreadInfo[] tinfos) {
StringBuilder sb = new StringBuilder();
for (ThreadInfo tinfo : tinfos) {
sb.append(getThreadDump(tinfo));
sb.append(CRLF);
}
return sb.toString();
}
/**
* Check if any threads are deadlocked. If any, print the thread dump for those threads.
*
* @return a deadlock message and the formatted thread dump of the deadlocked threads
*/
private static String findDeadlock() {
ThreadInfo[] tinfos = null;
long[] ids = threadMXBean.findDeadlockedThreads();
if (ids != null) {
tinfos = threadMXBean.getThreadInfo(threadMXBean.findDeadlockedThreads(), true, true);
if (tinfos != null) {
StringBuilder sb = new StringBuilder("Deadlock found between the following threads:");
sb.append(CRLF);
sb.append(getThreadDump(tinfos));
return sb.toString();
}
}
return "";
}
/**
* Retrieves a formatted JVM thread dump. The given list of locales will be used to retrieve a
* StringManager.
*
* @param requestedLocales list of locales to use
* @return the formatted JVM thread dump
*/
public static String getThreadDump(Enumeration<Locale> requestedLocales) {
return getThreadDump(StringManager.getManager(PACKAGE, requestedLocales));
}
/**
* Retrieve a JVM thread dump formatted using the given StringManager.
*
* @param requestedSm the StringManager to use
* @return the formatted JVM thread dump
*/
private static String getThreadDump(StringManager requestedSm) {
StringBuilder sb = new StringBuilder();
synchronized (timeformat) {
sb.append(timeformat.format(new Date()));
}
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.threadDumpTitle"));
sb.append(" ");
sb.append(runtimeMXBean.getVmName());
sb.append(" (");
sb.append(runtimeMXBean.getVmVersion());
String vminfo = System.getProperty(vminfoSystemProperty);
if (vminfo != null) {
sb.append(" " + vminfo);
}
sb.append("):" + CRLF);
sb.append(CRLF);
ThreadInfo[] tis = threadMXBean.dumpAllThreads(true, true);
sb.append(getThreadDump(tis));
sb.append(findDeadlock());
return sb.toString();
}
/**
* Format contents of a MemoryUsage object.
*
* @param name a text prefix used in formatting
* @param usage the MemoryUsage object to format
* @return the formatted contents
*/
private static String formatMemoryUsage(String name, MemoryUsage usage) {
if (usage != null) {
StringBuilder sb = new StringBuilder();
sb.append(INDENT1 + name + " init: " + usage.getInit() + CRLF);
sb.append(INDENT1 + name + " used: " + usage.getUsed() + CRLF);
sb.append(INDENT1 + name + " committed: " + usage.getCommitted() + CRLF);
sb.append(INDENT1 + name + " max: " + usage.getMax() + CRLF);
return sb.toString();
}
return "";
}
/**
* Retrieves a formatted JVM information text. The given list of locales will be used to retrieve
* a StringManager.
*
* @param requestedLocales list of locales to use
* @return the formatted JVM information text
*/
public static String getVMInfo(Enumeration<Locale> requestedLocales) {
return getVMInfo(StringManager.getManager(PACKAGE, requestedLocales));
}
/**
* Retrieve a JVM information text formatted using the given StringManager.
*
* @param requestedSm the StringManager to use
* @return the formatted JVM information text
*/
private static String getVMInfo(StringManager requestedSm) {
StringBuilder sb = new StringBuilder();
synchronized (timeformat) {
sb.append(timeformat.format(new Date()));
}
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoRuntime"));
sb.append(":" + CRLF);
sb.append(INDENT1 + "vmName: " + runtimeMXBean.getVmName() + CRLF);
sb.append(INDENT1 + "vmVersion: " + runtimeMXBean.getVmVersion() + CRLF);
sb.append(INDENT1 + "vmVendor: " + runtimeMXBean.getVmVendor() + CRLF);
sb.append(INDENT1 + "specName: " + runtimeMXBean.getSpecName() + CRLF);
sb.append(INDENT1 + "specVersion: " + runtimeMXBean.getSpecVersion() + CRLF);
sb.append(INDENT1 + "specVendor: " + runtimeMXBean.getSpecVendor() + CRLF);
sb.append(
INDENT1 + "managementSpecVersion: " + runtimeMXBean.getManagementSpecVersion() + CRLF);
sb.append(INDENT1 + "name: " + runtimeMXBean.getName() + CRLF);
sb.append(INDENT1 + "startTime: " + runtimeMXBean.getStartTime() + CRLF);
sb.append(INDENT1 + "uptime: " + runtimeMXBean.getUptime() + CRLF);
sb.append(
INDENT1 + "isBootClassPathSupported: " + runtimeMXBean.isBootClassPathSupported() + CRLF);
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoOs"));
sb.append(":" + CRLF);
sb.append(INDENT1 + "name: " + operatingSystemMXBean.getName() + CRLF);
sb.append(INDENT1 + "version: " + operatingSystemMXBean.getVersion() + CRLF);
sb.append(INDENT1 + "architecture: " + operatingSystemMXBean.getArch() + CRLF);
sb.append(
INDENT1 + "availableProcessors: " + operatingSystemMXBean.getAvailableProcessors() + CRLF);
sb.append(
INDENT1 + "systemLoadAverage: " + operatingSystemMXBean.getSystemLoadAverage() + CRLF);
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoThreadMxBean"));
sb.append(":" + CRLF);
sb.append(
INDENT1
+ "isCurrentThreadCpuTimeSupported: "
+ threadMXBean.isCurrentThreadCpuTimeSupported()
+ CRLF);
sb.append(
INDENT1 + "isThreadCpuTimeSupported: " + threadMXBean.isThreadCpuTimeSupported() + CRLF);
sb.append(INDENT1 + "isThreadCpuTimeEnabled: " + threadMXBean.isThreadCpuTimeEnabled() + CRLF);
sb.append(
INDENT1
+ "isObjectMonitorUsageSupported: "
+ threadMXBean.isObjectMonitorUsageSupported()
+ CRLF);
sb.append(
INDENT1
+ "isSynchronizerUsageSupported: "
+ threadMXBean.isSynchronizerUsageSupported()
+ CRLF);
sb.append(
INDENT1
+ "isThreadContentionMonitoringSupported: "
+ threadMXBean.isThreadContentionMonitoringSupported()
+ CRLF);
sb.append(
INDENT1
+ "isThreadContentionMonitoringEnabled: "
+ threadMXBean.isThreadContentionMonitoringEnabled()
+ CRLF);
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoThreadCounts"));
sb.append(":" + CRLF);
sb.append(INDENT1 + "daemon: " + threadMXBean.getDaemonThreadCount() + CRLF);
sb.append(INDENT1 + "total: " + threadMXBean.getThreadCount() + CRLF);
sb.append(INDENT1 + "peak: " + threadMXBean.getPeakThreadCount() + CRLF);
sb.append(INDENT1 + "totalStarted: " + threadMXBean.getTotalStartedThreadCount() + CRLF);
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoStartup"));
sb.append(":" + CRLF);
for (String arg : runtimeMXBean.getInputArguments()) {
sb.append(INDENT1 + arg + CRLF);
}
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoPath"));
sb.append(":" + CRLF);
sb.append(INDENT1 + "bootClassPath: " + runtimeMXBean.getBootClassPath() + CRLF);
sb.append(INDENT1 + "classPath: " + runtimeMXBean.getClassPath() + CRLF);
sb.append(INDENT1 + "libraryPath: " + runtimeMXBean.getLibraryPath() + CRLF);
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoClassLoading"));
sb.append(":" + CRLF);
sb.append(INDENT1 + "loaded: " + classLoadingMXBean.getLoadedClassCount() + CRLF);
sb.append(INDENT1 + "unloaded: " + classLoadingMXBean.getUnloadedClassCount() + CRLF);
sb.append(INDENT1 + "totalLoaded: " + classLoadingMXBean.getTotalLoadedClassCount() + CRLF);
sb.append(INDENT1 + "isVerbose: " + classLoadingMXBean.isVerbose() + CRLF);
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoClassCompilation"));
sb.append(":" + CRLF);
sb.append(INDENT1 + "name: " + compilationMXBean.getName() + CRLF);
sb.append(
INDENT1 + "totalCompilationTime: " + compilationMXBean.getTotalCompilationTime() + CRLF);
sb.append(
INDENT1
+ "isCompilationTimeMonitoringSupported: "
+ compilationMXBean.isCompilationTimeMonitoringSupported()
+ CRLF);
sb.append(CRLF);
for (MemoryManagerMXBean mbean : memoryManagerMXBeans) {
sb.append(requestedSm.getString("diagnostics.vmInfoMemoryManagers", mbean.getName()));
sb.append(":" + CRLF);
sb.append(INDENT1 + "isValid: " + mbean.isValid() + CRLF);
sb.append(INDENT1 + "mbean.getMemoryPoolNames: " + CRLF);
String[] names = mbean.getMemoryPoolNames();
Arrays.sort(names);
for (String name : names) {
sb.append(INDENT2 + name + CRLF);
}
sb.append(CRLF);
}
for (GarbageCollectorMXBean mbean : garbageCollectorMXBeans) {
sb.append(requestedSm.getString("diagnostics.vmInfoGarbageCollectors", mbean.getName()));
sb.append(":" + CRLF);
sb.append(INDENT1 + "isValid: " + mbean.isValid() + CRLF);
sb.append(INDENT1 + "mbean.getMemoryPoolNames: " + CRLF);
String[] names = mbean.getMemoryPoolNames();
Arrays.sort(names);
for (String name : names) {
sb.append(INDENT2 + name + CRLF);
}
sb.append(INDENT1 + "getCollectionCount: " + mbean.getCollectionCount() + CRLF);
sb.append(INDENT1 + "getCollectionTime: " + mbean.getCollectionTime() + CRLF);
sb.append(CRLF);
}
sb.append(requestedSm.getString("diagnostics.vmInfoMemory"));
sb.append(":" + CRLF);
sb.append(INDENT1 + "isVerbose: " + memoryMXBean.isVerbose() + CRLF);
sb.append(
INDENT1
+ "getObjectPendingFinalizationCount: "
+ memoryMXBean.getObjectPendingFinalizationCount()
+ CRLF);
sb.append(formatMemoryUsage("heap", memoryMXBean.getHeapMemoryUsage()));
sb.append(formatMemoryUsage("non-heap", memoryMXBean.getNonHeapMemoryUsage()));
sb.append(CRLF);
for (MemoryPoolMXBean mbean : memoryPoolMXBeans) {
sb.append(requestedSm.getString("diagnostics.vmInfoMemoryPools", mbean.getName()));
sb.append(":" + CRLF);
sb.append(INDENT1 + "isValid: " + mbean.isValid() + CRLF);
sb.append(INDENT1 + "getType: " + mbean.getType() + CRLF);
sb.append(INDENT1 + "mbean.getMemoryManagerNames: " + CRLF);
String[] names = mbean.getMemoryManagerNames();
Arrays.sort(names);
for (String name : names) {
sb.append(INDENT2 + name + CRLF);
}
sb.append(INDENT1 + "isUsageThresholdSupported: " + mbean.isUsageThresholdSupported() + CRLF);
try {
sb.append(INDENT1 + "isUsageThresholdExceeded: " + mbean.isUsageThresholdExceeded() + CRLF);
} catch (UnsupportedOperationException ex) {
// IGNORE
}
sb.append(
INDENT1
+ "isCollectionUsageThresholdSupported: "
+ mbean.isCollectionUsageThresholdSupported()
+ CRLF);
try {
sb.append(
INDENT1
+ "isCollectionUsageThresholdExceeded: "
+ mbean.isCollectionUsageThresholdExceeded()
+ CRLF);
} catch (UnsupportedOperationException ex) {
// IGNORE
}
try {
sb.append(INDENT1 + "getUsageThreshold: " + mbean.getUsageThreshold() + CRLF);
} catch (UnsupportedOperationException ex) {
// IGNORE
}
try {
sb.append(INDENT1 + "getUsageThresholdCount: " + mbean.getUsageThresholdCount() + CRLF);
} catch (UnsupportedOperationException ex) {
// IGNORE
}
try {
sb.append(
INDENT1 + "getCollectionUsageThreshold: " + mbean.getCollectionUsageThreshold() + CRLF);
} catch (UnsupportedOperationException ex) {
// IGNORE
}
try {
sb.append(
INDENT1
+ "getCollectionUsageThresholdCount: "
+ mbean.getCollectionUsageThresholdCount()
+ CRLF);
} catch (UnsupportedOperationException ex) {
// IGNORE
}
sb.append(formatMemoryUsage("current", mbean.getUsage()));
sb.append(formatMemoryUsage("collection", mbean.getCollectionUsage()));
sb.append(formatMemoryUsage("peak", mbean.getPeakUsage()));
sb.append(CRLF);
}
sb.append(requestedSm.getString("diagnostics.vmInfoSystem"));
sb.append(":" + CRLF);
Map<String, String> props = runtimeMXBean.getSystemProperties();
ArrayList<String> keys = new ArrayList<String>(props.keySet());
Collections.sort(keys);
for (String prop : keys) {
sb.append(INDENT1 + prop + ": " + props.get(prop) + CRLF);
}
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoLogger"));
sb.append(":" + CRLF);
List<String> loggers = loggingMXBean.getLoggerNames();
Collections.sort(loggers);
for (String logger : loggers) {
sb.append(
INDENT1
+ logger
+ ": level="
+ loggingMXBean.getLoggerLevel(logger)
+ ", parent="
+ loggingMXBean.getParentLoggerName(logger)
+ CRLF);
}
sb.append(CRLF);
return sb.toString();
}
}
private void logStats() {
long finishTime = System.currentTimeMillis();
long finalGCTimes = 0;
List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
for (GarbageCollectorMXBean garbageCollectorMXBean : gcmBeans) {
finalGCTimes += garbageCollectorMXBean.getCollectionTime();
}
CompilationMXBean compMxBean = ManagementFactory.getCompilationMXBean();
long finalCompileTimes = 0;
if (compMxBean.isCompilationTimeMonitoringSupported()) {
finalCompileTimes = compMxBean.getTotalCompilationTime();
}
double averageRate = totalSent.get() / (totalSendTime.get() / 1000.0);
double overallRate = totalAdded / ((finishTime - startTime) / 1000.0);
double finalSystemLoad = ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();
if (log.isTraceEnabled()) {
log.trace("");
log.trace("TABLET SERVER BATCH WRITER STATISTICS");
log.trace(String.format("Added : %,10d mutations", totalAdded));
log.trace(String.format("Sent : %,10d mutations", totalSent.get()));
log.trace(
String.format(
"Resent percentage : %10.2f%s",
(totalSent.get() - totalAdded) / (double) totalAdded * 100.0, "%"));
log.trace(
String.format("Overall time : %,10.2f secs", (finishTime - startTime) / 1000.0));
log.trace(String.format("Overall send rate : %,10.2f mutations/sec", overallRate));
log.trace(
String.format("Send efficiency : %10.2f%s", overallRate / averageRate * 100.0, "%"));
log.trace("");
log.trace("BACKGROUND WRITER PROCESS STATISTICS");
log.trace(
String.format(
"Total send time : %,10.2f secs %6.2f%s",
totalSendTime.get() / 1000.0,
100.0 * totalSendTime.get() / (finishTime - startTime),
"%"));
log.trace(String.format("Average send rate : %,10.2f mutations/sec", averageRate));
log.trace(
String.format(
"Total bin time : %,10.2f secs %6.2f%s",
totalBinTime.get() / 1000.0,
100.0 * totalBinTime.get() / (finishTime - startTime),
"%"));
log.trace(
String.format(
"Average bin rate : %,10.2f mutations/sec",
totalBinned.get() / (totalBinTime.get() / 1000.0)));
log.trace(
String.format(
"tservers per batch : %,8.2f avg %,6d min %,6d max",
(tabletServersBatchSum / (double) numBatches),
minTabletServersBatch,
maxTabletServersBatch));
log.trace(
String.format(
"tablets per batch : %,8.2f avg %,6d min %,6d max",
(tabletBatchSum / (double) numBatches), minTabletBatch, maxTabletBatch));
log.trace("");
log.trace("SYSTEM STATISTICS");
log.trace(
String.format(
"JVM GC Time : %,10.2f secs", ((finalGCTimes - initialGCTimes) / 1000.0)));
if (compMxBean.isCompilationTimeMonitoringSupported()) {
log.trace(
String.format(
"JVM Compile Time : %,10.2f secs",
((finalCompileTimes - initialCompileTimes) / 1000.0)));
}
log.trace(
String.format(
"System load average : initial=%6.2f final=%6.2f",
initialSystemLoad, finalSystemLoad));
}
}