public static String getGCStats() {
long totalGC = 0;
long gcTime = 0;
for (GarbageCollectorMXBean gc : ManagementFactory.getGarbageCollectorMXBeans()) {
long count = gc.getCollectionCount();
if (count >= 0) {
totalGC += count;
}
long time = gc.getCollectionTime();
if (time >= 0) {
gcTime += time;
}
}
StringBuilder sb = new StringBuilder();
sb.append(banner("memory stats"));
sb.append("\n- total collections: " + totalGC);
sb.append("\n- total collection time: " + formatTime(gcTime));
Runtime runtime = Runtime.getRuntime();
sb.append("\n- total memory: " + _.printMem(runtime.totalMemory()));
return sb.toString();
}
public static void main(String args[]) {
MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
MemoryUsage heapMemoryUsage = memoryMXBean.getHeapMemoryUsage();
long init = heapMemoryUsage.getInit();
long used = heapMemoryUsage.getUsed();
long max = heapMemoryUsage.getMax();
long committed = heapMemoryUsage.getCommitted();
long free = max - used;
int usedPercent = (int) (used / max);
int freePercent = (int) (usedPercent / max);
MemoryUsage nonHeapMemoryUsage = memoryMXBean.getNonHeapMemoryUsage();
List<GarbageCollectorMXBean> garbageCollectorMXBeans =
ManagementFactory.getGarbageCollectorMXBeans();
List<MemoryPoolMXBean> memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans();
com.sun.management.OperatingSystemMXBean operatingSystemMXBean =
(com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
long commitVirtual = operatingSystemMXBean.getCommittedVirtualMemorySize();
println(init, used, max, committed);
println(commitVirtual);
}
/**
* Returns the approximate accumulated garbage collection elapsed time in milliseconds.
*
* @return the approximate accumulated garbage collection elapsed time in milliseconds.
*/
public Number getTime() {
List<GarbageCollectorMXBean> garbageCollectors =
ManagementFactory.getGarbageCollectorMXBeans();
long sum = 0;
for (GarbageCollectorMXBean garbageCollector : garbageCollectors) {
sum += garbageCollector.getCollectionTime();
}
return Long.valueOf(sum);
}
private void doGarbageCollectionUpdates() {
List<GarbageCollectorMXBean> gcBeans = ManagementFactory.getGarbageCollectorMXBeans();
long count = 0;
long timeMillis = 0;
for (GarbageCollectorMXBean gcBean : gcBeans) {
count += gcBean.getCollectionCount();
timeMillis += gcBean.getCollectionTime();
}
metrics.incrMetric("gcCount", (int) (count - gcCount));
metrics.incrMetric("gcTimeMillis", (int) (timeMillis - gcTimeMillis));
gcCount = count;
gcTimeMillis = timeMillis;
}
public static void main(String[] args) {
List<GarbageCollectorMXBean> list = ManagementFactory.getGarbageCollectorMXBeans();
for (GarbageCollectorMXBean bean : list) {
System.out.println("Name: " + bean.getName());
System.out.println("Number of collections: " + bean.getCollectionCount());
System.out.println("Collection time: " + bean.getCollectionTime() + "ms");
System.out.println("Pool names");
for (String name : bean.getMemoryPoolNames()) {
System.out.println("\t" + name);
}
System.out.println();
}
}
@Override
protected void executeReadAttribute(OperationContext context, ModelNode operation)
throws OperationFailedException {
final String gcName =
PathAddress.pathAddress(operation.require(ModelDescriptionConstants.OP_ADDR))
.getLastElement()
.getValue();
final String name = operation.require(ModelDescriptionConstants.NAME).asString();
GarbageCollectorMXBean gcMBean = null;
for (GarbageCollectorMXBean mbean : ManagementFactory.getGarbageCollectorMXBeans()) {
if (gcName.equals(escapeMBeanName(mbean.getName()))) {
gcMBean = mbean;
}
}
if (gcMBean == null) {
throw PlatformMBeanLogger.ROOT_LOGGER.unknownGarbageCollector(gcName);
}
if (PlatformMBeanConstants.OBJECT_NAME.getName().equals(name)) {
final String objName =
PlatformMBeanUtil.getObjectNameStringWithNameKey(
ManagementFactory.GARBAGE_COLLECTOR_MXBEAN_DOMAIN_TYPE, gcName);
context.getResult().set(objName);
} else if (ModelDescriptionConstants.NAME.equals(name)) {
context.getResult().set(escapeMBeanName(gcMBean.getName()));
} else if (PlatformMBeanConstants.VALID.getName().equals(name)) {
context.getResult().set(gcMBean.isValid());
} else if (PlatformMBeanConstants.MEMORY_POOL_NAMES.equals(name)) {
final ModelNode result = context.getResult();
result.setEmptyList();
for (String pool : gcMBean.getMemoryPoolNames()) {
result.add(escapeMBeanName(pool));
}
} else if (PlatformMBeanConstants.COLLECTION_COUNT.equals(name)) {
context.getResult().set(gcMBean.getCollectionCount());
} else if (PlatformMBeanConstants.COLLECTION_TIME.equals(name)) {
context.getResult().set(gcMBean.getCollectionTime());
} else if (GarbageCollectorResourceDefinition.GARBAGE_COLLECTOR_READ_ATTRIBUTES.contains(name)
|| GarbageCollectorResourceDefinition.GARBAGE_COLLECTOR_METRICS.contains(name)) {
// Bug
throw PlatformMBeanLogger.ROOT_LOGGER.badReadAttributeImpl(name);
} else {
// Shouldn't happen; the global handler should reject
throw unknownAttribute(operation);
}
}
/** Add garbage collection metrics. */
protected void addGarbageCollecitonMetrics(Collection<Metric<?>> result) {
List<GarbageCollectorMXBean> garbageCollectorMxBeans =
ManagementFactory.getGarbageCollectorMXBeans();
for (int i = 0; i < garbageCollectorMxBeans.size(); i++) {
GarbageCollectorMXBean garbageCollectorMXBean = garbageCollectorMxBeans.get(i);
String name = beautifyGcName(garbageCollectorMXBean.getName());
result.add(
new Metric<Long>(
"gc." + name + ".count", new Long(garbageCollectorMXBean.getCollectionCount())));
result.add(
new Metric<Long>(
"gc." + name + ".time", new Long(garbageCollectorMXBean.getCollectionTime())));
}
}
public synchronized void addMutation(String table, Mutation m) throws MutationsRejectedException {
if (closed) throw new IllegalStateException("Closed");
if (m.size() == 0) throw new IllegalArgumentException("Can not add empty mutations");
checkForFailures();
while ((totalMemUsed >= maxMem || flushing) && !somethingFailed) {
waitRTE();
}
// do checks again since things could have changed while waiting and not holding lock
if (closed) throw new IllegalStateException("Closed");
checkForFailures();
if (startTime == 0) {
startTime = System.currentTimeMillis();
List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
for (GarbageCollectorMXBean garbageCollectorMXBean : gcmBeans) {
initialGCTimes += garbageCollectorMXBean.getCollectionTime();
}
CompilationMXBean compMxBean = ManagementFactory.getCompilationMXBean();
if (compMxBean.isCompilationTimeMonitoringSupported()) {
initialCompileTimes = compMxBean.getTotalCompilationTime();
}
initialSystemLoad = ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();
}
// create a copy of mutation so that after this method returns the user
// is free to reuse the mutation object, like calling readFields... this
// is important for the case where a mutation is passed from map to reduce
// to batch writer... the map reduce code will keep passing the same mutation
// object into the reduce method
m = new Mutation(m);
totalMemUsed += m.estimatedMemoryUsed();
mutations.addMutation(table, m);
totalAdded++;
if (mutations.getMemoryUsed() >= maxMem / 2) {
startProcessing();
checkForFailures();
}
}
static void MonitorGC(long h) {
handle = h;
List<GarbageCollectorMXBean> gcbeans =
java.lang.management.ManagementFactory.getGarbageCollectorMXBeans();
MBeanServer server = ManagementFactory.getPlatformMBeanServer();
try {
ObjectName gcName =
new ObjectName(ManagementFactory.GARBAGE_COLLECTOR_MXBEAN_DOMAIN_TYPE + ",*");
for (ObjectName name : server.queryNames(gcName, null)) {
GarbageCollectorMXBean gc =
ManagementFactory.newPlatformMXBeanProxy(
server, name.getCanonicalName(), GarbageCollectorMXBean.class);
gcbeans.add(gc);
NotificationEmitter emitter = (NotificationEmitter) gc;
NotificationListener listener =
new NotificationListener() {
@Override
public void handleNotification(Notification notification, Object handback) {
if (notification
.getType()
.equals(GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION)) {
CompositeData ndata = (CompositeData) notification.getUserData();
GarbageCollectionNotificationInfo info =
GarbageCollectionNotificationInfo.from(ndata);
boolean major = "end of major GC".equals(info.getGcAction());
long free = Runtime.getRuntime().freeMemory();
long total = Runtime.getRuntime().totalMemory();
long qty = (15 * total) - (free * 100);
if (qty > 0) {
NotifyOSv(handle, qty);
}
}
}
};
emitter.addNotificationListener(listener, null, null);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public static List<GarbageCollectorMXBean> gc() {
return ManagementFactory.getGarbageCollectorMXBeans();
}
public static void main(String[] args) {
if (args.length != 1) {
throw new Error("Expected memory counter name wasn't provided as command line argument");
}
MemoryCounter memoryCounter = MemoryCounter.valueOf(args[0].toUpperCase());
int byteArrayMemoryOverhead = Helpers.detectByteArrayAllocationOverhead();
// Largest non-humongous byte[]
int maxByteArrayNonHumongousSize = HALF_G1_REGION_SIZE - byteArrayMemoryOverhead;
// Maximum byte[] that takes one region
int maxByteArrayOneRegionSize = G1_REGION_SIZE - byteArrayMemoryOverhead;
List<Integer> allocationSizes =
Arrays.asList(
(int) maxByteArrayNonHumongousSize + 1,
(int) (0.8f * maxByteArrayOneRegionSize),
(int) (maxByteArrayOneRegionSize),
(int) (1.2f * maxByteArrayOneRegionSize),
(int) (1.5f * maxByteArrayOneRegionSize),
(int) (1.7f * maxByteArrayOneRegionSize),
(int) (2.0f * maxByteArrayOneRegionSize),
(int) (2.5f * maxByteArrayOneRegionSize));
List<Allocation> allocations = new ArrayList<>();
List<GarbageCollectorMXBean> gcBeans = ManagementFactory.getGarbageCollectorMXBeans();
long gcCountBefore =
gcBeans.stream().mapToLong(GarbageCollectorMXBean::getCollectionCount).sum();
System.out.println("Starting allocations - no GC should happen until we finish them");
for (int allocationSize : allocationSizes) {
long usedMemoryBefore = memoryCounter.getUsedMemory();
long expectedAllocationSize =
(long) Math.ceil((double) allocationSize / G1_REGION_SIZE) * G1_REGION_SIZE;
allocations.add(new Allocation(allocationSize, expectedAllocationSize));
long usedMemoryAfter = memoryCounter.getUsedMemory();
System.out.format(
"Expected allocation size: %d\nUsed memory before allocation: %d\n"
+ "Used memory after allocation: %d\n",
expectedAllocationSize, usedMemoryBefore, usedMemoryAfter);
long gcCountNow =
gcBeans.stream().mapToLong(GarbageCollectorMXBean::getCollectionCount).sum();
if (gcCountNow == gcCountBefore) {
// We should allocate at least allocation.expectedSize
Asserts.assertGreaterThanOrEqual(
usedMemoryAfter - usedMemoryBefore,
expectedAllocationSize,
"Counter of type "
+ memoryCounter.getClass().getSimpleName()
+ " returned wrong allocation size");
} else {
System.out.println("GC happened during allocation so the check is skipped");
gcCountBefore = gcCountNow;
}
}
System.out.println("Finished allocations - no GC should have happened before this line");
allocations
.stream()
.forEach(
allocation -> {
long usedMemoryBefore = memoryCounter.getUsedMemory();
allocation.forgetAllocation();
WHITE_BOX.fullGC();
long usedMemoryAfter = memoryCounter.getUsedMemory();
// We should free at least allocation.expectedSize * ALLOCATION_SIZE_TOLERANCE_FACTOR
Asserts.assertGreaterThanOrEqual(
usedMemoryBefore - usedMemoryAfter,
(long) (allocation.expectedSize * ALLOCATION_SIZE_TOLERANCE_FACTOR),
"Counter of type "
+ memoryCounter.getClass().getSimpleName()
+ " returned wrong allocation size");
});
}
public GcTimeUpdater() {
this.gcBeans = ManagementFactory.getGarbageCollectorMXBeans();
getElapsedGc(); // Initialize 'lastGcMillis' with the current time spent.
}
static void installGCMonitoring() {
List<GarbageCollectorMXBean> gcbeans = ManagementFactory.getGarbageCollectorMXBeans();
for (GarbageCollectorMXBean gcbean : gcbeans) {
System.out.println(gcbean);
NotificationEmitter emitter = (NotificationEmitter) gcbean;
// use an anonymously generated listener for this example
// - proper code should really use a named class
NotificationListener listener =
new NotificationListener() {
// keep a count of the total time spent in GCs
long totalGcDuration = 0;
// implement the notifier callback handler
@Override
public void handleNotification(Notification notification, Object handback) {
// we only handle GARBAGE_COLLECTION_NOTIFICATION notifications here
if (notification
.getType()
.equals(GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION)) {
// get the information associated with this notification
GarbageCollectionNotificationInfo info =
GarbageCollectionNotificationInfo.from(
(CompositeData) notification.getUserData());
// get all the info and pretty print it
long duration = info.getGcInfo().getDuration();
String gctype = info.getGcAction();
if ("end of minor GC".equals(gctype)) {
gctype = "Young Gen GC";
} else if ("end of major GC".equals(gctype)) {
gctype = "Old Gen GC";
}
System.out.println();
System.out.println(
gctype
+ ": - "
+ info.getGcInfo().getId()
+ " "
+ info.getGcName()
+ " (from "
+ info.getGcCause()
+ ") "
+ duration
+ " microseconds; start-end times "
+ info.getGcInfo().getStartTime()
+ "-"
+ info.getGcInfo().getEndTime());
// System.out.println("GcInfo CompositeType: " +
// info.getGcInfo().getCompositeType());
// System.out.println("GcInfo MemoryUsageAfterGc: " +
// info.getGcInfo().getMemoryUsageAfterGc());
// System.out.println("GcInfo MemoryUsageBeforeGc: " +
// info.getGcInfo().getMemoryUsageBeforeGc());
// Get the information about each memory space, and pretty print it
Map<String, MemoryUsage> membefore = info.getGcInfo().getMemoryUsageBeforeGc();
Map<String, MemoryUsage> mem = info.getGcInfo().getMemoryUsageAfterGc();
for (Map.Entry<String, MemoryUsage> entry : mem.entrySet()) {
String name = entry.getKey();
MemoryUsage memdetail = entry.getValue();
long memInit = memdetail.getInit();
long memCommitted = memdetail.getCommitted();
long memMax = memdetail.getMax();
long memUsed = memdetail.getUsed();
MemoryUsage before = membefore.get(name);
long beforepercent = ((before.getUsed() * 1000L) / before.getCommitted());
long percent =
((memUsed * 1000L) / before.getCommitted()); // >100% when it gets expanded
System.out.print(
name
+ (memCommitted == memMax ? "(fully expanded)" : "(still expandable)")
+ "used: "
+ (beforepercent / 10)
+ "."
+ (beforepercent % 10)
+ "%->"
+ (percent / 10)
+ "."
+ (percent % 10)
+ "%("
+ ((memUsed / 1048576) + 1)
+ "MB) / ");
}
System.out.println();
totalGcDuration += info.getGcInfo().getDuration();
long percent = totalGcDuration * 1000L / info.getGcInfo().getEndTime();
System.out.println(
"GC cumulated overhead " + (percent / 10) + "." + (percent % 10) + "%");
}
}
};
// Add the listener
emitter.addNotificationListener(listener, null, null);
}
}
/**
* All parameters are obtained from the {@link GlobalConfiguration}, which must be loaded prior to
* instantiating the task manager.
*/
public TaskManager(ExecutionMode executionMode) throws Exception {
if (executionMode == null) {
throw new NullPointerException("Execution mode must not be null.");
}
LOG.info("Execution mode: " + executionMode);
// IMPORTANT! At this point, the GlobalConfiguration must have been read!
final InetSocketAddress jobManagerAddress;
{
LOG.info("Reading location of job manager from configuration");
final String address =
GlobalConfiguration.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null);
final int port =
GlobalConfiguration.getInteger(
ConfigConstants.JOB_MANAGER_IPC_PORT_KEY,
ConfigConstants.DEFAULT_JOB_MANAGER_IPC_PORT);
if (address == null) {
throw new Exception("Job manager address not configured in the GlobalConfiguration.");
}
// Try to convert configured address to {@link InetAddress}
try {
final InetAddress tmpAddress = InetAddress.getByName(address);
jobManagerAddress = new InetSocketAddress(tmpAddress, port);
} catch (UnknownHostException e) {
LOG.fatal("Could not resolve JobManager host name.");
throw new Exception("Could not resolve JobManager host name: " + e.getMessage(), e);
}
LOG.info("Connecting to JobManager at: " + jobManagerAddress);
}
// Create RPC connection to the JobManager
try {
this.jobManager =
RPC.getProxy(JobManagerProtocol.class, jobManagerAddress, NetUtils.getSocketFactory());
} catch (IOException e) {
LOG.fatal("Could not connect to the JobManager: " + e.getMessage(), e);
throw new Exception("Failed to initialize connection to JobManager: " + e.getMessage(), e);
}
int ipcPort = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, -1);
int dataPort = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, -1);
if (ipcPort == -1) {
ipcPort = getAvailablePort();
}
if (dataPort == -1) {
dataPort = getAvailablePort();
}
// Determine our own public facing address and start the server
{
final InetAddress taskManagerAddress;
try {
taskManagerAddress = getTaskManagerAddress(jobManagerAddress);
} catch (Exception e) {
throw new RuntimeException(
"The TaskManager failed to determine its own network address.", e);
}
this.localInstanceConnectionInfo =
new InstanceConnectionInfo(taskManagerAddress, ipcPort, dataPort);
LOG.info("TaskManager connection information:" + this.localInstanceConnectionInfo);
// Start local RPC server
try {
this.taskManagerServer =
RPC.getServer(this, taskManagerAddress.getHostAddress(), ipcPort, IPC_HANDLER_COUNT);
this.taskManagerServer.start();
} catch (IOException e) {
LOG.fatal("Failed to start TaskManager server. " + e.getMessage(), e);
throw new Exception("Failed to start taskmanager server. " + e.getMessage(), e);
}
}
// Try to create local stub of the global input split provider
try {
this.globalInputSplitProvider =
RPC.getProxy(
InputSplitProviderProtocol.class, jobManagerAddress, NetUtils.getSocketFactory());
} catch (IOException e) {
LOG.fatal(e.getMessage(), e);
throw new Exception(
"Failed to initialize connection to global input split provider: " + e.getMessage(), e);
}
// Try to create local stub for the lookup service
try {
this.lookupService =
RPC.getProxy(ChannelLookupProtocol.class, jobManagerAddress, NetUtils.getSocketFactory());
} catch (IOException e) {
LOG.fatal(e.getMessage(), e);
throw new Exception("Failed to initialize channel lookup protocol. " + e.getMessage(), e);
}
// Try to create local stub for the accumulators
try {
this.accumulatorProtocolProxy =
RPC.getProxy(AccumulatorProtocol.class, jobManagerAddress, NetUtils.getSocketFactory());
} catch (IOException e) {
LOG.fatal("Failed to initialize accumulator protocol: " + e.getMessage(), e);
throw new Exception("Failed to initialize accumulator protocol: " + e.getMessage(), e);
}
// Load profiler if it should be used
if (GlobalConfiguration.getBoolean(ProfilingUtils.ENABLE_PROFILING_KEY, false)) {
final String profilerClassName =
GlobalConfiguration.getString(
ProfilingUtils.TASKMANAGER_CLASSNAME_KEY,
"org.apache.flink.runtime.profiling.impl.TaskManagerProfilerImpl");
this.profiler =
ProfilingUtils.loadTaskManagerProfiler(
profilerClassName, jobManagerAddress.getAddress(), this.localInstanceConnectionInfo);
if (this.profiler == null) {
LOG.error("Cannot find class name for the profiler.");
} else {
LOG.info("Profiling of jobs is enabled.");
}
} else {
this.profiler = null;
LOG.info("Profiling of jobs is disabled.");
}
// Get the directory for storing temporary files
final String[] tmpDirPaths =
GlobalConfiguration.getString(
ConfigConstants.TASK_MANAGER_TMP_DIR_KEY,
ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH)
.split(",|" + File.pathSeparator);
checkTempDirs(tmpDirPaths);
int numBuffers =
GlobalConfiguration.getInteger(
ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY,
ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS);
int bufferSize =
GlobalConfiguration.getInteger(
ConfigConstants.TASK_MANAGER_NETWORK_BUFFER_SIZE_KEY,
ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_BUFFER_SIZE);
// Initialize the channel manager
try {
NetworkConnectionManager networkConnectionManager = null;
switch (executionMode) {
case LOCAL:
networkConnectionManager = new LocalConnectionManager();
break;
case CLUSTER:
int numInThreads =
GlobalConfiguration.getInteger(
ConfigConstants.TASK_MANAGER_NET_NUM_IN_THREADS_KEY,
ConfigConstants.DEFAULT_TASK_MANAGER_NET_NUM_IN_THREADS);
int numOutThreads =
GlobalConfiguration.getInteger(
ConfigConstants.TASK_MANAGER_NET_NUM_OUT_THREADS_KEY,
ConfigConstants.DEFAULT_TASK_MANAGER_NET_NUM_OUT_THREADS);
int lowWaterMark =
GlobalConfiguration.getInteger(
ConfigConstants.TASK_MANAGER_NET_NETTY_LOW_WATER_MARK,
ConfigConstants.DEFAULT_TASK_MANAGER_NET_NETTY_LOW_WATER_MARK);
int highWaterMark =
GlobalConfiguration.getInteger(
ConfigConstants.TASK_MANAGER_NET_NETTY_HIGH_WATER_MARK,
ConfigConstants.DEFAULT_TASK_MANAGER_NET_NETTY_HIGH_WATER_MARK);
networkConnectionManager =
new NettyConnectionManager(
localInstanceConnectionInfo.address(),
localInstanceConnectionInfo.dataPort(),
bufferSize,
numInThreads,
numOutThreads,
lowWaterMark,
highWaterMark);
break;
}
channelManager =
new ChannelManager(
lookupService,
localInstanceConnectionInfo,
numBuffers,
bufferSize,
networkConnectionManager);
} catch (IOException ioe) {
LOG.error(StringUtils.stringifyException(ioe));
throw new Exception("Failed to instantiate ChannelManager.", ioe);
}
// initialize the number of slots
{
int slots = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, -1);
if (slots == -1) {
slots = 1;
LOG.info("Number of task slots not configured. Creating one task slot.");
} else if (slots <= 0) {
throw new Exception("Illegal value for the number of task slots: " + slots);
} else {
LOG.info("Creating " + slots + " task slot(s).");
}
this.numberOfSlots = slots;
}
this.hardwareDescription = HardwareDescriptionFactory.extractFromSystem();
// initialize the memory manager
{
// Check whether the memory size has been explicitly configured.
final long configuredMemorySize =
GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1);
final long memorySize;
if (configuredMemorySize == -1) {
// no manually configured memory. take a relative fraction of the free heap space
float fraction =
GlobalConfiguration.getFloat(
ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY,
ConfigConstants.DEFAULT_MEMORY_MANAGER_MEMORY_FRACTION);
memorySize = (long) (this.hardwareDescription.getSizeOfFreeMemory() * fraction);
LOG.info("Using " + fraction + " of the free heap space for managed memory.");
} else if (configuredMemorySize <= 0) {
throw new Exception(
"Invalid value for Memory Manager memory size: " + configuredMemorySize);
} else {
memorySize = configuredMemorySize << 20;
}
final int pageSize =
GlobalConfiguration.getInteger(
ConfigConstants.TASK_MANAGER_NETWORK_BUFFER_SIZE_KEY,
ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_BUFFER_SIZE);
// Initialize the memory manager
LOG.info(
"Initializing memory manager with "
+ (memorySize >>> 20)
+ " megabytes of memory. "
+ "Page size is "
+ pageSize
+ " bytes.");
try {
@SuppressWarnings("unused")
final boolean lazyAllocation =
GlobalConfiguration.getBoolean(
ConfigConstants.TASK_MANAGER_MEMORY_LAZY_ALLOCATION_KEY,
ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_LAZY_ALLOCATION);
this.memoryManager = new DefaultMemoryManager(memorySize, this.numberOfSlots, pageSize);
} catch (Throwable t) {
LOG.fatal(
"Unable to initialize memory manager with "
+ (memorySize >>> 20)
+ " megabytes of memory.",
t);
throw new Exception("Unable to initialize memory manager.", t);
}
}
this.ioManager = new IOManager(tmpDirPaths);
this.heartbeatThread =
new Thread() {
@Override
public void run() {
runHeartbeatLoop();
}
};
this.heartbeatThread.setName("Heartbeat Thread");
this.heartbeatThread.start();
// --------------------------------------------------------------------
// Memory Usage
// --------------------------------------------------------------------
final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
final List<GarbageCollectorMXBean> gcMXBeans = ManagementFactory.getGarbageCollectorMXBeans();
LOG.info(getMemoryUsageStatsAsString(memoryMXBean));
boolean startMemoryUsageLogThread =
GlobalConfiguration.getBoolean(
ConfigConstants.TASK_MANAGER_DEBUG_MEMORY_USAGE_START_LOG_THREAD,
ConfigConstants.DEFAULT_TASK_MANAGER_DEBUG_MEMORY_USAGE_START_LOG_THREAD);
if (startMemoryUsageLogThread && LOG.isDebugEnabled()) {
final int logIntervalMs =
GlobalConfiguration.getInteger(
ConfigConstants.TASK_MANAGER_DEBUG_MEMORY_USAGE_LOG_INTERVAL_MS,
ConfigConstants.DEFAULT_TASK_MANAGER_DEBUG_MEMORY_USAGE_LOG_INTERVAL_MS);
new Thread(
new Runnable() {
@Override
public void run() {
try {
while (!isShutDown()) {
Thread.sleep(logIntervalMs);
LOG.debug(getMemoryUsageStatsAsString(memoryMXBean));
LOG.debug(getGarbageCollectorStatsAsString(gcMXBeans));
}
} catch (InterruptedException e) {
LOG.warn("Unexpected interruption of memory usage logger thread.");
}
}
})
.start();
}
}
/** Exports stats related to the JVM and runtime environment. */
public static void export() {
final OperatingSystemMXBean osMbean = ManagementFactory.getOperatingSystemMXBean();
if (osMbean instanceof com.sun.management.OperatingSystemMXBean) {
final com.sun.management.OperatingSystemMXBean sunOsMbean =
(com.sun.management.OperatingSystemMXBean) osMbean;
Stats.exportAll(
ImmutableList.<Stat<? extends Number>>builder()
.add(
new StatImpl<Long>("system_free_physical_memory_mb") {
@Override
public Long read() {
return sunOsMbean.getFreePhysicalMemorySize() / BYTES_PER_MB;
}
})
.add(
new StatImpl<Long>("system_free_swap_mb") {
@Override
public Long read() {
return sunOsMbean.getFreeSwapSpaceSize() / BYTES_PER_MB;
}
})
.add(
Rate.of(
new StatImpl<Long>("process_cpu_time_nanos") {
@Override
public Long read() {
return sunOsMbean.getProcessCpuTime();
}
})
.withName("process_cpu_cores_utilized")
.withScaleFactor(SECS_PER_NANO)
.build())
.build());
}
if (osMbean instanceof com.sun.management.UnixOperatingSystemMXBean) {
final com.sun.management.UnixOperatingSystemMXBean unixOsMbean =
(com.sun.management.UnixOperatingSystemMXBean) osMbean;
Stats.exportAll(
ImmutableList.<Stat<? extends Number>>builder()
.add(
new StatImpl<Long>("process_max_fd_count") {
@Override
public Long read() {
return unixOsMbean.getMaxFileDescriptorCount();
}
})
.add(
new StatImpl<Long>("process_open_fd_count") {
@Override
public Long read() {
return unixOsMbean.getOpenFileDescriptorCount();
}
})
.build());
}
final Runtime runtime = Runtime.getRuntime();
final ClassLoadingMXBean classLoadingBean = ManagementFactory.getClassLoadingMXBean();
MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean();
final MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage();
final MemoryUsage nonHeapUsage = memoryBean.getNonHeapMemoryUsage();
final ThreadMXBean threads = ManagementFactory.getThreadMXBean();
final RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
Stats.exportAll(
ImmutableList.<Stat<? extends Number>>builder()
.add(
new StatImpl<Long>("jvm_time_ms") {
@Override
public Long read() {
return System.currentTimeMillis();
}
})
.add(
new StatImpl<Integer>("jvm_available_processors") {
@Override
public Integer read() {
return runtime.availableProcessors();
}
})
.add(
new StatImpl<Long>("jvm_memory_free_mb") {
@Override
public Long read() {
return runtime.freeMemory() / BYTES_PER_MB;
}
})
.add(
new StatImpl<Long>("jvm_memory_max_mb") {
@Override
public Long read() {
return runtime.maxMemory() / BYTES_PER_MB;
}
})
.add(
new StatImpl<Long>("jvm_memory_mb_total") {
@Override
public Long read() {
return runtime.totalMemory() / BYTES_PER_MB;
}
})
.add(
new StatImpl<Integer>("jvm_class_loaded_count") {
@Override
public Integer read() {
return classLoadingBean.getLoadedClassCount();
}
})
.add(
new StatImpl<Long>("jvm_class_total_loaded_count") {
@Override
public Long read() {
return classLoadingBean.getTotalLoadedClassCount();
}
})
.add(
new StatImpl<Long>("jvm_class_unloaded_count") {
@Override
public Long read() {
return classLoadingBean.getUnloadedClassCount();
}
})
.add(
new StatImpl<Long>("jvm_gc_collection_time_ms") {
@Override
public Long read() {
long collectionTimeMs = 0;
for (GarbageCollectorMXBean bean :
ManagementFactory.getGarbageCollectorMXBeans()) {
collectionTimeMs += bean.getCollectionTime();
}
return collectionTimeMs;
}
})
.add(
new StatImpl<Long>("jvm_gc_collection_count") {
@Override
public Long read() {
long collections = 0;
for (GarbageCollectorMXBean bean :
ManagementFactory.getGarbageCollectorMXBeans()) {
collections += bean.getCollectionCount();
}
return collections;
}
})
.add(
new StatImpl<Long>("jvm_memory_heap_mb_used") {
@Override
public Long read() {
return heapUsage.getUsed() / BYTES_PER_MB;
}
})
.add(
new StatImpl<Long>("jvm_memory_heap_mb_committed") {
@Override
public Long read() {
return heapUsage.getCommitted() / BYTES_PER_MB;
}
})
.add(
new StatImpl<Long>("jvm_memory_heap_mb_max") {
@Override
public Long read() {
return heapUsage.getMax() / BYTES_PER_MB;
}
})
.add(
new StatImpl<Long>("jvm_memory_non_heap_mb_used") {
@Override
public Long read() {
return nonHeapUsage.getUsed() / BYTES_PER_MB;
}
})
.add(
new StatImpl<Long>("jvm_memory_non_heap_mb_committed") {
@Override
public Long read() {
return nonHeapUsage.getCommitted() / BYTES_PER_MB;
}
})
.add(
new StatImpl<Long>("jvm_memory_non_heap_mb_max") {
@Override
public Long read() {
return nonHeapUsage.getMax() / BYTES_PER_MB;
}
})
.add(
new StatImpl<Long>("jvm_uptime_secs") {
@Override
public Long read() {
return runtimeMXBean.getUptime() / 1000;
}
})
.add(
new StatImpl<Double>("system_load_avg") {
@Override
public Double read() {
return osMbean.getSystemLoadAverage();
}
})
.add(
new StatImpl<Integer>("jvm_threads_peak") {
@Override
public Integer read() {
return threads.getPeakThreadCount();
}
})
.add(
new StatImpl<Long>("jvm_threads_started") {
@Override
public Long read() {
return threads.getTotalStartedThreadCount();
}
})
.add(
new StatImpl<Integer>("jvm_threads_daemon") {
@Override
public Integer read() {
return threads.getDaemonThreadCount();
}
})
.add(
new StatImpl<Integer>("jvm_threads_active") {
@Override
public Integer read() {
return threads.getThreadCount();
}
})
.build());
// Export per memory pool gc time and cycle count like Ostrich
// This is based on code in Bridcage: https://cgit.twitter.biz/birdcage/tree/ \
// ostrich/src/main/scala/com/twitter/ostrich/stats/StatsCollection.scala
Stats.exportAll(
Iterables.transform(
ManagementFactory.getGarbageCollectorMXBeans(),
new Function<GarbageCollectorMXBean, Stat<? extends Number>>() {
@Override
public Stat<? extends Number> apply(final GarbageCollectorMXBean gcMXBean) {
return new StatImpl<Long>(
"jvm_gc_" + Stats.normalizeName(gcMXBean.getName()) + "_collection_count") {
@Override
public Long read() {
return gcMXBean.getCollectionCount();
}
};
}
}));
Stats.exportAll(
Iterables.transform(
ManagementFactory.getGarbageCollectorMXBeans(),
new Function<GarbageCollectorMXBean, Stat<? extends Number>>() {
@Override
public Stat<? extends Number> apply(final GarbageCollectorMXBean gcMXBean) {
return new StatImpl<Long>(
"jvm_gc_" + Stats.normalizeName(gcMXBean.getName()) + "_collection_time_ms") {
@Override
public Long read() {
return gcMXBean.getCollectionTime();
}
};
}
}));
Stats.exportString(
new StatImpl<String>("jvm_input_arguments") {
@Override
public String read() {
return runtimeMXBean.getInputArguments().toString();
}
});
for (final String property : System.getProperties().stringPropertyNames()) {
Stats.exportString(
new StatImpl<String>("jvm_prop_" + Stats.normalizeName(property)) {
@Override
public String read() {
return System.getProperty(property);
}
});
}
}
/**
* Builds an XML report of the timings to be uploaded for parsing.
*
* @param sender Who to report to
*/
static void reportTimings(CommandSource sender) {
Platform platform = SpongeImpl.getGame().getPlatform();
JsonObjectBuilder builder =
JSONUtil.objectBuilder()
// Get some basic system details about the server
.add(
"version",
platform
.getImplementation()
.getVersion()
.orElse(platform.getMinecraftVersion().getName() + "-DEV"))
.add("maxplayers", SpongeImpl.getGame().getServer().getMaxPlayers())
.add("start", TimingsManager.timingStart / 1000)
.add("end", System.currentTimeMillis() / 1000)
.add("sampletime", (System.currentTimeMillis() - TimingsManager.timingStart) / 1000);
if (!TimingsManager.privacy) {
builder
.add("server", getServerName())
.add("motd", SpongeImpl.getGame().getServer().getMotd().toPlain())
.add("online-mode", SpongeImpl.getGame().getServer().getOnlineMode())
.add("icon", MinecraftServer.getServer().getServerStatusResponse().getFavicon());
}
final Runtime runtime = Runtime.getRuntime();
RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean();
builder.add(
"system",
JSONUtil.objectBuilder()
.add("timingcost", getCost())
.add("name", System.getProperty("os.name"))
.add("version", System.getProperty("os.version"))
.add("jvmversion", System.getProperty("java.version"))
.add("arch", System.getProperty("os.arch"))
.add("maxmem", runtime.maxMemory())
.add("cpu", runtime.availableProcessors())
.add("runtime", ManagementFactory.getRuntimeMXBean().getUptime())
.add("flags", RUNTIME_FLAG_JOINER.join(runtimeBean.getInputArguments()))
.add(
"gc",
JSONUtil.mapArrayToObject(
ManagementFactory.getGarbageCollectorMXBeans(),
(input) -> {
return JSONUtil.singleObjectPair(
input.getName(),
JSONUtil.arrayOf(input.getCollectionCount(), input.getCollectionTime()));
})));
Set<BlockType> blockTypeSet = Sets.newHashSet();
Set<EntityType> entityTypeSet = Sets.newHashSet();
int size = HISTORY.size();
TimingHistory[] history = new TimingHistory[size + 1];
int i = 0;
for (TimingHistory timingHistory : HISTORY) {
blockTypeSet.addAll(timingHistory.blockTypeSet);
entityTypeSet.addAll(timingHistory.entityTypeSet);
history[i++] = timingHistory;
}
history[i] = new TimingHistory(); // Current snapshot
blockTypeSet.addAll(history[i].blockTypeSet);
entityTypeSet.addAll(history[i].entityTypeSet);
JsonObjectBuilder handlersBuilder = JSONUtil.objectBuilder();
for (TimingIdentifier.TimingGroup group : TimingIdentifier.GROUP_MAP.values()) {
for (TimingHandler id : group.handlers) {
if (!id.timed && !id.isSpecial()) {
continue;
}
handlersBuilder.add(id.id, JSONUtil.arrayOf(group.id, id.name));
}
}
builder.add(
"idmap",
JSONUtil.objectBuilder()
.add(
"groups",
JSONUtil.mapArrayToObject(
TimingIdentifier.GROUP_MAP.values(),
(group) -> {
return JSONUtil.singleObjectPair(group.id, group.name);
}))
.add("handlers", handlersBuilder)
.add(
"worlds",
JSONUtil.mapArrayToObject(
TimingHistory.worldMap.entrySet(),
(entry) -> {
return JSONUtil.singleObjectPair(entry.getValue(), entry.getKey());
}))
.add(
"tileentity",
JSONUtil.mapArrayToObject(
blockTypeSet,
(blockType) -> {
return JSONUtil.singleObjectPair(
Block.getIdFromBlock((Block) blockType), blockType.getId());
}))
.add(
"entity",
JSONUtil.mapArrayToObject(
entityTypeSet,
(entityType) -> {
return JSONUtil.singleObjectPair(
((SpongeEntityType) entityType).entityTypeId, entityType.getId());
})));
// Information about loaded plugins
builder.add(
"plugins",
JSONUtil.mapArrayToObject(
SpongeImpl.getGame().getPluginManager().getPlugins(),
(plugin) -> {
return JSONUtil.objectBuilder()
.add(
plugin.getId(),
JSONUtil.objectBuilder()
.add("version", plugin.getVersion().orElse(""))
.add("description", plugin.getDescription().orElse(""))
.add("website", plugin.getUrl().orElse(""))
.add("authors", AUTHOR_LIST_JOINER.join(plugin.getAuthors())))
.build();
}));
// Information on the users Config
builder.add(
"config",
JSONUtil.objectBuilder()
.add("sponge", serializeConfigNode(SpongeImpl.getGlobalConfig().getRootNode())));
new TimingsExport(sender, builder.build(), history).start();
}
static {
garbageCollectorMXBeans = ManagementFactory.getGarbageCollectorMXBeans();
counter = Long.MIN_VALUE;
times = new HashMap<Long, Pair<Long, Long>>();
}
static {
RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
// returns the <process id>@<host>
long pid;
String xPid = runtimeMXBean.getName();
try {
xPid = xPid.split("@")[0];
pid = Long.parseLong(xPid);
} catch (Exception e) {
pid = -1;
}
JvmInfo info = new JvmInfo();
info.pid = pid;
info.startTime = runtimeMXBean.getStartTime();
info.version = System.getProperty("java.version");
info.vmName = runtimeMXBean.getVmName();
info.vmVendor = runtimeMXBean.getVmVendor();
info.vmVersion = runtimeMXBean.getVmVersion();
info.mem = new Mem();
info.mem.heapInit =
memoryMXBean.getHeapMemoryUsage().getInit() < 0
? 0
: memoryMXBean.getHeapMemoryUsage().getInit();
info.mem.heapMax =
memoryMXBean.getHeapMemoryUsage().getMax() < 0
? 0
: memoryMXBean.getHeapMemoryUsage().getMax();
info.mem.nonHeapInit =
memoryMXBean.getNonHeapMemoryUsage().getInit() < 0
? 0
: memoryMXBean.getNonHeapMemoryUsage().getInit();
info.mem.nonHeapMax =
memoryMXBean.getNonHeapMemoryUsage().getMax() < 0
? 0
: memoryMXBean.getNonHeapMemoryUsage().getMax();
try {
Class<?> vmClass = Class.forName("sun.misc.VM");
info.mem.directMemoryMax = (Long) vmClass.getMethod("maxDirectMemory").invoke(null);
} catch (Exception t) {
// ignore
}
info.inputArguments =
runtimeMXBean
.getInputArguments()
.toArray(new String[runtimeMXBean.getInputArguments().size()]);
try {
info.bootClassPath = runtimeMXBean.getBootClassPath();
} catch (UnsupportedOperationException e) {
// oracle java 9
info.bootClassPath = System.getProperty("sun.boot.class.path");
if (info.bootClassPath == null) {
// something else
info.bootClassPath = "<unknown>";
}
}
info.classPath = runtimeMXBean.getClassPath();
info.systemProperties = Collections.unmodifiableMap(runtimeMXBean.getSystemProperties());
List<GarbageCollectorMXBean> gcMxBeans = ManagementFactory.getGarbageCollectorMXBeans();
info.gcCollectors = new String[gcMxBeans.size()];
for (int i = 0; i < gcMxBeans.size(); i++) {
GarbageCollectorMXBean gcMxBean = gcMxBeans.get(i);
info.gcCollectors[i] = gcMxBean.getName();
}
List<MemoryPoolMXBean> memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans();
info.memoryPools = new String[memoryPoolMXBeans.size()];
for (int i = 0; i < memoryPoolMXBeans.size(); i++) {
MemoryPoolMXBean memoryPoolMXBean = memoryPoolMXBeans.get(i);
info.memoryPools[i] = memoryPoolMXBean.getName();
}
try {
@SuppressWarnings("unchecked")
Class<? extends PlatformManagedObject> clazz =
(Class<? extends PlatformManagedObject>)
Class.forName("com.sun.management.HotSpotDiagnosticMXBean");
Class<?> vmOptionClazz = Class.forName("com.sun.management.VMOption");
PlatformManagedObject hotSpotDiagnosticMXBean = ManagementFactory.getPlatformMXBean(clazz);
Method vmOptionMethod = clazz.getMethod("getVMOption", String.class);
Method valueMethod = vmOptionClazz.getMethod("getValue");
try {
Object onError = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnError");
info.onError = (String) valueMethod.invoke(onError);
} catch (Exception ignored) {
}
try {
Object onOutOfMemoryError =
vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnOutOfMemoryError");
info.onOutOfMemoryError = (String) valueMethod.invoke(onOutOfMemoryError);
} catch (Exception ignored) {
}
try {
Object useCompressedOopsVmOption =
vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseCompressedOops");
info.useCompressedOops = (String) valueMethod.invoke(useCompressedOopsVmOption);
} catch (Exception ignored) {
}
try {
Object useG1GCVmOption = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseG1GC");
info.useG1GC = (String) valueMethod.invoke(useG1GCVmOption);
} catch (Exception ignored) {
}
try {
Object initialHeapSizeVmOption =
vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "InitialHeapSize");
info.configuredInitialHeapSize =
Long.parseLong((String) valueMethod.invoke(initialHeapSizeVmOption));
} catch (Exception ignored) {
}
try {
Object maxHeapSizeVmOption = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "MaxHeapSize");
info.configuredMaxHeapSize =
Long.parseLong((String) valueMethod.invoke(maxHeapSizeVmOption));
} catch (Exception ignored) {
}
} catch (Exception ignored) {
}
INSTANCE = info;
}
public class Diagnostics {
private static final String PACKAGE = "org.apache.tomcat.util";
private static final String INDENT1 = " ";
private static final String INDENT2 = "\t";
private static final String INDENT3 = " ";
private static final String CRLF = "\r\n";
private static final String vminfoSystemProperty = "java.vm.info";
private static final SimpleDateFormat timeformat =
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
/* Some platform MBeans */
private static final ClassLoadingMXBean classLoadingMXBean =
ManagementFactory.getClassLoadingMXBean();
private static final CompilationMXBean compilationMXBean =
ManagementFactory.getCompilationMXBean();
private static final OperatingSystemMXBean operatingSystemMXBean =
ManagementFactory.getOperatingSystemMXBean();
private static final RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
private static final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
// XXX Not sure whether the following MBeans should better
// be retrieved on demand, i.e. whether they can change
// dynamically in the MBeanServer.
private static final LoggingMXBean loggingMXBean = LogManager.getLoggingMXBean();
private static final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
private static final List<GarbageCollectorMXBean> garbageCollectorMXBeans =
ManagementFactory.getGarbageCollectorMXBeans();
private static final List<MemoryManagerMXBean> memoryManagerMXBeans =
ManagementFactory.getMemoryManagerMXBeans();
private static final List<MemoryPoolMXBean> memoryPoolMXBeans =
ManagementFactory.getMemoryPoolMXBeans();
/**
* Formats the thread dump header for one thread.
*
* @param ti the ThreadInfo describing the thread
* @return the formatted thread dump header
*/
private static String getThreadDumpHeader(ThreadInfo ti) {
StringBuilder sb = new StringBuilder("\"" + ti.getThreadName() + "\"");
sb.append(" Id=" + ti.getThreadId());
sb.append(" cpu=" + threadMXBean.getThreadCpuTime(ti.getThreadId()) + " ns");
sb.append(" usr="******" ns");
sb.append(" blocked " + ti.getBlockedCount() + " for " + ti.getBlockedTime() + " ms");
sb.append(" waited " + ti.getWaitedCount() + " for " + ti.getWaitedTime() + " ms");
if (ti.isSuspended()) {
sb.append(" (suspended)");
}
if (ti.isInNative()) {
sb.append(" (running in native)");
}
sb.append(CRLF);
sb.append(INDENT3 + "java.lang.Thread.State: " + ti.getThreadState());
sb.append(CRLF);
return sb.toString();
}
/**
* Formats the thread dump for one thread.
*
* @param ti the ThreadInfo describing the thread
* @return the formatted thread dump
*/
private static String getThreadDump(ThreadInfo ti) {
StringBuilder sb = new StringBuilder(getThreadDumpHeader(ti));
for (LockInfo li : ti.getLockedSynchronizers()) {
sb.append(INDENT2 + "locks " + li.toString() + CRLF);
}
boolean start = true;
StackTraceElement[] stes = ti.getStackTrace();
Object[] monitorDepths = new Object[stes.length];
MonitorInfo[] mis = ti.getLockedMonitors();
for (int i = 0; i < mis.length; i++) {
monitorDepths[mis[i].getLockedStackDepth()] = mis[i];
}
for (int i = 0; i < stes.length; i++) {
StackTraceElement ste = stes[i];
sb.append(INDENT2 + "at " + ste.toString() + CRLF);
if (start) {
if (ti.getLockName() != null) {
sb.append(INDENT2 + "- waiting on (a " + ti.getLockName() + ")");
if (ti.getLockOwnerName() != null) {
sb.append(" owned by " + ti.getLockOwnerName() + " Id=" + ti.getLockOwnerId());
}
sb.append(CRLF);
}
start = false;
}
if (monitorDepths[i] != null) {
MonitorInfo mi = (MonitorInfo) monitorDepths[i];
sb.append(
INDENT2
+ "- locked (a "
+ mi.toString()
+ ")"
+ " index "
+ mi.getLockedStackDepth()
+ " frame "
+ mi.getLockedStackFrame().toString());
sb.append(CRLF);
}
}
return sb.toString();
}
/**
* Formats the thread dump for a list of threads.
*
* @param tinfos the ThreadInfo array describing the thread list
* @return the formatted thread dump
*/
private static String getThreadDump(ThreadInfo[] tinfos) {
StringBuilder sb = new StringBuilder();
for (ThreadInfo tinfo : tinfos) {
sb.append(getThreadDump(tinfo));
sb.append(CRLF);
}
return sb.toString();
}
/**
* Check if any threads are deadlocked. If any, print the thread dump for those threads.
*
* @return a deadlock message and the formatted thread dump of the deadlocked threads
*/
private static String findDeadlock() {
ThreadInfo[] tinfos = null;
long[] ids = threadMXBean.findDeadlockedThreads();
if (ids != null) {
tinfos = threadMXBean.getThreadInfo(threadMXBean.findDeadlockedThreads(), true, true);
if (tinfos != null) {
StringBuilder sb = new StringBuilder("Deadlock found between the following threads:");
sb.append(CRLF);
sb.append(getThreadDump(tinfos));
return sb.toString();
}
}
return "";
}
/**
* Retrieves a formatted JVM thread dump. The given list of locales will be used to retrieve a
* StringManager.
*
* @param requestedLocales list of locales to use
* @return the formatted JVM thread dump
*/
public static String getThreadDump(Enumeration<Locale> requestedLocales) {
return getThreadDump(StringManager.getManager(PACKAGE, requestedLocales));
}
/**
* Retrieve a JVM thread dump formatted using the given StringManager.
*
* @param requestedSm the StringManager to use
* @return the formatted JVM thread dump
*/
private static String getThreadDump(StringManager requestedSm) {
StringBuilder sb = new StringBuilder();
synchronized (timeformat) {
sb.append(timeformat.format(new Date()));
}
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.threadDumpTitle"));
sb.append(" ");
sb.append(runtimeMXBean.getVmName());
sb.append(" (");
sb.append(runtimeMXBean.getVmVersion());
String vminfo = System.getProperty(vminfoSystemProperty);
if (vminfo != null) {
sb.append(" " + vminfo);
}
sb.append("):" + CRLF);
sb.append(CRLF);
ThreadInfo[] tis = threadMXBean.dumpAllThreads(true, true);
sb.append(getThreadDump(tis));
sb.append(findDeadlock());
return sb.toString();
}
/**
* Format contents of a MemoryUsage object.
*
* @param name a text prefix used in formatting
* @param usage the MemoryUsage object to format
* @return the formatted contents
*/
private static String formatMemoryUsage(String name, MemoryUsage usage) {
if (usage != null) {
StringBuilder sb = new StringBuilder();
sb.append(INDENT1 + name + " init: " + usage.getInit() + CRLF);
sb.append(INDENT1 + name + " used: " + usage.getUsed() + CRLF);
sb.append(INDENT1 + name + " committed: " + usage.getCommitted() + CRLF);
sb.append(INDENT1 + name + " max: " + usage.getMax() + CRLF);
return sb.toString();
}
return "";
}
/**
* Retrieves a formatted JVM information text. The given list of locales will be used to retrieve
* a StringManager.
*
* @param requestedLocales list of locales to use
* @return the formatted JVM information text
*/
public static String getVMInfo(Enumeration<Locale> requestedLocales) {
return getVMInfo(StringManager.getManager(PACKAGE, requestedLocales));
}
/**
* Retrieve a JVM information text formatted using the given StringManager.
*
* @param requestedSm the StringManager to use
* @return the formatted JVM information text
*/
private static String getVMInfo(StringManager requestedSm) {
StringBuilder sb = new StringBuilder();
synchronized (timeformat) {
sb.append(timeformat.format(new Date()));
}
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoRuntime"));
sb.append(":" + CRLF);
sb.append(INDENT1 + "vmName: " + runtimeMXBean.getVmName() + CRLF);
sb.append(INDENT1 + "vmVersion: " + runtimeMXBean.getVmVersion() + CRLF);
sb.append(INDENT1 + "vmVendor: " + runtimeMXBean.getVmVendor() + CRLF);
sb.append(INDENT1 + "specName: " + runtimeMXBean.getSpecName() + CRLF);
sb.append(INDENT1 + "specVersion: " + runtimeMXBean.getSpecVersion() + CRLF);
sb.append(INDENT1 + "specVendor: " + runtimeMXBean.getSpecVendor() + CRLF);
sb.append(
INDENT1 + "managementSpecVersion: " + runtimeMXBean.getManagementSpecVersion() + CRLF);
sb.append(INDENT1 + "name: " + runtimeMXBean.getName() + CRLF);
sb.append(INDENT1 + "startTime: " + runtimeMXBean.getStartTime() + CRLF);
sb.append(INDENT1 + "uptime: " + runtimeMXBean.getUptime() + CRLF);
sb.append(
INDENT1 + "isBootClassPathSupported: " + runtimeMXBean.isBootClassPathSupported() + CRLF);
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoOs"));
sb.append(":" + CRLF);
sb.append(INDENT1 + "name: " + operatingSystemMXBean.getName() + CRLF);
sb.append(INDENT1 + "version: " + operatingSystemMXBean.getVersion() + CRLF);
sb.append(INDENT1 + "architecture: " + operatingSystemMXBean.getArch() + CRLF);
sb.append(
INDENT1 + "availableProcessors: " + operatingSystemMXBean.getAvailableProcessors() + CRLF);
sb.append(
INDENT1 + "systemLoadAverage: " + operatingSystemMXBean.getSystemLoadAverage() + CRLF);
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoThreadMxBean"));
sb.append(":" + CRLF);
sb.append(
INDENT1
+ "isCurrentThreadCpuTimeSupported: "
+ threadMXBean.isCurrentThreadCpuTimeSupported()
+ CRLF);
sb.append(
INDENT1 + "isThreadCpuTimeSupported: " + threadMXBean.isThreadCpuTimeSupported() + CRLF);
sb.append(INDENT1 + "isThreadCpuTimeEnabled: " + threadMXBean.isThreadCpuTimeEnabled() + CRLF);
sb.append(
INDENT1
+ "isObjectMonitorUsageSupported: "
+ threadMXBean.isObjectMonitorUsageSupported()
+ CRLF);
sb.append(
INDENT1
+ "isSynchronizerUsageSupported: "
+ threadMXBean.isSynchronizerUsageSupported()
+ CRLF);
sb.append(
INDENT1
+ "isThreadContentionMonitoringSupported: "
+ threadMXBean.isThreadContentionMonitoringSupported()
+ CRLF);
sb.append(
INDENT1
+ "isThreadContentionMonitoringEnabled: "
+ threadMXBean.isThreadContentionMonitoringEnabled()
+ CRLF);
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoThreadCounts"));
sb.append(":" + CRLF);
sb.append(INDENT1 + "daemon: " + threadMXBean.getDaemonThreadCount() + CRLF);
sb.append(INDENT1 + "total: " + threadMXBean.getThreadCount() + CRLF);
sb.append(INDENT1 + "peak: " + threadMXBean.getPeakThreadCount() + CRLF);
sb.append(INDENT1 + "totalStarted: " + threadMXBean.getTotalStartedThreadCount() + CRLF);
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoStartup"));
sb.append(":" + CRLF);
for (String arg : runtimeMXBean.getInputArguments()) {
sb.append(INDENT1 + arg + CRLF);
}
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoPath"));
sb.append(":" + CRLF);
sb.append(INDENT1 + "bootClassPath: " + runtimeMXBean.getBootClassPath() + CRLF);
sb.append(INDENT1 + "classPath: " + runtimeMXBean.getClassPath() + CRLF);
sb.append(INDENT1 + "libraryPath: " + runtimeMXBean.getLibraryPath() + CRLF);
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoClassLoading"));
sb.append(":" + CRLF);
sb.append(INDENT1 + "loaded: " + classLoadingMXBean.getLoadedClassCount() + CRLF);
sb.append(INDENT1 + "unloaded: " + classLoadingMXBean.getUnloadedClassCount() + CRLF);
sb.append(INDENT1 + "totalLoaded: " + classLoadingMXBean.getTotalLoadedClassCount() + CRLF);
sb.append(INDENT1 + "isVerbose: " + classLoadingMXBean.isVerbose() + CRLF);
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoClassCompilation"));
sb.append(":" + CRLF);
sb.append(INDENT1 + "name: " + compilationMXBean.getName() + CRLF);
sb.append(
INDENT1 + "totalCompilationTime: " + compilationMXBean.getTotalCompilationTime() + CRLF);
sb.append(
INDENT1
+ "isCompilationTimeMonitoringSupported: "
+ compilationMXBean.isCompilationTimeMonitoringSupported()
+ CRLF);
sb.append(CRLF);
for (MemoryManagerMXBean mbean : memoryManagerMXBeans) {
sb.append(requestedSm.getString("diagnostics.vmInfoMemoryManagers", mbean.getName()));
sb.append(":" + CRLF);
sb.append(INDENT1 + "isValid: " + mbean.isValid() + CRLF);
sb.append(INDENT1 + "mbean.getMemoryPoolNames: " + CRLF);
String[] names = mbean.getMemoryPoolNames();
Arrays.sort(names);
for (String name : names) {
sb.append(INDENT2 + name + CRLF);
}
sb.append(CRLF);
}
for (GarbageCollectorMXBean mbean : garbageCollectorMXBeans) {
sb.append(requestedSm.getString("diagnostics.vmInfoGarbageCollectors", mbean.getName()));
sb.append(":" + CRLF);
sb.append(INDENT1 + "isValid: " + mbean.isValid() + CRLF);
sb.append(INDENT1 + "mbean.getMemoryPoolNames: " + CRLF);
String[] names = mbean.getMemoryPoolNames();
Arrays.sort(names);
for (String name : names) {
sb.append(INDENT2 + name + CRLF);
}
sb.append(INDENT1 + "getCollectionCount: " + mbean.getCollectionCount() + CRLF);
sb.append(INDENT1 + "getCollectionTime: " + mbean.getCollectionTime() + CRLF);
sb.append(CRLF);
}
sb.append(requestedSm.getString("diagnostics.vmInfoMemory"));
sb.append(":" + CRLF);
sb.append(INDENT1 + "isVerbose: " + memoryMXBean.isVerbose() + CRLF);
sb.append(
INDENT1
+ "getObjectPendingFinalizationCount: "
+ memoryMXBean.getObjectPendingFinalizationCount()
+ CRLF);
sb.append(formatMemoryUsage("heap", memoryMXBean.getHeapMemoryUsage()));
sb.append(formatMemoryUsage("non-heap", memoryMXBean.getNonHeapMemoryUsage()));
sb.append(CRLF);
for (MemoryPoolMXBean mbean : memoryPoolMXBeans) {
sb.append(requestedSm.getString("diagnostics.vmInfoMemoryPools", mbean.getName()));
sb.append(":" + CRLF);
sb.append(INDENT1 + "isValid: " + mbean.isValid() + CRLF);
sb.append(INDENT1 + "getType: " + mbean.getType() + CRLF);
sb.append(INDENT1 + "mbean.getMemoryManagerNames: " + CRLF);
String[] names = mbean.getMemoryManagerNames();
Arrays.sort(names);
for (String name : names) {
sb.append(INDENT2 + name + CRLF);
}
sb.append(INDENT1 + "isUsageThresholdSupported: " + mbean.isUsageThresholdSupported() + CRLF);
try {
sb.append(INDENT1 + "isUsageThresholdExceeded: " + mbean.isUsageThresholdExceeded() + CRLF);
} catch (UnsupportedOperationException ex) {
// IGNORE
}
sb.append(
INDENT1
+ "isCollectionUsageThresholdSupported: "
+ mbean.isCollectionUsageThresholdSupported()
+ CRLF);
try {
sb.append(
INDENT1
+ "isCollectionUsageThresholdExceeded: "
+ mbean.isCollectionUsageThresholdExceeded()
+ CRLF);
} catch (UnsupportedOperationException ex) {
// IGNORE
}
try {
sb.append(INDENT1 + "getUsageThreshold: " + mbean.getUsageThreshold() + CRLF);
} catch (UnsupportedOperationException ex) {
// IGNORE
}
try {
sb.append(INDENT1 + "getUsageThresholdCount: " + mbean.getUsageThresholdCount() + CRLF);
} catch (UnsupportedOperationException ex) {
// IGNORE
}
try {
sb.append(
INDENT1 + "getCollectionUsageThreshold: " + mbean.getCollectionUsageThreshold() + CRLF);
} catch (UnsupportedOperationException ex) {
// IGNORE
}
try {
sb.append(
INDENT1
+ "getCollectionUsageThresholdCount: "
+ mbean.getCollectionUsageThresholdCount()
+ CRLF);
} catch (UnsupportedOperationException ex) {
// IGNORE
}
sb.append(formatMemoryUsage("current", mbean.getUsage()));
sb.append(formatMemoryUsage("collection", mbean.getCollectionUsage()));
sb.append(formatMemoryUsage("peak", mbean.getPeakUsage()));
sb.append(CRLF);
}
sb.append(requestedSm.getString("diagnostics.vmInfoSystem"));
sb.append(":" + CRLF);
Map<String, String> props = runtimeMXBean.getSystemProperties();
ArrayList<String> keys = new ArrayList<String>(props.keySet());
Collections.sort(keys);
for (String prop : keys) {
sb.append(INDENT1 + prop + ": " + props.get(prop) + CRLF);
}
sb.append(CRLF);
sb.append(requestedSm.getString("diagnostics.vmInfoLogger"));
sb.append(":" + CRLF);
List<String> loggers = loggingMXBean.getLoggerNames();
Collections.sort(loggers);
for (String logger : loggers) {
sb.append(
INDENT1
+ logger
+ ": level="
+ loggingMXBean.getLoggerLevel(logger)
+ ", parent="
+ loggingMXBean.getParentLoggerName(logger)
+ CRLF);
}
sb.append(CRLF);
return sb.toString();
}
}
public synchronized void logGCInfo(AccumuloConfiguration conf) {
final long now = System.currentTimeMillis();
List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
Runtime rt = Runtime.getRuntime();
StringBuilder sb = new StringBuilder("gc");
boolean sawChange = false;
long maxIncreaseInCollectionTime = 0;
for (GarbageCollectorMXBean gcBean : gcmBeans) {
Long prevTime = prevGcTime.get(gcBean.getName());
long pt = 0;
if (prevTime != null) {
pt = prevTime;
}
long time = gcBean.getCollectionTime();
if (time - pt != 0) {
sawChange = true;
}
long increaseInCollectionTime = time - pt;
sb.append(
String.format(
" %s=%,.2f(+%,.2f) secs",
gcBean.getName(), time / 1000.0, increaseInCollectionTime / 1000.0));
maxIncreaseInCollectionTime = Math.max(increaseInCollectionTime, maxIncreaseInCollectionTime);
prevGcTime.put(gcBean.getName(), time);
}
long mem = rt.freeMemory();
if (maxIncreaseInCollectionTime == 0) {
gcTimeIncreasedCount = 0;
} else {
gcTimeIncreasedCount++;
if (gcTimeIncreasedCount > 3 && mem < rt.maxMemory() * 0.05) {
log.warn("Running low on memory");
gcTimeIncreasedCount = 0;
}
}
if (mem > lastMemorySize) {
sawChange = true;
}
String sign = "+";
if (mem - lastMemorySize <= 0) {
sign = "";
}
sb.append(
String.format(
" freemem=%,d(%s%,d) totalmem=%,d",
mem, sign, (mem - lastMemorySize), rt.totalMemory()));
if (sawChange) {
log.debug(sb.toString());
}
final long keepAliveTimeout = conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
if (lastMemoryCheckTime > 0 && lastMemoryCheckTime < now) {
final long diff = now - lastMemoryCheckTime;
if (diff > keepAliveTimeout) {
log.warn(
String.format(
"GC pause checker not called in a timely fashion. Expected every %.1f seconds but was %.1f seconds since last check",
keepAliveTimeout / 1000., diff / 1000.));
}
lastMemoryCheckTime = now;
return;
}
if (maxIncreaseInCollectionTime > keepAliveTimeout) {
Halt.halt("Garbage collection may be interfering with lock keep-alive. Halting.", -1);
}
lastMemorySize = mem;
lastMemoryCheckTime = now;
}
private void logStats() {
long finishTime = System.currentTimeMillis();
long finalGCTimes = 0;
List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
for (GarbageCollectorMXBean garbageCollectorMXBean : gcmBeans) {
finalGCTimes += garbageCollectorMXBean.getCollectionTime();
}
CompilationMXBean compMxBean = ManagementFactory.getCompilationMXBean();
long finalCompileTimes = 0;
if (compMxBean.isCompilationTimeMonitoringSupported()) {
finalCompileTimes = compMxBean.getTotalCompilationTime();
}
double averageRate = totalSent.get() / (totalSendTime.get() / 1000.0);
double overallRate = totalAdded / ((finishTime - startTime) / 1000.0);
double finalSystemLoad = ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();
if (log.isTraceEnabled()) {
log.trace("");
log.trace("TABLET SERVER BATCH WRITER STATISTICS");
log.trace(String.format("Added : %,10d mutations", totalAdded));
log.trace(String.format("Sent : %,10d mutations", totalSent.get()));
log.trace(
String.format(
"Resent percentage : %10.2f%s",
(totalSent.get() - totalAdded) / (double) totalAdded * 100.0, "%"));
log.trace(
String.format("Overall time : %,10.2f secs", (finishTime - startTime) / 1000.0));
log.trace(String.format("Overall send rate : %,10.2f mutations/sec", overallRate));
log.trace(
String.format("Send efficiency : %10.2f%s", overallRate / averageRate * 100.0, "%"));
log.trace("");
log.trace("BACKGROUND WRITER PROCESS STATISTICS");
log.trace(
String.format(
"Total send time : %,10.2f secs %6.2f%s",
totalSendTime.get() / 1000.0,
100.0 * totalSendTime.get() / (finishTime - startTime),
"%"));
log.trace(String.format("Average send rate : %,10.2f mutations/sec", averageRate));
log.trace(
String.format(
"Total bin time : %,10.2f secs %6.2f%s",
totalBinTime.get() / 1000.0,
100.0 * totalBinTime.get() / (finishTime - startTime),
"%"));
log.trace(
String.format(
"Average bin rate : %,10.2f mutations/sec",
totalBinned.get() / (totalBinTime.get() / 1000.0)));
log.trace(
String.format(
"tservers per batch : %,8.2f avg %,6d min %,6d max",
(tabletServersBatchSum / (double) numBatches),
minTabletServersBatch,
maxTabletServersBatch));
log.trace(
String.format(
"tablets per batch : %,8.2f avg %,6d min %,6d max",
(tabletBatchSum / (double) numBatches), minTabletBatch, maxTabletBatch));
log.trace("");
log.trace("SYSTEM STATISTICS");
log.trace(
String.format(
"JVM GC Time : %,10.2f secs", ((finalGCTimes - initialGCTimes) / 1000.0)));
if (compMxBean.isCompilationTimeMonitoringSupported()) {
log.trace(
String.format(
"JVM Compile Time : %,10.2f secs",
((finalCompileTimes - initialCompileTimes) / 1000.0)));
}
log.trace(
String.format(
"System load average : initial=%6.2f final=%6.2f",
initialSystemLoad, finalSystemLoad));
}
}