Beispiel #1
0
    public static String getGCStats() {
        long totalGC = 0;
        long gcTime = 0;

        for (GarbageCollectorMXBean gc : ManagementFactory.getGarbageCollectorMXBeans()) {
            long count = gc.getCollectionCount();

            if (count >= 0) {
                totalGC += count;
            }

            long time = gc.getCollectionTime();

            if (time >= 0) {
                gcTime += time;
            }
        }

        StringBuilder sb = new StringBuilder();

        sb.append(banner("memory stats"));
        sb.append("\n- total collections: " + totalGC);
        sb.append("\n- total collection time: " + formatTime(gcTime));

        Runtime runtime = Runtime.getRuntime();
        sb.append("\n- total memory: " + _.printMem(runtime.totalMemory()));

        return sb.toString();
    }
Beispiel #2
0
  public static void main(String args[]) {

    MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
    MemoryUsage heapMemoryUsage = memoryMXBean.getHeapMemoryUsage();

    long init = heapMemoryUsage.getInit();
    long used = heapMemoryUsage.getUsed();
    long max = heapMemoryUsage.getMax();
    long committed = heapMemoryUsage.getCommitted();
    long free = max - used;
    int usedPercent = (int) (used / max);
    int freePercent = (int) (usedPercent / max);

    MemoryUsage nonHeapMemoryUsage = memoryMXBean.getNonHeapMemoryUsage();

    List<GarbageCollectorMXBean> garbageCollectorMXBeans =
        ManagementFactory.getGarbageCollectorMXBeans();

    List<MemoryPoolMXBean> memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans();

    com.sun.management.OperatingSystemMXBean operatingSystemMXBean =
        (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
    long commitVirtual = operatingSystemMXBean.getCommittedVirtualMemorySize();

    println(init, used, max, committed);
    println(commitVirtual);
  }
 /**
  * Returns the approximate accumulated garbage collection elapsed time in milliseconds.
  *
  * @return the approximate accumulated garbage collection elapsed time in milliseconds.
  */
 public Number getTime() {
   List<GarbageCollectorMXBean> garbageCollectors =
       ManagementFactory.getGarbageCollectorMXBeans();
   long sum = 0;
   for (GarbageCollectorMXBean garbageCollector : garbageCollectors) {
     sum += garbageCollector.getCollectionTime();
   }
   return Long.valueOf(sum);
 }
Beispiel #4
0
 private void doGarbageCollectionUpdates() {
   List<GarbageCollectorMXBean> gcBeans = ManagementFactory.getGarbageCollectorMXBeans();
   long count = 0;
   long timeMillis = 0;
   for (GarbageCollectorMXBean gcBean : gcBeans) {
     count += gcBean.getCollectionCount();
     timeMillis += gcBean.getCollectionTime();
   }
   metrics.incrMetric("gcCount", (int) (count - gcCount));
   metrics.incrMetric("gcTimeMillis", (int) (timeMillis - gcTimeMillis));
   gcCount = count;
   gcTimeMillis = timeMillis;
 }
Beispiel #5
0
  public static void main(String[] args) {
    List<GarbageCollectorMXBean> list = ManagementFactory.getGarbageCollectorMXBeans();
    for (GarbageCollectorMXBean bean : list) {
      System.out.println("Name: " + bean.getName());
      System.out.println("Number of collections: " + bean.getCollectionCount());
      System.out.println("Collection time: " + bean.getCollectionTime() + "ms");
      System.out.println("Pool names");

      for (String name : bean.getMemoryPoolNames()) {
        System.out.println("\t" + name);
      }
      System.out.println();
    }
  }
  @Override
  protected void executeReadAttribute(OperationContext context, ModelNode operation)
      throws OperationFailedException {

    final String gcName =
        PathAddress.pathAddress(operation.require(ModelDescriptionConstants.OP_ADDR))
            .getLastElement()
            .getValue();
    final String name = operation.require(ModelDescriptionConstants.NAME).asString();

    GarbageCollectorMXBean gcMBean = null;

    for (GarbageCollectorMXBean mbean : ManagementFactory.getGarbageCollectorMXBeans()) {
      if (gcName.equals(escapeMBeanName(mbean.getName()))) {
        gcMBean = mbean;
      }
    }

    if (gcMBean == null) {
      throw PlatformMBeanLogger.ROOT_LOGGER.unknownGarbageCollector(gcName);
    }

    if (PlatformMBeanConstants.OBJECT_NAME.getName().equals(name)) {
      final String objName =
          PlatformMBeanUtil.getObjectNameStringWithNameKey(
              ManagementFactory.GARBAGE_COLLECTOR_MXBEAN_DOMAIN_TYPE, gcName);
      context.getResult().set(objName);
    } else if (ModelDescriptionConstants.NAME.equals(name)) {
      context.getResult().set(escapeMBeanName(gcMBean.getName()));
    } else if (PlatformMBeanConstants.VALID.getName().equals(name)) {
      context.getResult().set(gcMBean.isValid());
    } else if (PlatformMBeanConstants.MEMORY_POOL_NAMES.equals(name)) {
      final ModelNode result = context.getResult();
      result.setEmptyList();
      for (String pool : gcMBean.getMemoryPoolNames()) {
        result.add(escapeMBeanName(pool));
      }
    } else if (PlatformMBeanConstants.COLLECTION_COUNT.equals(name)) {
      context.getResult().set(gcMBean.getCollectionCount());
    } else if (PlatformMBeanConstants.COLLECTION_TIME.equals(name)) {
      context.getResult().set(gcMBean.getCollectionTime());
    } else if (GarbageCollectorResourceDefinition.GARBAGE_COLLECTOR_READ_ATTRIBUTES.contains(name)
        || GarbageCollectorResourceDefinition.GARBAGE_COLLECTOR_METRICS.contains(name)) {
      // Bug
      throw PlatformMBeanLogger.ROOT_LOGGER.badReadAttributeImpl(name);
    } else {
      // Shouldn't happen; the global handler should reject
      throw unknownAttribute(operation);
    }
  }
 /** Add garbage collection metrics. */
 protected void addGarbageCollecitonMetrics(Collection<Metric<?>> result) {
   List<GarbageCollectorMXBean> garbageCollectorMxBeans =
       ManagementFactory.getGarbageCollectorMXBeans();
   for (int i = 0; i < garbageCollectorMxBeans.size(); i++) {
     GarbageCollectorMXBean garbageCollectorMXBean = garbageCollectorMxBeans.get(i);
     String name = beautifyGcName(garbageCollectorMXBean.getName());
     result.add(
         new Metric<Long>(
             "gc." + name + ".count", new Long(garbageCollectorMXBean.getCollectionCount())));
     result.add(
         new Metric<Long>(
             "gc." + name + ".time", new Long(garbageCollectorMXBean.getCollectionTime())));
   }
 }
  public synchronized void addMutation(String table, Mutation m) throws MutationsRejectedException {

    if (closed) throw new IllegalStateException("Closed");
    if (m.size() == 0) throw new IllegalArgumentException("Can not add empty mutations");

    checkForFailures();

    while ((totalMemUsed >= maxMem || flushing) && !somethingFailed) {
      waitRTE();
    }

    // do checks again since things could have changed while waiting and not holding lock
    if (closed) throw new IllegalStateException("Closed");
    checkForFailures();

    if (startTime == 0) {
      startTime = System.currentTimeMillis();

      List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
      for (GarbageCollectorMXBean garbageCollectorMXBean : gcmBeans) {
        initialGCTimes += garbageCollectorMXBean.getCollectionTime();
      }

      CompilationMXBean compMxBean = ManagementFactory.getCompilationMXBean();
      if (compMxBean.isCompilationTimeMonitoringSupported()) {
        initialCompileTimes = compMxBean.getTotalCompilationTime();
      }

      initialSystemLoad = ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();
    }

    // create a copy of mutation so that after this method returns the user
    // is free to reuse the mutation object, like calling readFields... this
    // is important for the case where a mutation is passed from map to reduce
    // to batch writer... the map reduce code will keep passing the same mutation
    // object into the reduce method
    m = new Mutation(m);

    totalMemUsed += m.estimatedMemoryUsed();
    mutations.addMutation(table, m);
    totalAdded++;

    if (mutations.getMemoryUsed() >= maxMem / 2) {
      startProcessing();
      checkForFailures();
    }
  }
Beispiel #9
0
  static void MonitorGC(long h) {

    handle = h;
    List<GarbageCollectorMXBean> gcbeans =
        java.lang.management.ManagementFactory.getGarbageCollectorMXBeans();
    MBeanServer server = ManagementFactory.getPlatformMBeanServer();
    try {
      ObjectName gcName =
          new ObjectName(ManagementFactory.GARBAGE_COLLECTOR_MXBEAN_DOMAIN_TYPE + ",*");
      for (ObjectName name : server.queryNames(gcName, null)) {
        GarbageCollectorMXBean gc =
            ManagementFactory.newPlatformMXBeanProxy(
                server, name.getCanonicalName(), GarbageCollectorMXBean.class);
        gcbeans.add(gc);

        NotificationEmitter emitter = (NotificationEmitter) gc;
        NotificationListener listener =
            new NotificationListener() {
              @Override
              public void handleNotification(Notification notification, Object handback) {
                if (notification
                    .getType()
                    .equals(GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION)) {
                  CompositeData ndata = (CompositeData) notification.getUserData();
                  GarbageCollectionNotificationInfo info =
                      GarbageCollectionNotificationInfo.from(ndata);
                  boolean major = "end of major GC".equals(info.getGcAction());
                  long free = Runtime.getRuntime().freeMemory();
                  long total = Runtime.getRuntime().totalMemory();
                  long qty = (15 * total) - (free * 100);
                  if (qty > 0) {
                    NotifyOSv(handle, qty);
                  }
                }
              }
            };
        emitter.addNotificationListener(listener, null, null);
      }
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }
Beispiel #10
0
 public static List<GarbageCollectorMXBean> gc() {
   return ManagementFactory.getGarbageCollectorMXBeans();
 }
Beispiel #11
0
  public static void main(String[] args) {

    if (args.length != 1) {
      throw new Error("Expected memory counter name wasn't provided as command line argument");
    }
    MemoryCounter memoryCounter = MemoryCounter.valueOf(args[0].toUpperCase());

    int byteArrayMemoryOverhead = Helpers.detectByteArrayAllocationOverhead();

    // Largest non-humongous byte[]
    int maxByteArrayNonHumongousSize = HALF_G1_REGION_SIZE - byteArrayMemoryOverhead;

    // Maximum byte[] that takes one region
    int maxByteArrayOneRegionSize = G1_REGION_SIZE - byteArrayMemoryOverhead;

    List<Integer> allocationSizes =
        Arrays.asList(
            (int) maxByteArrayNonHumongousSize + 1,
            (int) (0.8f * maxByteArrayOneRegionSize),
            (int) (maxByteArrayOneRegionSize),
            (int) (1.2f * maxByteArrayOneRegionSize),
            (int) (1.5f * maxByteArrayOneRegionSize),
            (int) (1.7f * maxByteArrayOneRegionSize),
            (int) (2.0f * maxByteArrayOneRegionSize),
            (int) (2.5f * maxByteArrayOneRegionSize));

    List<Allocation> allocations = new ArrayList<>();
    List<GarbageCollectorMXBean> gcBeans = ManagementFactory.getGarbageCollectorMXBeans();

    long gcCountBefore =
        gcBeans.stream().mapToLong(GarbageCollectorMXBean::getCollectionCount).sum();

    System.out.println("Starting allocations - no GC should happen until we finish them");

    for (int allocationSize : allocationSizes) {

      long usedMemoryBefore = memoryCounter.getUsedMemory();
      long expectedAllocationSize =
          (long) Math.ceil((double) allocationSize / G1_REGION_SIZE) * G1_REGION_SIZE;
      allocations.add(new Allocation(allocationSize, expectedAllocationSize));
      long usedMemoryAfter = memoryCounter.getUsedMemory();

      System.out.format(
          "Expected allocation size: %d\nUsed memory before allocation: %d\n"
              + "Used memory after allocation: %d\n",
          expectedAllocationSize, usedMemoryBefore, usedMemoryAfter);

      long gcCountNow =
          gcBeans.stream().mapToLong(GarbageCollectorMXBean::getCollectionCount).sum();

      if (gcCountNow == gcCountBefore) {
        // We should allocate at least allocation.expectedSize
        Asserts.assertGreaterThanOrEqual(
            usedMemoryAfter - usedMemoryBefore,
            expectedAllocationSize,
            "Counter of type "
                + memoryCounter.getClass().getSimpleName()
                + " returned wrong allocation size");
      } else {
        System.out.println("GC happened during allocation so the check is skipped");
        gcCountBefore = gcCountNow;
      }
    }

    System.out.println("Finished allocations - no GC should have happened before this line");

    allocations
        .stream()
        .forEach(
            allocation -> {
              long usedMemoryBefore = memoryCounter.getUsedMemory();
              allocation.forgetAllocation();

              WHITE_BOX.fullGC();

              long usedMemoryAfter = memoryCounter.getUsedMemory();

              // We should free at least allocation.expectedSize * ALLOCATION_SIZE_TOLERANCE_FACTOR
              Asserts.assertGreaterThanOrEqual(
                  usedMemoryBefore - usedMemoryAfter,
                  (long) (allocation.expectedSize * ALLOCATION_SIZE_TOLERANCE_FACTOR),
                  "Counter of type "
                      + memoryCounter.getClass().getSimpleName()
                      + " returned wrong allocation size");
            });
  }
Beispiel #12
0
 public GcTimeUpdater() {
   this.gcBeans = ManagementFactory.getGarbageCollectorMXBeans();
   getElapsedGc(); // Initialize 'lastGcMillis' with the current time spent.
 }
Beispiel #13
0
  static void installGCMonitoring() {
    List<GarbageCollectorMXBean> gcbeans = ManagementFactory.getGarbageCollectorMXBeans();

    for (GarbageCollectorMXBean gcbean : gcbeans) {
      System.out.println(gcbean);
      NotificationEmitter emitter = (NotificationEmitter) gcbean;
      // use an anonymously generated listener for this example
      // - proper code should really use a named class
      NotificationListener listener =
          new NotificationListener() {
            // keep a count of the total time spent in GCs
            long totalGcDuration = 0;

            // implement the notifier callback handler
            @Override
            public void handleNotification(Notification notification, Object handback) {
              // we only handle GARBAGE_COLLECTION_NOTIFICATION notifications here
              if (notification
                  .getType()
                  .equals(GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION)) {
                // get the information associated with this notification
                GarbageCollectionNotificationInfo info =
                    GarbageCollectionNotificationInfo.from(
                        (CompositeData) notification.getUserData());
                // get all the info and pretty print it
                long duration = info.getGcInfo().getDuration();
                String gctype = info.getGcAction();
                if ("end of minor GC".equals(gctype)) {
                  gctype = "Young Gen GC";
                } else if ("end of major GC".equals(gctype)) {
                  gctype = "Old Gen GC";
                }
                System.out.println();
                System.out.println(
                    gctype
                        + ": - "
                        + info.getGcInfo().getId()
                        + " "
                        + info.getGcName()
                        + " (from "
                        + info.getGcCause()
                        + ") "
                        + duration
                        + " microseconds; start-end times "
                        + info.getGcInfo().getStartTime()
                        + "-"
                        + info.getGcInfo().getEndTime());
                // System.out.println("GcInfo CompositeType: " +
                // info.getGcInfo().getCompositeType());
                // System.out.println("GcInfo MemoryUsageAfterGc: " +
                // info.getGcInfo().getMemoryUsageAfterGc());
                // System.out.println("GcInfo MemoryUsageBeforeGc: " +
                // info.getGcInfo().getMemoryUsageBeforeGc());

                // Get the information about each memory space, and pretty print it
                Map<String, MemoryUsage> membefore = info.getGcInfo().getMemoryUsageBeforeGc();
                Map<String, MemoryUsage> mem = info.getGcInfo().getMemoryUsageAfterGc();
                for (Map.Entry<String, MemoryUsage> entry : mem.entrySet()) {
                  String name = entry.getKey();
                  MemoryUsage memdetail = entry.getValue();
                  long memInit = memdetail.getInit();
                  long memCommitted = memdetail.getCommitted();
                  long memMax = memdetail.getMax();
                  long memUsed = memdetail.getUsed();
                  MemoryUsage before = membefore.get(name);
                  long beforepercent = ((before.getUsed() * 1000L) / before.getCommitted());
                  long percent =
                      ((memUsed * 1000L) / before.getCommitted()); // >100% when it gets expanded

                  System.out.print(
                      name
                          + (memCommitted == memMax ? "(fully expanded)" : "(still expandable)")
                          + "used: "
                          + (beforepercent / 10)
                          + "."
                          + (beforepercent % 10)
                          + "%->"
                          + (percent / 10)
                          + "."
                          + (percent % 10)
                          + "%("
                          + ((memUsed / 1048576) + 1)
                          + "MB) / ");
                }
                System.out.println();
                totalGcDuration += info.getGcInfo().getDuration();
                long percent = totalGcDuration * 1000L / info.getGcInfo().getEndTime();
                System.out.println(
                    "GC cumulated overhead " + (percent / 10) + "." + (percent % 10) + "%");
              }
            }
          };

      // Add the listener
      emitter.addNotificationListener(listener, null, null);
    }
  }
  /**
   * All parameters are obtained from the {@link GlobalConfiguration}, which must be loaded prior to
   * instantiating the task manager.
   */
  public TaskManager(ExecutionMode executionMode) throws Exception {
    if (executionMode == null) {
      throw new NullPointerException("Execution mode must not be null.");
    }

    LOG.info("Execution mode: " + executionMode);

    // IMPORTANT! At this point, the GlobalConfiguration must have been read!

    final InetSocketAddress jobManagerAddress;
    {
      LOG.info("Reading location of job manager from configuration");

      final String address =
          GlobalConfiguration.getString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, null);
      final int port =
          GlobalConfiguration.getInteger(
              ConfigConstants.JOB_MANAGER_IPC_PORT_KEY,
              ConfigConstants.DEFAULT_JOB_MANAGER_IPC_PORT);

      if (address == null) {
        throw new Exception("Job manager address not configured in the GlobalConfiguration.");
      }

      // Try to convert configured address to {@link InetAddress}
      try {
        final InetAddress tmpAddress = InetAddress.getByName(address);
        jobManagerAddress = new InetSocketAddress(tmpAddress, port);
      } catch (UnknownHostException e) {
        LOG.fatal("Could not resolve JobManager host name.");
        throw new Exception("Could not resolve JobManager host name: " + e.getMessage(), e);
      }

      LOG.info("Connecting to JobManager at: " + jobManagerAddress);
    }

    // Create RPC connection to the JobManager
    try {
      this.jobManager =
          RPC.getProxy(JobManagerProtocol.class, jobManagerAddress, NetUtils.getSocketFactory());
    } catch (IOException e) {
      LOG.fatal("Could not connect to the JobManager: " + e.getMessage(), e);
      throw new Exception("Failed to initialize connection to JobManager: " + e.getMessage(), e);
    }

    int ipcPort = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_IPC_PORT_KEY, -1);
    int dataPort = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, -1);
    if (ipcPort == -1) {
      ipcPort = getAvailablePort();
    }
    if (dataPort == -1) {
      dataPort = getAvailablePort();
    }

    // Determine our own public facing address and start the server
    {
      final InetAddress taskManagerAddress;
      try {
        taskManagerAddress = getTaskManagerAddress(jobManagerAddress);
      } catch (Exception e) {
        throw new RuntimeException(
            "The TaskManager failed to determine its own network address.", e);
      }

      this.localInstanceConnectionInfo =
          new InstanceConnectionInfo(taskManagerAddress, ipcPort, dataPort);
      LOG.info("TaskManager connection information:" + this.localInstanceConnectionInfo);

      // Start local RPC server
      try {
        this.taskManagerServer =
            RPC.getServer(this, taskManagerAddress.getHostAddress(), ipcPort, IPC_HANDLER_COUNT);
        this.taskManagerServer.start();
      } catch (IOException e) {
        LOG.fatal("Failed to start TaskManager server. " + e.getMessage(), e);
        throw new Exception("Failed to start taskmanager server. " + e.getMessage(), e);
      }
    }

    // Try to create local stub of the global input split provider
    try {
      this.globalInputSplitProvider =
          RPC.getProxy(
              InputSplitProviderProtocol.class, jobManagerAddress, NetUtils.getSocketFactory());
    } catch (IOException e) {
      LOG.fatal(e.getMessage(), e);
      throw new Exception(
          "Failed to initialize connection to global input split provider: " + e.getMessage(), e);
    }

    // Try to create local stub for the lookup service
    try {
      this.lookupService =
          RPC.getProxy(ChannelLookupProtocol.class, jobManagerAddress, NetUtils.getSocketFactory());
    } catch (IOException e) {
      LOG.fatal(e.getMessage(), e);
      throw new Exception("Failed to initialize channel lookup protocol. " + e.getMessage(), e);
    }

    // Try to create local stub for the accumulators
    try {
      this.accumulatorProtocolProxy =
          RPC.getProxy(AccumulatorProtocol.class, jobManagerAddress, NetUtils.getSocketFactory());
    } catch (IOException e) {
      LOG.fatal("Failed to initialize accumulator protocol: " + e.getMessage(), e);
      throw new Exception("Failed to initialize accumulator protocol: " + e.getMessage(), e);
    }

    // Load profiler if it should be used
    if (GlobalConfiguration.getBoolean(ProfilingUtils.ENABLE_PROFILING_KEY, false)) {

      final String profilerClassName =
          GlobalConfiguration.getString(
              ProfilingUtils.TASKMANAGER_CLASSNAME_KEY,
              "org.apache.flink.runtime.profiling.impl.TaskManagerProfilerImpl");

      this.profiler =
          ProfilingUtils.loadTaskManagerProfiler(
              profilerClassName, jobManagerAddress.getAddress(), this.localInstanceConnectionInfo);

      if (this.profiler == null) {
        LOG.error("Cannot find class name for the profiler.");
      } else {
        LOG.info("Profiling of jobs is enabled.");
      }
    } else {
      this.profiler = null;
      LOG.info("Profiling of jobs is disabled.");
    }

    // Get the directory for storing temporary files
    final String[] tmpDirPaths =
        GlobalConfiguration.getString(
                ConfigConstants.TASK_MANAGER_TMP_DIR_KEY,
                ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH)
            .split(",|" + File.pathSeparator);

    checkTempDirs(tmpDirPaths);

    int numBuffers =
        GlobalConfiguration.getInteger(
            ConfigConstants.TASK_MANAGER_NETWORK_NUM_BUFFERS_KEY,
            ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_NUM_BUFFERS);

    int bufferSize =
        GlobalConfiguration.getInteger(
            ConfigConstants.TASK_MANAGER_NETWORK_BUFFER_SIZE_KEY,
            ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_BUFFER_SIZE);

    // Initialize the channel manager
    try {
      NetworkConnectionManager networkConnectionManager = null;

      switch (executionMode) {
        case LOCAL:
          networkConnectionManager = new LocalConnectionManager();
          break;
        case CLUSTER:
          int numInThreads =
              GlobalConfiguration.getInteger(
                  ConfigConstants.TASK_MANAGER_NET_NUM_IN_THREADS_KEY,
                  ConfigConstants.DEFAULT_TASK_MANAGER_NET_NUM_IN_THREADS);

          int numOutThreads =
              GlobalConfiguration.getInteger(
                  ConfigConstants.TASK_MANAGER_NET_NUM_OUT_THREADS_KEY,
                  ConfigConstants.DEFAULT_TASK_MANAGER_NET_NUM_OUT_THREADS);

          int lowWaterMark =
              GlobalConfiguration.getInteger(
                  ConfigConstants.TASK_MANAGER_NET_NETTY_LOW_WATER_MARK,
                  ConfigConstants.DEFAULT_TASK_MANAGER_NET_NETTY_LOW_WATER_MARK);

          int highWaterMark =
              GlobalConfiguration.getInteger(
                  ConfigConstants.TASK_MANAGER_NET_NETTY_HIGH_WATER_MARK,
                  ConfigConstants.DEFAULT_TASK_MANAGER_NET_NETTY_HIGH_WATER_MARK);

          networkConnectionManager =
              new NettyConnectionManager(
                  localInstanceConnectionInfo.address(),
                  localInstanceConnectionInfo.dataPort(),
                  bufferSize,
                  numInThreads,
                  numOutThreads,
                  lowWaterMark,
                  highWaterMark);
          break;
      }

      channelManager =
          new ChannelManager(
              lookupService,
              localInstanceConnectionInfo,
              numBuffers,
              bufferSize,
              networkConnectionManager);
    } catch (IOException ioe) {
      LOG.error(StringUtils.stringifyException(ioe));
      throw new Exception("Failed to instantiate ChannelManager.", ioe);
    }

    // initialize the number of slots
    {
      int slots = GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, -1);
      if (slots == -1) {
        slots = 1;
        LOG.info("Number of task slots not configured. Creating one task slot.");
      } else if (slots <= 0) {
        throw new Exception("Illegal value for the number of task slots: " + slots);
      } else {
        LOG.info("Creating " + slots + " task slot(s).");
      }
      this.numberOfSlots = slots;
    }

    this.hardwareDescription = HardwareDescriptionFactory.extractFromSystem();

    // initialize the memory manager
    {
      // Check whether the memory size has been explicitly configured.
      final long configuredMemorySize =
          GlobalConfiguration.getInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1);
      final long memorySize;

      if (configuredMemorySize == -1) {
        // no manually configured memory. take a relative fraction of the free heap space
        float fraction =
            GlobalConfiguration.getFloat(
                ConfigConstants.TASK_MANAGER_MEMORY_FRACTION_KEY,
                ConfigConstants.DEFAULT_MEMORY_MANAGER_MEMORY_FRACTION);
        memorySize = (long) (this.hardwareDescription.getSizeOfFreeMemory() * fraction);
        LOG.info("Using " + fraction + " of the free heap space for managed memory.");
      } else if (configuredMemorySize <= 0) {
        throw new Exception(
            "Invalid value for Memory Manager memory size: " + configuredMemorySize);
      } else {
        memorySize = configuredMemorySize << 20;
      }

      final int pageSize =
          GlobalConfiguration.getInteger(
              ConfigConstants.TASK_MANAGER_NETWORK_BUFFER_SIZE_KEY,
              ConfigConstants.DEFAULT_TASK_MANAGER_NETWORK_BUFFER_SIZE);

      // Initialize the memory manager
      LOG.info(
          "Initializing memory manager with "
              + (memorySize >>> 20)
              + " megabytes of memory. "
              + "Page size is "
              + pageSize
              + " bytes.");

      try {
        @SuppressWarnings("unused")
        final boolean lazyAllocation =
            GlobalConfiguration.getBoolean(
                ConfigConstants.TASK_MANAGER_MEMORY_LAZY_ALLOCATION_KEY,
                ConfigConstants.DEFAULT_TASK_MANAGER_MEMORY_LAZY_ALLOCATION);

        this.memoryManager = new DefaultMemoryManager(memorySize, this.numberOfSlots, pageSize);
      } catch (Throwable t) {
        LOG.fatal(
            "Unable to initialize memory manager with "
                + (memorySize >>> 20)
                + " megabytes of memory.",
            t);
        throw new Exception("Unable to initialize memory manager.", t);
      }
    }

    this.ioManager = new IOManager(tmpDirPaths);

    this.heartbeatThread =
        new Thread() {
          @Override
          public void run() {
            runHeartbeatLoop();
          }
        };

    this.heartbeatThread.setName("Heartbeat Thread");
    this.heartbeatThread.start();

    // --------------------------------------------------------------------
    // Memory Usage
    // --------------------------------------------------------------------

    final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
    final List<GarbageCollectorMXBean> gcMXBeans = ManagementFactory.getGarbageCollectorMXBeans();

    LOG.info(getMemoryUsageStatsAsString(memoryMXBean));

    boolean startMemoryUsageLogThread =
        GlobalConfiguration.getBoolean(
            ConfigConstants.TASK_MANAGER_DEBUG_MEMORY_USAGE_START_LOG_THREAD,
            ConfigConstants.DEFAULT_TASK_MANAGER_DEBUG_MEMORY_USAGE_START_LOG_THREAD);

    if (startMemoryUsageLogThread && LOG.isDebugEnabled()) {
      final int logIntervalMs =
          GlobalConfiguration.getInteger(
              ConfigConstants.TASK_MANAGER_DEBUG_MEMORY_USAGE_LOG_INTERVAL_MS,
              ConfigConstants.DEFAULT_TASK_MANAGER_DEBUG_MEMORY_USAGE_LOG_INTERVAL_MS);

      new Thread(
              new Runnable() {
                @Override
                public void run() {
                  try {
                    while (!isShutDown()) {
                      Thread.sleep(logIntervalMs);

                      LOG.debug(getMemoryUsageStatsAsString(memoryMXBean));

                      LOG.debug(getGarbageCollectorStatsAsString(gcMXBeans));
                    }
                  } catch (InterruptedException e) {
                    LOG.warn("Unexpected interruption of memory usage logger thread.");
                  }
                }
              })
          .start();
    }
  }
Beispiel #15
0
  /** Exports stats related to the JVM and runtime environment. */
  public static void export() {
    final OperatingSystemMXBean osMbean = ManagementFactory.getOperatingSystemMXBean();
    if (osMbean instanceof com.sun.management.OperatingSystemMXBean) {
      final com.sun.management.OperatingSystemMXBean sunOsMbean =
          (com.sun.management.OperatingSystemMXBean) osMbean;

      Stats.exportAll(
          ImmutableList.<Stat<? extends Number>>builder()
              .add(
                  new StatImpl<Long>("system_free_physical_memory_mb") {
                    @Override
                    public Long read() {
                      return sunOsMbean.getFreePhysicalMemorySize() / BYTES_PER_MB;
                    }
                  })
              .add(
                  new StatImpl<Long>("system_free_swap_mb") {
                    @Override
                    public Long read() {
                      return sunOsMbean.getFreeSwapSpaceSize() / BYTES_PER_MB;
                    }
                  })
              .add(
                  Rate.of(
                          new StatImpl<Long>("process_cpu_time_nanos") {
                            @Override
                            public Long read() {
                              return sunOsMbean.getProcessCpuTime();
                            }
                          })
                      .withName("process_cpu_cores_utilized")
                      .withScaleFactor(SECS_PER_NANO)
                      .build())
              .build());
    }
    if (osMbean instanceof com.sun.management.UnixOperatingSystemMXBean) {
      final com.sun.management.UnixOperatingSystemMXBean unixOsMbean =
          (com.sun.management.UnixOperatingSystemMXBean) osMbean;

      Stats.exportAll(
          ImmutableList.<Stat<? extends Number>>builder()
              .add(
                  new StatImpl<Long>("process_max_fd_count") {
                    @Override
                    public Long read() {
                      return unixOsMbean.getMaxFileDescriptorCount();
                    }
                  })
              .add(
                  new StatImpl<Long>("process_open_fd_count") {
                    @Override
                    public Long read() {
                      return unixOsMbean.getOpenFileDescriptorCount();
                    }
                  })
              .build());
    }

    final Runtime runtime = Runtime.getRuntime();
    final ClassLoadingMXBean classLoadingBean = ManagementFactory.getClassLoadingMXBean();
    MemoryMXBean memoryBean = ManagementFactory.getMemoryMXBean();
    final MemoryUsage heapUsage = memoryBean.getHeapMemoryUsage();
    final MemoryUsage nonHeapUsage = memoryBean.getNonHeapMemoryUsage();
    final ThreadMXBean threads = ManagementFactory.getThreadMXBean();
    final RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();

    Stats.exportAll(
        ImmutableList.<Stat<? extends Number>>builder()
            .add(
                new StatImpl<Long>("jvm_time_ms") {
                  @Override
                  public Long read() {
                    return System.currentTimeMillis();
                  }
                })
            .add(
                new StatImpl<Integer>("jvm_available_processors") {
                  @Override
                  public Integer read() {
                    return runtime.availableProcessors();
                  }
                })
            .add(
                new StatImpl<Long>("jvm_memory_free_mb") {
                  @Override
                  public Long read() {
                    return runtime.freeMemory() / BYTES_PER_MB;
                  }
                })
            .add(
                new StatImpl<Long>("jvm_memory_max_mb") {
                  @Override
                  public Long read() {
                    return runtime.maxMemory() / BYTES_PER_MB;
                  }
                })
            .add(
                new StatImpl<Long>("jvm_memory_mb_total") {
                  @Override
                  public Long read() {
                    return runtime.totalMemory() / BYTES_PER_MB;
                  }
                })
            .add(
                new StatImpl<Integer>("jvm_class_loaded_count") {
                  @Override
                  public Integer read() {
                    return classLoadingBean.getLoadedClassCount();
                  }
                })
            .add(
                new StatImpl<Long>("jvm_class_total_loaded_count") {
                  @Override
                  public Long read() {
                    return classLoadingBean.getTotalLoadedClassCount();
                  }
                })
            .add(
                new StatImpl<Long>("jvm_class_unloaded_count") {
                  @Override
                  public Long read() {
                    return classLoadingBean.getUnloadedClassCount();
                  }
                })
            .add(
                new StatImpl<Long>("jvm_gc_collection_time_ms") {
                  @Override
                  public Long read() {
                    long collectionTimeMs = 0;
                    for (GarbageCollectorMXBean bean :
                        ManagementFactory.getGarbageCollectorMXBeans()) {
                      collectionTimeMs += bean.getCollectionTime();
                    }
                    return collectionTimeMs;
                  }
                })
            .add(
                new StatImpl<Long>("jvm_gc_collection_count") {
                  @Override
                  public Long read() {
                    long collections = 0;
                    for (GarbageCollectorMXBean bean :
                        ManagementFactory.getGarbageCollectorMXBeans()) {
                      collections += bean.getCollectionCount();
                    }
                    return collections;
                  }
                })
            .add(
                new StatImpl<Long>("jvm_memory_heap_mb_used") {
                  @Override
                  public Long read() {
                    return heapUsage.getUsed() / BYTES_PER_MB;
                  }
                })
            .add(
                new StatImpl<Long>("jvm_memory_heap_mb_committed") {
                  @Override
                  public Long read() {
                    return heapUsage.getCommitted() / BYTES_PER_MB;
                  }
                })
            .add(
                new StatImpl<Long>("jvm_memory_heap_mb_max") {
                  @Override
                  public Long read() {
                    return heapUsage.getMax() / BYTES_PER_MB;
                  }
                })
            .add(
                new StatImpl<Long>("jvm_memory_non_heap_mb_used") {
                  @Override
                  public Long read() {
                    return nonHeapUsage.getUsed() / BYTES_PER_MB;
                  }
                })
            .add(
                new StatImpl<Long>("jvm_memory_non_heap_mb_committed") {
                  @Override
                  public Long read() {
                    return nonHeapUsage.getCommitted() / BYTES_PER_MB;
                  }
                })
            .add(
                new StatImpl<Long>("jvm_memory_non_heap_mb_max") {
                  @Override
                  public Long read() {
                    return nonHeapUsage.getMax() / BYTES_PER_MB;
                  }
                })
            .add(
                new StatImpl<Long>("jvm_uptime_secs") {
                  @Override
                  public Long read() {
                    return runtimeMXBean.getUptime() / 1000;
                  }
                })
            .add(
                new StatImpl<Double>("system_load_avg") {
                  @Override
                  public Double read() {
                    return osMbean.getSystemLoadAverage();
                  }
                })
            .add(
                new StatImpl<Integer>("jvm_threads_peak") {
                  @Override
                  public Integer read() {
                    return threads.getPeakThreadCount();
                  }
                })
            .add(
                new StatImpl<Long>("jvm_threads_started") {
                  @Override
                  public Long read() {
                    return threads.getTotalStartedThreadCount();
                  }
                })
            .add(
                new StatImpl<Integer>("jvm_threads_daemon") {
                  @Override
                  public Integer read() {
                    return threads.getDaemonThreadCount();
                  }
                })
            .add(
                new StatImpl<Integer>("jvm_threads_active") {
                  @Override
                  public Integer read() {
                    return threads.getThreadCount();
                  }
                })
            .build());

    // Export per memory pool gc time and cycle count like Ostrich
    // This is based on code in Bridcage: https://cgit.twitter.biz/birdcage/tree/ \
    // ostrich/src/main/scala/com/twitter/ostrich/stats/StatsCollection.scala
    Stats.exportAll(
        Iterables.transform(
            ManagementFactory.getGarbageCollectorMXBeans(),
            new Function<GarbageCollectorMXBean, Stat<? extends Number>>() {
              @Override
              public Stat<? extends Number> apply(final GarbageCollectorMXBean gcMXBean) {
                return new StatImpl<Long>(
                    "jvm_gc_" + Stats.normalizeName(gcMXBean.getName()) + "_collection_count") {
                  @Override
                  public Long read() {
                    return gcMXBean.getCollectionCount();
                  }
                };
              }
            }));

    Stats.exportAll(
        Iterables.transform(
            ManagementFactory.getGarbageCollectorMXBeans(),
            new Function<GarbageCollectorMXBean, Stat<? extends Number>>() {
              @Override
              public Stat<? extends Number> apply(final GarbageCollectorMXBean gcMXBean) {
                return new StatImpl<Long>(
                    "jvm_gc_" + Stats.normalizeName(gcMXBean.getName()) + "_collection_time_ms") {
                  @Override
                  public Long read() {
                    return gcMXBean.getCollectionTime();
                  }
                };
              }
            }));

    Stats.exportString(
        new StatImpl<String>("jvm_input_arguments") {
          @Override
          public String read() {
            return runtimeMXBean.getInputArguments().toString();
          }
        });

    for (final String property : System.getProperties().stringPropertyNames()) {
      Stats.exportString(
          new StatImpl<String>("jvm_prop_" + Stats.normalizeName(property)) {
            @Override
            public String read() {
              return System.getProperty(property);
            }
          });
    }
  }
Beispiel #16
0
  /**
   * Builds an XML report of the timings to be uploaded for parsing.
   *
   * @param sender Who to report to
   */
  static void reportTimings(CommandSource sender) {
    Platform platform = SpongeImpl.getGame().getPlatform();
    JsonObjectBuilder builder =
        JSONUtil.objectBuilder()
            // Get some basic system details about the server
            .add(
                "version",
                platform
                    .getImplementation()
                    .getVersion()
                    .orElse(platform.getMinecraftVersion().getName() + "-DEV"))
            .add("maxplayers", SpongeImpl.getGame().getServer().getMaxPlayers())
            .add("start", TimingsManager.timingStart / 1000)
            .add("end", System.currentTimeMillis() / 1000)
            .add("sampletime", (System.currentTimeMillis() - TimingsManager.timingStart) / 1000);
    if (!TimingsManager.privacy) {
      builder
          .add("server", getServerName())
          .add("motd", SpongeImpl.getGame().getServer().getMotd().toPlain())
          .add("online-mode", SpongeImpl.getGame().getServer().getOnlineMode())
          .add("icon", MinecraftServer.getServer().getServerStatusResponse().getFavicon());
    }

    final Runtime runtime = Runtime.getRuntime();
    RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean();
    builder.add(
        "system",
        JSONUtil.objectBuilder()
            .add("timingcost", getCost())
            .add("name", System.getProperty("os.name"))
            .add("version", System.getProperty("os.version"))
            .add("jvmversion", System.getProperty("java.version"))
            .add("arch", System.getProperty("os.arch"))
            .add("maxmem", runtime.maxMemory())
            .add("cpu", runtime.availableProcessors())
            .add("runtime", ManagementFactory.getRuntimeMXBean().getUptime())
            .add("flags", RUNTIME_FLAG_JOINER.join(runtimeBean.getInputArguments()))
            .add(
                "gc",
                JSONUtil.mapArrayToObject(
                    ManagementFactory.getGarbageCollectorMXBeans(),
                    (input) -> {
                      return JSONUtil.singleObjectPair(
                          input.getName(),
                          JSONUtil.arrayOf(input.getCollectionCount(), input.getCollectionTime()));
                    })));

    Set<BlockType> blockTypeSet = Sets.newHashSet();
    Set<EntityType> entityTypeSet = Sets.newHashSet();

    int size = HISTORY.size();
    TimingHistory[] history = new TimingHistory[size + 1];
    int i = 0;
    for (TimingHistory timingHistory : HISTORY) {
      blockTypeSet.addAll(timingHistory.blockTypeSet);
      entityTypeSet.addAll(timingHistory.entityTypeSet);
      history[i++] = timingHistory;
    }

    history[i] = new TimingHistory(); // Current snapshot
    blockTypeSet.addAll(history[i].blockTypeSet);
    entityTypeSet.addAll(history[i].entityTypeSet);

    JsonObjectBuilder handlersBuilder = JSONUtil.objectBuilder();
    for (TimingIdentifier.TimingGroup group : TimingIdentifier.GROUP_MAP.values()) {
      for (TimingHandler id : group.handlers) {
        if (!id.timed && !id.isSpecial()) {
          continue;
        }
        handlersBuilder.add(id.id, JSONUtil.arrayOf(group.id, id.name));
      }
    }

    builder.add(
        "idmap",
        JSONUtil.objectBuilder()
            .add(
                "groups",
                JSONUtil.mapArrayToObject(
                    TimingIdentifier.GROUP_MAP.values(),
                    (group) -> {
                      return JSONUtil.singleObjectPair(group.id, group.name);
                    }))
            .add("handlers", handlersBuilder)
            .add(
                "worlds",
                JSONUtil.mapArrayToObject(
                    TimingHistory.worldMap.entrySet(),
                    (entry) -> {
                      return JSONUtil.singleObjectPair(entry.getValue(), entry.getKey());
                    }))
            .add(
                "tileentity",
                JSONUtil.mapArrayToObject(
                    blockTypeSet,
                    (blockType) -> {
                      return JSONUtil.singleObjectPair(
                          Block.getIdFromBlock((Block) blockType), blockType.getId());
                    }))
            .add(
                "entity",
                JSONUtil.mapArrayToObject(
                    entityTypeSet,
                    (entityType) -> {
                      return JSONUtil.singleObjectPair(
                          ((SpongeEntityType) entityType).entityTypeId, entityType.getId());
                    })));

    // Information about loaded plugins

    builder.add(
        "plugins",
        JSONUtil.mapArrayToObject(
            SpongeImpl.getGame().getPluginManager().getPlugins(),
            (plugin) -> {
              return JSONUtil.objectBuilder()
                  .add(
                      plugin.getId(),
                      JSONUtil.objectBuilder()
                          .add("version", plugin.getVersion().orElse(""))
                          .add("description", plugin.getDescription().orElse(""))
                          .add("website", plugin.getUrl().orElse(""))
                          .add("authors", AUTHOR_LIST_JOINER.join(plugin.getAuthors())))
                  .build();
            }));

    // Information on the users Config

    builder.add(
        "config",
        JSONUtil.objectBuilder()
            .add("sponge", serializeConfigNode(SpongeImpl.getGlobalConfig().getRootNode())));

    new TimingsExport(sender, builder.build(), history).start();
  }
Beispiel #17
0
 static {
   garbageCollectorMXBeans = ManagementFactory.getGarbageCollectorMXBeans();
   counter = Long.MIN_VALUE;
   times = new HashMap<Long, Pair<Long, Long>>();
 }
Beispiel #18
0
  static {
    RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
    MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();

    // returns the <process id>@<host>
    long pid;
    String xPid = runtimeMXBean.getName();
    try {
      xPid = xPid.split("@")[0];
      pid = Long.parseLong(xPid);
    } catch (Exception e) {
      pid = -1;
    }
    JvmInfo info = new JvmInfo();
    info.pid = pid;
    info.startTime = runtimeMXBean.getStartTime();
    info.version = System.getProperty("java.version");
    info.vmName = runtimeMXBean.getVmName();
    info.vmVendor = runtimeMXBean.getVmVendor();
    info.vmVersion = runtimeMXBean.getVmVersion();
    info.mem = new Mem();
    info.mem.heapInit =
        memoryMXBean.getHeapMemoryUsage().getInit() < 0
            ? 0
            : memoryMXBean.getHeapMemoryUsage().getInit();
    info.mem.heapMax =
        memoryMXBean.getHeapMemoryUsage().getMax() < 0
            ? 0
            : memoryMXBean.getHeapMemoryUsage().getMax();
    info.mem.nonHeapInit =
        memoryMXBean.getNonHeapMemoryUsage().getInit() < 0
            ? 0
            : memoryMXBean.getNonHeapMemoryUsage().getInit();
    info.mem.nonHeapMax =
        memoryMXBean.getNonHeapMemoryUsage().getMax() < 0
            ? 0
            : memoryMXBean.getNonHeapMemoryUsage().getMax();
    try {
      Class<?> vmClass = Class.forName("sun.misc.VM");
      info.mem.directMemoryMax = (Long) vmClass.getMethod("maxDirectMemory").invoke(null);
    } catch (Exception t) {
      // ignore
    }
    info.inputArguments =
        runtimeMXBean
            .getInputArguments()
            .toArray(new String[runtimeMXBean.getInputArguments().size()]);
    try {
      info.bootClassPath = runtimeMXBean.getBootClassPath();
    } catch (UnsupportedOperationException e) {
      // oracle java 9
      info.bootClassPath = System.getProperty("sun.boot.class.path");
      if (info.bootClassPath == null) {
        // something else
        info.bootClassPath = "<unknown>";
      }
    }
    info.classPath = runtimeMXBean.getClassPath();
    info.systemProperties = Collections.unmodifiableMap(runtimeMXBean.getSystemProperties());

    List<GarbageCollectorMXBean> gcMxBeans = ManagementFactory.getGarbageCollectorMXBeans();
    info.gcCollectors = new String[gcMxBeans.size()];
    for (int i = 0; i < gcMxBeans.size(); i++) {
      GarbageCollectorMXBean gcMxBean = gcMxBeans.get(i);
      info.gcCollectors[i] = gcMxBean.getName();
    }

    List<MemoryPoolMXBean> memoryPoolMXBeans = ManagementFactory.getMemoryPoolMXBeans();
    info.memoryPools = new String[memoryPoolMXBeans.size()];
    for (int i = 0; i < memoryPoolMXBeans.size(); i++) {
      MemoryPoolMXBean memoryPoolMXBean = memoryPoolMXBeans.get(i);
      info.memoryPools[i] = memoryPoolMXBean.getName();
    }

    try {
      @SuppressWarnings("unchecked")
      Class<? extends PlatformManagedObject> clazz =
          (Class<? extends PlatformManagedObject>)
              Class.forName("com.sun.management.HotSpotDiagnosticMXBean");
      Class<?> vmOptionClazz = Class.forName("com.sun.management.VMOption");
      PlatformManagedObject hotSpotDiagnosticMXBean = ManagementFactory.getPlatformMXBean(clazz);
      Method vmOptionMethod = clazz.getMethod("getVMOption", String.class);
      Method valueMethod = vmOptionClazz.getMethod("getValue");

      try {
        Object onError = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnError");
        info.onError = (String) valueMethod.invoke(onError);
      } catch (Exception ignored) {
      }

      try {
        Object onOutOfMemoryError =
            vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnOutOfMemoryError");
        info.onOutOfMemoryError = (String) valueMethod.invoke(onOutOfMemoryError);
      } catch (Exception ignored) {
      }

      try {
        Object useCompressedOopsVmOption =
            vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseCompressedOops");
        info.useCompressedOops = (String) valueMethod.invoke(useCompressedOopsVmOption);
      } catch (Exception ignored) {
      }

      try {
        Object useG1GCVmOption = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseG1GC");
        info.useG1GC = (String) valueMethod.invoke(useG1GCVmOption);
      } catch (Exception ignored) {
      }

      try {
        Object initialHeapSizeVmOption =
            vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "InitialHeapSize");
        info.configuredInitialHeapSize =
            Long.parseLong((String) valueMethod.invoke(initialHeapSizeVmOption));
      } catch (Exception ignored) {
      }

      try {
        Object maxHeapSizeVmOption = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "MaxHeapSize");
        info.configuredMaxHeapSize =
            Long.parseLong((String) valueMethod.invoke(maxHeapSizeVmOption));
      } catch (Exception ignored) {
      }
    } catch (Exception ignored) {

    }

    INSTANCE = info;
  }
Beispiel #19
0
public class Diagnostics {

  private static final String PACKAGE = "org.apache.tomcat.util";

  private static final String INDENT1 = "  ";
  private static final String INDENT2 = "\t";
  private static final String INDENT3 = "   ";
  private static final String CRLF = "\r\n";
  private static final String vminfoSystemProperty = "java.vm.info";

  private static final SimpleDateFormat timeformat =
      new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");

  /* Some platform MBeans */
  private static final ClassLoadingMXBean classLoadingMXBean =
      ManagementFactory.getClassLoadingMXBean();
  private static final CompilationMXBean compilationMXBean =
      ManagementFactory.getCompilationMXBean();
  private static final OperatingSystemMXBean operatingSystemMXBean =
      ManagementFactory.getOperatingSystemMXBean();
  private static final RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
  private static final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();

  // XXX Not sure whether the following MBeans should better
  // be retrieved on demand, i.e. whether they can change
  // dynamically in the MBeanServer.
  private static final LoggingMXBean loggingMXBean = LogManager.getLoggingMXBean();
  private static final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
  private static final List<GarbageCollectorMXBean> garbageCollectorMXBeans =
      ManagementFactory.getGarbageCollectorMXBeans();
  private static final List<MemoryManagerMXBean> memoryManagerMXBeans =
      ManagementFactory.getMemoryManagerMXBeans();
  private static final List<MemoryPoolMXBean> memoryPoolMXBeans =
      ManagementFactory.getMemoryPoolMXBeans();

  /**
   * Formats the thread dump header for one thread.
   *
   * @param ti the ThreadInfo describing the thread
   * @return the formatted thread dump header
   */
  private static String getThreadDumpHeader(ThreadInfo ti) {
    StringBuilder sb = new StringBuilder("\"" + ti.getThreadName() + "\"");
    sb.append(" Id=" + ti.getThreadId());
    sb.append(" cpu=" + threadMXBean.getThreadCpuTime(ti.getThreadId()) + " ns");
    sb.append(" usr="******" ns");
    sb.append(" blocked " + ti.getBlockedCount() + " for " + ti.getBlockedTime() + " ms");
    sb.append(" waited " + ti.getWaitedCount() + " for " + ti.getWaitedTime() + " ms");

    if (ti.isSuspended()) {
      sb.append(" (suspended)");
    }
    if (ti.isInNative()) {
      sb.append(" (running in native)");
    }
    sb.append(CRLF);
    sb.append(INDENT3 + "java.lang.Thread.State: " + ti.getThreadState());
    sb.append(CRLF);
    return sb.toString();
  }

  /**
   * Formats the thread dump for one thread.
   *
   * @param ti the ThreadInfo describing the thread
   * @return the formatted thread dump
   */
  private static String getThreadDump(ThreadInfo ti) {
    StringBuilder sb = new StringBuilder(getThreadDumpHeader(ti));
    for (LockInfo li : ti.getLockedSynchronizers()) {
      sb.append(INDENT2 + "locks " + li.toString() + CRLF);
    }
    boolean start = true;
    StackTraceElement[] stes = ti.getStackTrace();
    Object[] monitorDepths = new Object[stes.length];
    MonitorInfo[] mis = ti.getLockedMonitors();
    for (int i = 0; i < mis.length; i++) {
      monitorDepths[mis[i].getLockedStackDepth()] = mis[i];
    }
    for (int i = 0; i < stes.length; i++) {
      StackTraceElement ste = stes[i];
      sb.append(INDENT2 + "at " + ste.toString() + CRLF);
      if (start) {
        if (ti.getLockName() != null) {
          sb.append(INDENT2 + "- waiting on (a " + ti.getLockName() + ")");
          if (ti.getLockOwnerName() != null) {
            sb.append(" owned by " + ti.getLockOwnerName() + " Id=" + ti.getLockOwnerId());
          }
          sb.append(CRLF);
        }
        start = false;
      }
      if (monitorDepths[i] != null) {
        MonitorInfo mi = (MonitorInfo) monitorDepths[i];
        sb.append(
            INDENT2
                + "- locked (a "
                + mi.toString()
                + ")"
                + " index "
                + mi.getLockedStackDepth()
                + " frame "
                + mi.getLockedStackFrame().toString());
        sb.append(CRLF);
      }
    }
    return sb.toString();
  }

  /**
   * Formats the thread dump for a list of threads.
   *
   * @param tinfos the ThreadInfo array describing the thread list
   * @return the formatted thread dump
   */
  private static String getThreadDump(ThreadInfo[] tinfos) {
    StringBuilder sb = new StringBuilder();
    for (ThreadInfo tinfo : tinfos) {
      sb.append(getThreadDump(tinfo));
      sb.append(CRLF);
    }
    return sb.toString();
  }

  /**
   * Check if any threads are deadlocked. If any, print the thread dump for those threads.
   *
   * @return a deadlock message and the formatted thread dump of the deadlocked threads
   */
  private static String findDeadlock() {
    ThreadInfo[] tinfos = null;
    long[] ids = threadMXBean.findDeadlockedThreads();
    if (ids != null) {
      tinfos = threadMXBean.getThreadInfo(threadMXBean.findDeadlockedThreads(), true, true);
      if (tinfos != null) {
        StringBuilder sb = new StringBuilder("Deadlock found between the following threads:");
        sb.append(CRLF);
        sb.append(getThreadDump(tinfos));
        return sb.toString();
      }
    }
    return "";
  }

  /**
   * Retrieves a formatted JVM thread dump. The given list of locales will be used to retrieve a
   * StringManager.
   *
   * @param requestedLocales list of locales to use
   * @return the formatted JVM thread dump
   */
  public static String getThreadDump(Enumeration<Locale> requestedLocales) {
    return getThreadDump(StringManager.getManager(PACKAGE, requestedLocales));
  }

  /**
   * Retrieve a JVM thread dump formatted using the given StringManager.
   *
   * @param requestedSm the StringManager to use
   * @return the formatted JVM thread dump
   */
  private static String getThreadDump(StringManager requestedSm) {
    StringBuilder sb = new StringBuilder();

    synchronized (timeformat) {
      sb.append(timeformat.format(new Date()));
    }
    sb.append(CRLF);

    sb.append(requestedSm.getString("diagnostics.threadDumpTitle"));
    sb.append(" ");
    sb.append(runtimeMXBean.getVmName());
    sb.append(" (");
    sb.append(runtimeMXBean.getVmVersion());
    String vminfo = System.getProperty(vminfoSystemProperty);
    if (vminfo != null) {
      sb.append(" " + vminfo);
    }
    sb.append("):" + CRLF);
    sb.append(CRLF);

    ThreadInfo[] tis = threadMXBean.dumpAllThreads(true, true);
    sb.append(getThreadDump(tis));

    sb.append(findDeadlock());
    return sb.toString();
  }

  /**
   * Format contents of a MemoryUsage object.
   *
   * @param name a text prefix used in formatting
   * @param usage the MemoryUsage object to format
   * @return the formatted contents
   */
  private static String formatMemoryUsage(String name, MemoryUsage usage) {
    if (usage != null) {
      StringBuilder sb = new StringBuilder();
      sb.append(INDENT1 + name + " init: " + usage.getInit() + CRLF);
      sb.append(INDENT1 + name + " used: " + usage.getUsed() + CRLF);
      sb.append(INDENT1 + name + " committed: " + usage.getCommitted() + CRLF);
      sb.append(INDENT1 + name + " max: " + usage.getMax() + CRLF);
      return sb.toString();
    }
    return "";
  }

  /**
   * Retrieves a formatted JVM information text. The given list of locales will be used to retrieve
   * a StringManager.
   *
   * @param requestedLocales list of locales to use
   * @return the formatted JVM information text
   */
  public static String getVMInfo(Enumeration<Locale> requestedLocales) {
    return getVMInfo(StringManager.getManager(PACKAGE, requestedLocales));
  }

  /**
   * Retrieve a JVM information text formatted using the given StringManager.
   *
   * @param requestedSm the StringManager to use
   * @return the formatted JVM information text
   */
  private static String getVMInfo(StringManager requestedSm) {
    StringBuilder sb = new StringBuilder();

    synchronized (timeformat) {
      sb.append(timeformat.format(new Date()));
    }
    sb.append(CRLF);

    sb.append(requestedSm.getString("diagnostics.vmInfoRuntime"));
    sb.append(":" + CRLF);
    sb.append(INDENT1 + "vmName: " + runtimeMXBean.getVmName() + CRLF);
    sb.append(INDENT1 + "vmVersion: " + runtimeMXBean.getVmVersion() + CRLF);
    sb.append(INDENT1 + "vmVendor: " + runtimeMXBean.getVmVendor() + CRLF);
    sb.append(INDENT1 + "specName: " + runtimeMXBean.getSpecName() + CRLF);
    sb.append(INDENT1 + "specVersion: " + runtimeMXBean.getSpecVersion() + CRLF);
    sb.append(INDENT1 + "specVendor: " + runtimeMXBean.getSpecVendor() + CRLF);
    sb.append(
        INDENT1 + "managementSpecVersion: " + runtimeMXBean.getManagementSpecVersion() + CRLF);
    sb.append(INDENT1 + "name: " + runtimeMXBean.getName() + CRLF);
    sb.append(INDENT1 + "startTime: " + runtimeMXBean.getStartTime() + CRLF);
    sb.append(INDENT1 + "uptime: " + runtimeMXBean.getUptime() + CRLF);
    sb.append(
        INDENT1 + "isBootClassPathSupported: " + runtimeMXBean.isBootClassPathSupported() + CRLF);
    sb.append(CRLF);

    sb.append(requestedSm.getString("diagnostics.vmInfoOs"));
    sb.append(":" + CRLF);
    sb.append(INDENT1 + "name: " + operatingSystemMXBean.getName() + CRLF);
    sb.append(INDENT1 + "version: " + operatingSystemMXBean.getVersion() + CRLF);
    sb.append(INDENT1 + "architecture: " + operatingSystemMXBean.getArch() + CRLF);
    sb.append(
        INDENT1 + "availableProcessors: " + operatingSystemMXBean.getAvailableProcessors() + CRLF);
    sb.append(
        INDENT1 + "systemLoadAverage: " + operatingSystemMXBean.getSystemLoadAverage() + CRLF);
    sb.append(CRLF);

    sb.append(requestedSm.getString("diagnostics.vmInfoThreadMxBean"));
    sb.append(":" + CRLF);
    sb.append(
        INDENT1
            + "isCurrentThreadCpuTimeSupported: "
            + threadMXBean.isCurrentThreadCpuTimeSupported()
            + CRLF);
    sb.append(
        INDENT1 + "isThreadCpuTimeSupported: " + threadMXBean.isThreadCpuTimeSupported() + CRLF);
    sb.append(INDENT1 + "isThreadCpuTimeEnabled: " + threadMXBean.isThreadCpuTimeEnabled() + CRLF);
    sb.append(
        INDENT1
            + "isObjectMonitorUsageSupported: "
            + threadMXBean.isObjectMonitorUsageSupported()
            + CRLF);
    sb.append(
        INDENT1
            + "isSynchronizerUsageSupported: "
            + threadMXBean.isSynchronizerUsageSupported()
            + CRLF);
    sb.append(
        INDENT1
            + "isThreadContentionMonitoringSupported: "
            + threadMXBean.isThreadContentionMonitoringSupported()
            + CRLF);
    sb.append(
        INDENT1
            + "isThreadContentionMonitoringEnabled: "
            + threadMXBean.isThreadContentionMonitoringEnabled()
            + CRLF);
    sb.append(CRLF);

    sb.append(requestedSm.getString("diagnostics.vmInfoThreadCounts"));
    sb.append(":" + CRLF);
    sb.append(INDENT1 + "daemon: " + threadMXBean.getDaemonThreadCount() + CRLF);
    sb.append(INDENT1 + "total: " + threadMXBean.getThreadCount() + CRLF);
    sb.append(INDENT1 + "peak: " + threadMXBean.getPeakThreadCount() + CRLF);
    sb.append(INDENT1 + "totalStarted: " + threadMXBean.getTotalStartedThreadCount() + CRLF);
    sb.append(CRLF);

    sb.append(requestedSm.getString("diagnostics.vmInfoStartup"));
    sb.append(":" + CRLF);
    for (String arg : runtimeMXBean.getInputArguments()) {
      sb.append(INDENT1 + arg + CRLF);
    }
    sb.append(CRLF);

    sb.append(requestedSm.getString("diagnostics.vmInfoPath"));
    sb.append(":" + CRLF);
    sb.append(INDENT1 + "bootClassPath: " + runtimeMXBean.getBootClassPath() + CRLF);
    sb.append(INDENT1 + "classPath: " + runtimeMXBean.getClassPath() + CRLF);
    sb.append(INDENT1 + "libraryPath: " + runtimeMXBean.getLibraryPath() + CRLF);
    sb.append(CRLF);

    sb.append(requestedSm.getString("diagnostics.vmInfoClassLoading"));
    sb.append(":" + CRLF);
    sb.append(INDENT1 + "loaded: " + classLoadingMXBean.getLoadedClassCount() + CRLF);
    sb.append(INDENT1 + "unloaded: " + classLoadingMXBean.getUnloadedClassCount() + CRLF);
    sb.append(INDENT1 + "totalLoaded: " + classLoadingMXBean.getTotalLoadedClassCount() + CRLF);
    sb.append(INDENT1 + "isVerbose: " + classLoadingMXBean.isVerbose() + CRLF);
    sb.append(CRLF);

    sb.append(requestedSm.getString("diagnostics.vmInfoClassCompilation"));
    sb.append(":" + CRLF);
    sb.append(INDENT1 + "name: " + compilationMXBean.getName() + CRLF);
    sb.append(
        INDENT1 + "totalCompilationTime: " + compilationMXBean.getTotalCompilationTime() + CRLF);
    sb.append(
        INDENT1
            + "isCompilationTimeMonitoringSupported: "
            + compilationMXBean.isCompilationTimeMonitoringSupported()
            + CRLF);
    sb.append(CRLF);

    for (MemoryManagerMXBean mbean : memoryManagerMXBeans) {
      sb.append(requestedSm.getString("diagnostics.vmInfoMemoryManagers", mbean.getName()));
      sb.append(":" + CRLF);
      sb.append(INDENT1 + "isValid: " + mbean.isValid() + CRLF);
      sb.append(INDENT1 + "mbean.getMemoryPoolNames: " + CRLF);
      String[] names = mbean.getMemoryPoolNames();
      Arrays.sort(names);
      for (String name : names) {
        sb.append(INDENT2 + name + CRLF);
      }
      sb.append(CRLF);
    }

    for (GarbageCollectorMXBean mbean : garbageCollectorMXBeans) {
      sb.append(requestedSm.getString("diagnostics.vmInfoGarbageCollectors", mbean.getName()));
      sb.append(":" + CRLF);
      sb.append(INDENT1 + "isValid: " + mbean.isValid() + CRLF);
      sb.append(INDENT1 + "mbean.getMemoryPoolNames: " + CRLF);
      String[] names = mbean.getMemoryPoolNames();
      Arrays.sort(names);
      for (String name : names) {
        sb.append(INDENT2 + name + CRLF);
      }
      sb.append(INDENT1 + "getCollectionCount: " + mbean.getCollectionCount() + CRLF);
      sb.append(INDENT1 + "getCollectionTime: " + mbean.getCollectionTime() + CRLF);
      sb.append(CRLF);
    }

    sb.append(requestedSm.getString("diagnostics.vmInfoMemory"));
    sb.append(":" + CRLF);
    sb.append(INDENT1 + "isVerbose: " + memoryMXBean.isVerbose() + CRLF);
    sb.append(
        INDENT1
            + "getObjectPendingFinalizationCount: "
            + memoryMXBean.getObjectPendingFinalizationCount()
            + CRLF);
    sb.append(formatMemoryUsage("heap", memoryMXBean.getHeapMemoryUsage()));
    sb.append(formatMemoryUsage("non-heap", memoryMXBean.getNonHeapMemoryUsage()));
    sb.append(CRLF);

    for (MemoryPoolMXBean mbean : memoryPoolMXBeans) {
      sb.append(requestedSm.getString("diagnostics.vmInfoMemoryPools", mbean.getName()));
      sb.append(":" + CRLF);
      sb.append(INDENT1 + "isValid: " + mbean.isValid() + CRLF);
      sb.append(INDENT1 + "getType: " + mbean.getType() + CRLF);
      sb.append(INDENT1 + "mbean.getMemoryManagerNames: " + CRLF);
      String[] names = mbean.getMemoryManagerNames();
      Arrays.sort(names);
      for (String name : names) {
        sb.append(INDENT2 + name + CRLF);
      }
      sb.append(INDENT1 + "isUsageThresholdSupported: " + mbean.isUsageThresholdSupported() + CRLF);
      try {
        sb.append(INDENT1 + "isUsageThresholdExceeded: " + mbean.isUsageThresholdExceeded() + CRLF);
      } catch (UnsupportedOperationException ex) {
        // IGNORE
      }
      sb.append(
          INDENT1
              + "isCollectionUsageThresholdSupported: "
              + mbean.isCollectionUsageThresholdSupported()
              + CRLF);
      try {
        sb.append(
            INDENT1
                + "isCollectionUsageThresholdExceeded: "
                + mbean.isCollectionUsageThresholdExceeded()
                + CRLF);
      } catch (UnsupportedOperationException ex) {
        // IGNORE
      }
      try {
        sb.append(INDENT1 + "getUsageThreshold: " + mbean.getUsageThreshold() + CRLF);
      } catch (UnsupportedOperationException ex) {
        // IGNORE
      }
      try {
        sb.append(INDENT1 + "getUsageThresholdCount: " + mbean.getUsageThresholdCount() + CRLF);
      } catch (UnsupportedOperationException ex) {
        // IGNORE
      }
      try {
        sb.append(
            INDENT1 + "getCollectionUsageThreshold: " + mbean.getCollectionUsageThreshold() + CRLF);
      } catch (UnsupportedOperationException ex) {
        // IGNORE
      }
      try {
        sb.append(
            INDENT1
                + "getCollectionUsageThresholdCount: "
                + mbean.getCollectionUsageThresholdCount()
                + CRLF);
      } catch (UnsupportedOperationException ex) {
        // IGNORE
      }
      sb.append(formatMemoryUsage("current", mbean.getUsage()));
      sb.append(formatMemoryUsage("collection", mbean.getCollectionUsage()));
      sb.append(formatMemoryUsage("peak", mbean.getPeakUsage()));
      sb.append(CRLF);
    }

    sb.append(requestedSm.getString("diagnostics.vmInfoSystem"));
    sb.append(":" + CRLF);
    Map<String, String> props = runtimeMXBean.getSystemProperties();
    ArrayList<String> keys = new ArrayList<String>(props.keySet());
    Collections.sort(keys);
    for (String prop : keys) {
      sb.append(INDENT1 + prop + ": " + props.get(prop) + CRLF);
    }
    sb.append(CRLF);

    sb.append(requestedSm.getString("diagnostics.vmInfoLogger"));
    sb.append(":" + CRLF);
    List<String> loggers = loggingMXBean.getLoggerNames();
    Collections.sort(loggers);
    for (String logger : loggers) {
      sb.append(
          INDENT1
              + logger
              + ": level="
              + loggingMXBean.getLoggerLevel(logger)
              + ", parent="
              + loggingMXBean.getParentLoggerName(logger)
              + CRLF);
    }
    sb.append(CRLF);

    return sb.toString();
  }
}
  public synchronized void logGCInfo(AccumuloConfiguration conf) {
    final long now = System.currentTimeMillis();

    List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
    Runtime rt = Runtime.getRuntime();

    StringBuilder sb = new StringBuilder("gc");

    boolean sawChange = false;

    long maxIncreaseInCollectionTime = 0;

    for (GarbageCollectorMXBean gcBean : gcmBeans) {
      Long prevTime = prevGcTime.get(gcBean.getName());
      long pt = 0;
      if (prevTime != null) {
        pt = prevTime;
      }

      long time = gcBean.getCollectionTime();

      if (time - pt != 0) {
        sawChange = true;
      }

      long increaseInCollectionTime = time - pt;
      sb.append(
          String.format(
              " %s=%,.2f(+%,.2f) secs",
              gcBean.getName(), time / 1000.0, increaseInCollectionTime / 1000.0));
      maxIncreaseInCollectionTime = Math.max(increaseInCollectionTime, maxIncreaseInCollectionTime);
      prevGcTime.put(gcBean.getName(), time);
    }

    long mem = rt.freeMemory();
    if (maxIncreaseInCollectionTime == 0) {
      gcTimeIncreasedCount = 0;
    } else {
      gcTimeIncreasedCount++;
      if (gcTimeIncreasedCount > 3 && mem < rt.maxMemory() * 0.05) {
        log.warn("Running low on memory");
        gcTimeIncreasedCount = 0;
      }
    }

    if (mem > lastMemorySize) {
      sawChange = true;
    }

    String sign = "+";
    if (mem - lastMemorySize <= 0) {
      sign = "";
    }

    sb.append(
        String.format(
            " freemem=%,d(%s%,d) totalmem=%,d",
            mem, sign, (mem - lastMemorySize), rt.totalMemory()));

    if (sawChange) {
      log.debug(sb.toString());
    }

    final long keepAliveTimeout = conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT);
    if (lastMemoryCheckTime > 0 && lastMemoryCheckTime < now) {
      final long diff = now - lastMemoryCheckTime;
      if (diff > keepAliveTimeout) {
        log.warn(
            String.format(
                "GC pause checker not called in a timely fashion. Expected every %.1f seconds but was %.1f seconds since last check",
                keepAliveTimeout / 1000., diff / 1000.));
      }
      lastMemoryCheckTime = now;
      return;
    }

    if (maxIncreaseInCollectionTime > keepAliveTimeout) {
      Halt.halt("Garbage collection may be interfering with lock keep-alive.  Halting.", -1);
    }

    lastMemorySize = mem;
    lastMemoryCheckTime = now;
  }
  private void logStats() {
    long finishTime = System.currentTimeMillis();

    long finalGCTimes = 0;
    List<GarbageCollectorMXBean> gcmBeans = ManagementFactory.getGarbageCollectorMXBeans();
    for (GarbageCollectorMXBean garbageCollectorMXBean : gcmBeans) {
      finalGCTimes += garbageCollectorMXBean.getCollectionTime();
    }

    CompilationMXBean compMxBean = ManagementFactory.getCompilationMXBean();
    long finalCompileTimes = 0;
    if (compMxBean.isCompilationTimeMonitoringSupported()) {
      finalCompileTimes = compMxBean.getTotalCompilationTime();
    }

    double averageRate = totalSent.get() / (totalSendTime.get() / 1000.0);
    double overallRate = totalAdded / ((finishTime - startTime) / 1000.0);

    double finalSystemLoad = ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();

    if (log.isTraceEnabled()) {
      log.trace("");
      log.trace("TABLET SERVER BATCH WRITER STATISTICS");
      log.trace(String.format("Added                : %,10d mutations", totalAdded));
      log.trace(String.format("Sent                 : %,10d mutations", totalSent.get()));
      log.trace(
          String.format(
              "Resent percentage   : %10.2f%s",
              (totalSent.get() - totalAdded) / (double) totalAdded * 100.0, "%"));
      log.trace(
          String.format("Overall time         : %,10.2f secs", (finishTime - startTime) / 1000.0));
      log.trace(String.format("Overall send rate    : %,10.2f mutations/sec", overallRate));
      log.trace(
          String.format("Send efficiency      : %10.2f%s", overallRate / averageRate * 100.0, "%"));
      log.trace("");
      log.trace("BACKGROUND WRITER PROCESS STATISTICS");
      log.trace(
          String.format(
              "Total send time      : %,10.2f secs %6.2f%s",
              totalSendTime.get() / 1000.0,
              100.0 * totalSendTime.get() / (finishTime - startTime),
              "%"));
      log.trace(String.format("Average send rate    : %,10.2f mutations/sec", averageRate));
      log.trace(
          String.format(
              "Total bin time       : %,10.2f secs %6.2f%s",
              totalBinTime.get() / 1000.0,
              100.0 * totalBinTime.get() / (finishTime - startTime),
              "%"));
      log.trace(
          String.format(
              "Average bin rate     : %,10.2f mutations/sec",
              totalBinned.get() / (totalBinTime.get() / 1000.0)));
      log.trace(
          String.format(
              "tservers per batch   : %,8.2f avg  %,6d min %,6d max",
              (tabletServersBatchSum / (double) numBatches),
              minTabletServersBatch,
              maxTabletServersBatch));
      log.trace(
          String.format(
              "tablets per batch    : %,8.2f avg  %,6d min %,6d max",
              (tabletBatchSum / (double) numBatches), minTabletBatch, maxTabletBatch));
      log.trace("");
      log.trace("SYSTEM STATISTICS");
      log.trace(
          String.format(
              "JVM GC Time          : %,10.2f secs", ((finalGCTimes - initialGCTimes) / 1000.0)));
      if (compMxBean.isCompilationTimeMonitoringSupported()) {
        log.trace(
            String.format(
                "JVM Compile Time     : %,10.2f secs",
                ((finalCompileTimes - initialCompileTimes) / 1000.0)));
      }
      log.trace(
          String.format(
              "System load average : initial=%6.2f final=%6.2f",
              initialSystemLoad, finalSystemLoad));
    }
  }