private void doBadTserverList(HttpServletRequest req, StringBuilder sb) {
   if (Monitor.getMmi() != null && !Monitor.getMmi().badTServers.isEmpty()) {
     Table badTServerList =
         new Table("badtservers", "Non-Functioning Tablet Servers", "error");
     badTServerList.setSubCaption(
         "The following tablet servers reported a status other than Online.");
     badTServerList.addSortableColumn("Tablet Server");
     badTServerList.addSortableColumn("Tablet Server Status");
     for (Entry<String, Byte> badserver : Monitor.getMmi().badTServers.entrySet())
       badTServerList.addRow(
           badserver.getKey(), TabletServerState.getStateById(badserver.getValue()).name());
     badTServerList.generate(req, sb);
   }
 }
Exemple #2
0
 public void doGet(HttpServletRequest req, HttpServletResponse resp)
     throws ServletException, IOException {
   StringBuilder sb = new StringBuilder();
   try {
     Monitor.fetchData();
     bannerText =
         sanitize(ServerConfiguration.getSystemConfiguration().get(Property.MONITOR_BANNER_TEXT));
     bannerColor =
         ServerConfiguration.getSystemConfiguration()
             .get(Property.MONITOR_BANNER_COLOR)
             .replace("'", "&#39;");
     bannerBackground =
         ServerConfiguration.getSystemConfiguration()
             .get(Property.MONITOR_BANNER_BACKGROUND)
             .replace("'", "&#39;");
     pageStart(req, resp, sb);
     pageBody(req, resp, sb);
     pageEnd(req, resp, sb);
   } catch (Throwable t) {
     log.error("Error building page " + req.getRequestURI(), t);
     sb.append("\n<pre>\n");
     StringWriter sw = new StringWriter();
     t.printStackTrace(new PrintWriter(sw));
     sb.append(sanitize(sw.getBuffer().toString()));
     sb.append("</pre>\n");
   } finally {
     resp.getWriter().print(sb);
     resp.getWriter().flush();
   }
 }
 private void doDeadTserverList(HttpServletRequest req, StringBuilder sb) {
   MasterMonitorInfo mmi = Monitor.getMmi();
   if (mmi != null) {
     List<DeadServer> obit = mmi.deadTabletServers;
     Table deadTServerList = new Table("deaddtservers", "Dead&nbsp;Tablet&nbsp;Servers", "error");
     deadTServerList.setSubCaption("The following tablet servers are no longer reachable.");
     doDeadServerTable(req, sb, deadTServerList, obit);
   }
 }
Exemple #4
0
  protected void pageStart(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb)
      throws Exception {
    resp.setContentType(DEFAULT_CONTENT_TYPE);
    int refresh = -1;
    Cookie c = getCookie(req, "page.refresh.rate");
    if (c != null && c.getValue() != null) {
      try {
        refresh = Integer.parseInt(c.getValue());
      } catch (NumberFormatException e) {
        // ignore improperly formatted user cookie
      }
    }
    synchronized (BasicServlet.class) {
      // Learn our instance name asynchronously so we don't hang up if zookeeper is down
      if (cachedInstanceName == null) {
        SimpleTimer.getInstance()
            .schedule(
                new TimerTask() {
                  @Override
                  public void run() {
                    synchronized (BasicServlet.class) {
                      if (cachedInstanceName == null) {
                        cachedInstanceName = HdfsZooInstance.getInstance().getInstanceName();
                      }
                    }
                  }
                },
                1000);
      }
    }

    // BEGIN PAGE
    sb.append("<html>\n");

    // BEGIN HEADER
    sb.append("<head>\n");
    sb.append("<title>")
        .append(getTitle(req))
        .append(" - Accumulo ")
        .append(Constants.VERSION)
        .append("</title>\n");
    if ((refresh > 0)
        && (req.getRequestURI().startsWith("/docs") == false)
        && (req.getRequestURI().startsWith("/vis") == false))
      sb.append("<meta http-equiv='refresh' content='" + refresh + "' />\n");
    sb.append("<meta http-equiv='Content-Type' content='")
        .append(DEFAULT_CONTENT_TYPE)
        .append("' />\n");
    sb.append("<meta http-equiv='Content-Script-Type' content='text/javascript' />\n");
    sb.append("<meta http-equiv='Content-Style-Type' content='text/css' />\n");
    sb.append("<link rel='shortcut icon' type='image/jpg' href='/web/favicon.png' />\n");
    sb.append("<link rel='stylesheet' type='text/css' href='/web/screen.css' media='screen' />\n");
    sb.append("<script src='/web/functions.js' type='text/javascript'></script>\n");

    sb.append(
        "<!--[if lte IE 8]><script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/excanvas.min.js\"></script><![endif]-->\n");
    sb.append(
        "<script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/jquery.js\"></script>\n");
    sb.append(
        "<script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/jquery.flot.js\"></script>\n");

    sb.append("</head>\n");

    // BEGIN BODY OPENING
    sb.append("\n<body>\n");
    sb.append("<div id='content-wrapper'>\n");
    sb.append("<div id='content'>\n");
    sb.append("<div id='header'>");
    if (!bannerText.isEmpty()) {
      sb.append("<div id='banner' style='color:")
          .append(bannerColor)
          .append(";background:")
          .append(bannerBackground)
          .append("'>")
          .append(bannerText)
          .append("</div>\n");
    }
    sb.append("<div id='headertitle'>");
    sb.append("<h1>").append(getTitle(req)).append("</h1></div>\n");
    sb.append("<div id='subheader'>Instance&nbsp;Name:&nbsp;")
        .append(cachedInstanceName)
        .append("&nbsp;&nbsp;&nbsp;Version:&nbsp;")
        .append(Constants.VERSION)
        .append("\n");
    sb.append("<br><span class='smalltext'>Instance&nbsp;ID:&nbsp;")
        .append(HdfsZooInstance.getInstance().getInstanceID())
        .append("</span>\n");
    sb.append("<br><span class='smalltext'>")
        .append(new Date().toString().replace(" ", "&nbsp;"))
        .append("</span>");
    sb.append("</div>\n"); // end <div id='subheader'>
    sb.append("</div>\n"); // end <div id='header'>

    // BEGIN LEFT SIDE
    sb.append("<div id='nav'>\n");
    sb.append("<span id='nav-title'><a href='/'>Overview</a></span><br />\n");
    sb.append("<hr />\n");
    sb.append("<a href='/master'>Master&nbsp;Server</a><br />\n");
    sb.append("<a href='/tservers'>Tablet&nbsp;Servers</a><br />\n");
    sb.append("<a href='/loggers'>Logger&nbsp;Servers</a><br />\n");
    sb.append("<a href='/vis'>Server Activity</a><br />\n");
    sb.append("<a href='/gc'>Garbage&nbsp;Collector</a><br />\n");
    sb.append("<a href='/tables'>Tables</a><br />\n");
    sb.append("<a href='/trace/summary?minutes=10'>Recent&nbsp;Traces</a><br />\n");
    sb.append("<a href='/docs'>Documentation</a><br />\n");
    int numLogs = LogService.getInstance().getEvents().size();
    if (numLogs > 0)
      sb.append(
          "<span class='error'><a href='/log'>Recent&nbsp;Logs&nbsp;<span class='smalltext'>("
              + numLogs
              + ")</a></span></span><br />\n");
    int numProblems = Monitor.getProblemSummary().entrySet().size();
    if (numProblems > 0)
      sb.append(
          "<span class='error'><a href='/problems'>Table&nbsp;Problems&nbsp;<span class='smalltext'>("
              + numProblems
              + ")</a></span></span><br />\n");
    sb.append("<hr />\n");
    sb.append("<a href='/xml'>XML</a><hr />\n");
    sb.append("<div class='smalltext'>[<a href='")
        .append("/op?action=refresh&value=")
        .append(refresh < 1 ? "5" : "-1");
    sb.append("&redir=").append(currentPage(req)).append("'>");
    sb.append(refresh < 1 ? "en" : "dis").append("able&nbsp;auto-refresh</a>]</div>\n");
    sb.append("</div>\n"); // end <div id='nav'>

    sb.append("<div id='main'");
    if (bannerText.isEmpty()) sb.append(" style='bottom:0'");
    sb.append(">\n");
    sb.append("<!-- BEGIN MAIN BODY CONTENT -->\n\n");
  }
  @Override
  protected void pageBody(HttpServletRequest req, HttpServletResponse response, StringBuilder sb)
      throws Exception {
    String tserverAddress = req.getParameter("s");

    // Check to make sure tserver is a known address
    boolean tserverExists = false;
    if (tserverAddress != null && tserverAddress.isEmpty() == false) {
      for (TabletServerStatus ts : Monitor.getMmi().getTServerInfo()) {
        if (tserverAddress.equals(ts.getName())) {
          tserverExists = true;
          break;
        }
      }
    }

    if (tserverAddress == null || tserverAddress.isEmpty() || tserverExists == false) {
      doBadTserverList(req, sb);

      doDeadTserverList(req, sb);

      ArrayList<TabletServerStatus> tservers = new ArrayList<TabletServerStatus>();
      if (Monitor.getMmi() != null) tservers.addAll(Monitor.getMmi().tServerInfo);

      Table tServerList = new Table("tservers", "Tablet&nbsp;Servers");
      tServerList.setSubCaption(
          "Click on the <span style='color: #0000ff;'>server address</span> to view detailed performance statistics for that server.");

      doTserverList(req, sb, tservers, null, tServerList);
      return;
    }

    double totalElapsedForAll = 0;
    double splitStdDev = 0;
    double minorStdDev = 0;
    double minorQueueStdDev = 0;
    double majorStdDev = 0;
    double majorQueueStdDev = 0;
    double currentMinorAvg = 0;
    double currentMajorAvg = 0;
    double currentMinorStdDev = 0;
    double currentMajorStdDev = 0;
    TabletStats total =
        new TabletStats(null, new ActionStats(), new ActionStats(), new ActionStats(), 0, 0, 0, 0);

    InetSocketAddress address = AddressUtil.parseAddress(tserverAddress, -1);
    TabletStats historical =
        new TabletStats(null, new ActionStats(), new ActionStats(), new ActionStats(), 0, 0, 0, 0);
    List<TabletStats> tsStats = new ArrayList<TabletStats>();
    try {
      TabletClientService.Client client =
          ThriftUtil.getClient(
              new TabletClientService.Client.Factory(), address, Monitor.getSystemConfiguration());
      try {
        for (String tableId : Monitor.getMmi().tableMap.keySet()) {
          tsStats.addAll(
              client.getTabletStats(
                  Tracer.traceInfo(),
                  SystemCredentials.get().toThrift(Monitor.getInstance()),
                  tableId));
        }
        historical =
            client.getHistoricalStats(
                Tracer.traceInfo(), SystemCredentials.get().toThrift(Monitor.getInstance()));
      } finally {
        ThriftUtil.returnClient(client);
      }
    } catch (Exception e) {
      banner(sb, "error", "No Such Tablet ServerAvailable");
      log.error(e, e);
      return;
    }

    Table perTabletResults = new Table("perTabletResults", "Detailed&nbsp;Current&nbsp;Operations");
    perTabletResults.setSubCaption("Per-tablet&nbsp;Details");
    perTabletResults.addSortableColumn("Table", new TableLinkType(), null);
    perTabletResults.addSortableColumn("Tablet");
    perTabletResults.addSortableColumn("Entries", new NumberType<Long>(), null);
    perTabletResults.addSortableColumn("Ingest", new NumberType<Long>(), null);
    perTabletResults.addSortableColumn("Query", new NumberType<Long>(), null);
    perTabletResults.addSortableColumn("Minor&nbsp;Avg", new SecondType(), null);
    perTabletResults.addSortableColumn("Minor&nbsp;Std&nbsp;Dev", new SecondType(), null);
    perTabletResults.addSortableColumn("Minor&nbsp;Avg&nbsp;e/s", new NumberType<Double>(), null);
    perTabletResults.addSortableColumn("Major&nbsp;Avg", new SecondType(), null);
    perTabletResults.addSortableColumn("Major&nbsp;Std&nbsp;Dev", new SecondType(), null);
    perTabletResults.addSortableColumn("Major&nbsp;Avg&nbsp;e/s", new NumberType<Double>(), null);

    for (TabletStats info : tsStats) {
      if (info.extent == null) {
        historical = info;
        continue;
      }
      total.numEntries += info.numEntries;
      TabletStatsKeeper.update(total.minors, info.minors);
      TabletStatsKeeper.update(total.majors, info.majors);

      KeyExtent extent = new KeyExtent(info.extent);
      String tableId = extent.getTableId().toString();
      MessageDigest digester = MessageDigest.getInstance("MD5");
      if (extent.getEndRow() != null && extent.getEndRow().getLength() > 0) {
        digester.update(extent.getEndRow().getBytes(), 0, extent.getEndRow().getLength());
      }
      String obscuredExtent = new String(Base64.encodeBase64(digester.digest()));
      String displayExtent = String.format("<code>[%s]</code>", obscuredExtent);

      TableRow row = perTabletResults.prepareRow();
      row.add(tableId);
      row.add(displayExtent);
      row.add(info.numEntries);
      row.add(info.ingestRate);
      row.add(info.queryRate);
      row.add(info.minors.num != 0 ? info.minors.elapsed / info.minors.num : null);
      row.add(stddev(info.minors.elapsed, info.minors.num, info.minors.sumDev));
      row.add(info.minors.elapsed != 0 ? info.minors.count / info.minors.elapsed : null);
      row.add(info.majors.num != 0 ? info.majors.elapsed / info.majors.num : null);
      row.add(stddev(info.majors.elapsed, info.majors.num, info.majors.sumDev));
      row.add(info.majors.elapsed != 0 ? info.majors.count / info.majors.elapsed : null);
      perTabletResults.addRow(row);
    }

    // Calculate current averages oldServer adding in historical data
    if (total.minors.num != 0) currentMinorAvg = (long) (total.minors.elapsed / total.minors.num);
    if (total.minors.elapsed != 0 && total.minors.num != 0)
      currentMinorStdDev = stddev(total.minors.elapsed, total.minors.num, total.minors.sumDev);
    if (total.majors.num != 0) currentMajorAvg = total.majors.elapsed / total.majors.num;
    if (total.majors.elapsed != 0
        && total.majors.num != 0
        && total.majors.elapsed > total.majors.num)
      currentMajorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);

    // After these += operations, these variables are now total for current
    // tablets and historical tablets
    TabletStatsKeeper.update(total.minors, historical.minors);
    TabletStatsKeeper.update(total.majors, historical.majors);
    totalElapsedForAll += total.majors.elapsed + historical.splits.elapsed + total.minors.elapsed;

    minorStdDev = stddev(total.minors.elapsed, total.minors.num, total.minors.sumDev);
    minorQueueStdDev = stddev(total.minors.queueTime, total.minors.num, total.minors.queueSumDev);
    majorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);
    majorQueueStdDev = stddev(total.majors.queueTime, total.majors.num, total.majors.queueSumDev);
    splitStdDev =
        stddev(historical.splits.num, historical.splits.elapsed, historical.splits.sumDev);

    doDetailTable(req, sb, address, tsStats.size(), total, historical);
    doAllTimeTable(
        req,
        sb,
        total,
        historical,
        majorQueueStdDev,
        minorQueueStdDev,
        totalElapsedForAll,
        splitStdDev,
        majorStdDev,
        minorStdDev);
    doCurrentTabletOps(
        req, sb, currentMinorAvg, currentMinorStdDev, currentMajorAvg, currentMajorStdDev);
    perTabletResults.generate(req, sb);
  }
  static void doTserverList(
      HttpServletRequest req,
      StringBuilder sb,
      List<TabletServerStatus> tservers,
      String tableId,
      Table tServerList) {
    int guessHighLoad = ManagementFactory.getOperatingSystemMXBean().getAvailableProcessors();
    long now = System.currentTimeMillis();

    double avgLastContact = 0.;
    for (TabletServerStatus status : tservers) {
      avgLastContact += (now - status.lastContact);
    }
    final long MINUTES = 3 * 60 * 1000;
    tServerList.addSortableColumn("Server", new TServerLinkType(), null);
    tServerList.addSortableColumn(
        "Hosted&nbsp;Tablets", new NumberType<Integer>(0, Integer.MAX_VALUE), null);
    tServerList.addSortableColumn(
        "Last&nbsp;Contact",
        new DurationType(0l, (long) Math.min(avgLastContact * 4, MINUTES)),
        null);
    tServerList.addSortableColumn(
        "Entries", new NumberType<Long>(), "The number of key/value pairs.");
    tServerList.addSortableColumn(
        "Ingest",
        new NumberType<Long>(),
        "The number of key/value pairs inserted. (Note that deletes are also 'inserted')");
    tServerList.addSortableColumn(
        "Query",
        new NumberType<Long>(),
        "The number of key/value pairs returned to clients. (Not the number of scans)");
    tServerList.addSortableColumn(
        "Hold&nbsp;Time",
        new DurationType(),
        "The amount of time ingest is suspended waiting for data to be written to disk.");
    tServerList.addSortableColumn(
        "Running<br />Scans",
        new CompactionsType("scans"),
        "The number of scans running and queued on this tablet server.");
    tServerList.addSortableColumn(
        "Minor<br />Compactions",
        new CompactionsType("minor"),
        "The number of minor compactions running and (queued waiting for resources). Minor compactions are the operations where entries are flushed from memory to disk.");
    tServerList.addSortableColumn(
        "Major<br />Compactions",
        new CompactionsType("major"),
        "The number of major compactions running and (queued waiting for resources). "
            + "Major compactions are the operations where many smaller files are grouped into a larger file, eliminating duplicates and cleaning up deletes.");
    tServerList.addSortableColumn(
        "Index Cache<br />Hit Rate", new PercentageType(), "The recent index cache hit rate.");
    tServerList.addSortableColumn(
        "Data Cache<br />Hit Rate", new PercentageType(), "The recent data cache hit rate.");
    tServerList.addSortableColumn(
        "OS&nbsp;Load",
        new NumberType<Double>(0., guessHighLoad * 1., 0., guessHighLoad * 3.),
        "The Unix one minute load average. The average number of processes in the run queue over a one minute interval.");

    log.debug("tableId: " + tableId);
    for (TabletServerStatus status : tservers) {
      if (status == null) status = NO_STATUS;
      TableInfo summary = Monitor.summarizeTableStats(status);
      if (tableId != null) summary = status.tableMap.get(tableId);
      if (summary == null) continue;
      TableRow row = tServerList.prepareRow();
      row.add(status); // add for server name
      row.add(summary.tablets);
      row.add(now - status.lastContact);
      row.add(summary.recs);
      row.add(summary.ingestRate);
      row.add(summary.queryRate);
      row.add(status.holdTime);
      row.add(summary); // add for scans
      row.add(summary); // add for minor compactions
      row.add(summary); // add for major compactions
      double indexCacheHitRate =
          status.indexCacheHits / (double) Math.max(status.indexCacheRequest, 1);
      row.add(indexCacheHitRate);
      double dataCacheHitRate =
          status.dataCacheHits / (double) Math.max(status.dataCacheRequest, 1);
      row.add(dataCacheHitRate);
      row.add(status.osLoad);
      tServerList.addRow(row);
    }
    tServerList.generate(req, sb);
  }