private void doBadTserverList(HttpServletRequest req, StringBuilder sb) {
if (Monitor.getMmi() != null && !Monitor.getMmi().badTServers.isEmpty()) {
Table badTServerList =
new Table("badtservers", "Non-Functioning Tablet Servers", "error");
badTServerList.setSubCaption(
"The following tablet servers reported a status other than Online.");
badTServerList.addSortableColumn("Tablet Server");
badTServerList.addSortableColumn("Tablet Server Status");
for (Entry<String, Byte> badserver : Monitor.getMmi().badTServers.entrySet())
badTServerList.addRow(
badserver.getKey(), TabletServerState.getStateById(badserver.getValue()).name());
badTServerList.generate(req, sb);
}
}
public void doGet(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
StringBuilder sb = new StringBuilder();
try {
Monitor.fetchData();
bannerText =
sanitize(ServerConfiguration.getSystemConfiguration().get(Property.MONITOR_BANNER_TEXT));
bannerColor =
ServerConfiguration.getSystemConfiguration()
.get(Property.MONITOR_BANNER_COLOR)
.replace("'", "'");
bannerBackground =
ServerConfiguration.getSystemConfiguration()
.get(Property.MONITOR_BANNER_BACKGROUND)
.replace("'", "'");
pageStart(req, resp, sb);
pageBody(req, resp, sb);
pageEnd(req, resp, sb);
} catch (Throwable t) {
log.error("Error building page " + req.getRequestURI(), t);
sb.append("\n<pre>\n");
StringWriter sw = new StringWriter();
t.printStackTrace(new PrintWriter(sw));
sb.append(sanitize(sw.getBuffer().toString()));
sb.append("</pre>\n");
} finally {
resp.getWriter().print(sb);
resp.getWriter().flush();
}
}
private void doDeadTserverList(HttpServletRequest req, StringBuilder sb) {
MasterMonitorInfo mmi = Monitor.getMmi();
if (mmi != null) {
List<DeadServer> obit = mmi.deadTabletServers;
Table deadTServerList = new Table("deaddtservers", "Dead Tablet Servers", "error");
deadTServerList.setSubCaption("The following tablet servers are no longer reachable.");
doDeadServerTable(req, sb, deadTServerList, obit);
}
}
protected void pageStart(HttpServletRequest req, HttpServletResponse resp, StringBuilder sb)
throws Exception {
resp.setContentType(DEFAULT_CONTENT_TYPE);
int refresh = -1;
Cookie c = getCookie(req, "page.refresh.rate");
if (c != null && c.getValue() != null) {
try {
refresh = Integer.parseInt(c.getValue());
} catch (NumberFormatException e) {
// ignore improperly formatted user cookie
}
}
synchronized (BasicServlet.class) {
// Learn our instance name asynchronously so we don't hang up if zookeeper is down
if (cachedInstanceName == null) {
SimpleTimer.getInstance()
.schedule(
new TimerTask() {
@Override
public void run() {
synchronized (BasicServlet.class) {
if (cachedInstanceName == null) {
cachedInstanceName = HdfsZooInstance.getInstance().getInstanceName();
}
}
}
},
1000);
}
}
// BEGIN PAGE
sb.append("<html>\n");
// BEGIN HEADER
sb.append("<head>\n");
sb.append("<title>")
.append(getTitle(req))
.append(" - Accumulo ")
.append(Constants.VERSION)
.append("</title>\n");
if ((refresh > 0)
&& (req.getRequestURI().startsWith("/docs") == false)
&& (req.getRequestURI().startsWith("/vis") == false))
sb.append("<meta http-equiv='refresh' content='" + refresh + "' />\n");
sb.append("<meta http-equiv='Content-Type' content='")
.append(DEFAULT_CONTENT_TYPE)
.append("' />\n");
sb.append("<meta http-equiv='Content-Script-Type' content='text/javascript' />\n");
sb.append("<meta http-equiv='Content-Style-Type' content='text/css' />\n");
sb.append("<link rel='shortcut icon' type='image/jpg' href='/web/favicon.png' />\n");
sb.append("<link rel='stylesheet' type='text/css' href='/web/screen.css' media='screen' />\n");
sb.append("<script src='/web/functions.js' type='text/javascript'></script>\n");
sb.append(
"<!--[if lte IE 8]><script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/excanvas.min.js\"></script><![endif]-->\n");
sb.append(
"<script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/jquery.js\"></script>\n");
sb.append(
"<script language=\"javascript\" type=\"text/javascript\" src=\"/web/flot/jquery.flot.js\"></script>\n");
sb.append("</head>\n");
// BEGIN BODY OPENING
sb.append("\n<body>\n");
sb.append("<div id='content-wrapper'>\n");
sb.append("<div id='content'>\n");
sb.append("<div id='header'>");
if (!bannerText.isEmpty()) {
sb.append("<div id='banner' style='color:")
.append(bannerColor)
.append(";background:")
.append(bannerBackground)
.append("'>")
.append(bannerText)
.append("</div>\n");
}
sb.append("<div id='headertitle'>");
sb.append("<h1>").append(getTitle(req)).append("</h1></div>\n");
sb.append("<div id='subheader'>Instance Name: ")
.append(cachedInstanceName)
.append(" Version: ")
.append(Constants.VERSION)
.append("\n");
sb.append("<br><span class='smalltext'>Instance ID: ")
.append(HdfsZooInstance.getInstance().getInstanceID())
.append("</span>\n");
sb.append("<br><span class='smalltext'>")
.append(new Date().toString().replace(" ", " "))
.append("</span>");
sb.append("</div>\n"); // end <div id='subheader'>
sb.append("</div>\n"); // end <div id='header'>
// BEGIN LEFT SIDE
sb.append("<div id='nav'>\n");
sb.append("<span id='nav-title'><a href='/'>Overview</a></span><br />\n");
sb.append("<hr />\n");
sb.append("<a href='/master'>Master Server</a><br />\n");
sb.append("<a href='/tservers'>Tablet Servers</a><br />\n");
sb.append("<a href='/loggers'>Logger Servers</a><br />\n");
sb.append("<a href='/vis'>Server Activity</a><br />\n");
sb.append("<a href='/gc'>Garbage Collector</a><br />\n");
sb.append("<a href='/tables'>Tables</a><br />\n");
sb.append("<a href='/trace/summary?minutes=10'>Recent Traces</a><br />\n");
sb.append("<a href='/docs'>Documentation</a><br />\n");
int numLogs = LogService.getInstance().getEvents().size();
if (numLogs > 0)
sb.append(
"<span class='error'><a href='/log'>Recent Logs <span class='smalltext'>("
+ numLogs
+ ")</a></span></span><br />\n");
int numProblems = Monitor.getProblemSummary().entrySet().size();
if (numProblems > 0)
sb.append(
"<span class='error'><a href='/problems'>Table Problems <span class='smalltext'>("
+ numProblems
+ ")</a></span></span><br />\n");
sb.append("<hr />\n");
sb.append("<a href='/xml'>XML</a><hr />\n");
sb.append("<div class='smalltext'>[<a href='")
.append("/op?action=refresh&value=")
.append(refresh < 1 ? "5" : "-1");
sb.append("&redir=").append(currentPage(req)).append("'>");
sb.append(refresh < 1 ? "en" : "dis").append("able auto-refresh</a>]</div>\n");
sb.append("</div>\n"); // end <div id='nav'>
sb.append("<div id='main'");
if (bannerText.isEmpty()) sb.append(" style='bottom:0'");
sb.append(">\n");
sb.append("<!-- BEGIN MAIN BODY CONTENT -->\n\n");
}
@Override
protected void pageBody(HttpServletRequest req, HttpServletResponse response, StringBuilder sb)
throws Exception {
String tserverAddress = req.getParameter("s");
// Check to make sure tserver is a known address
boolean tserverExists = false;
if (tserverAddress != null && tserverAddress.isEmpty() == false) {
for (TabletServerStatus ts : Monitor.getMmi().getTServerInfo()) {
if (tserverAddress.equals(ts.getName())) {
tserverExists = true;
break;
}
}
}
if (tserverAddress == null || tserverAddress.isEmpty() || tserverExists == false) {
doBadTserverList(req, sb);
doDeadTserverList(req, sb);
ArrayList<TabletServerStatus> tservers = new ArrayList<TabletServerStatus>();
if (Monitor.getMmi() != null) tservers.addAll(Monitor.getMmi().tServerInfo);
Table tServerList = new Table("tservers", "Tablet Servers");
tServerList.setSubCaption(
"Click on the <span style='color: #0000ff;'>server address</span> to view detailed performance statistics for that server.");
doTserverList(req, sb, tservers, null, tServerList);
return;
}
double totalElapsedForAll = 0;
double splitStdDev = 0;
double minorStdDev = 0;
double minorQueueStdDev = 0;
double majorStdDev = 0;
double majorQueueStdDev = 0;
double currentMinorAvg = 0;
double currentMajorAvg = 0;
double currentMinorStdDev = 0;
double currentMajorStdDev = 0;
TabletStats total =
new TabletStats(null, new ActionStats(), new ActionStats(), new ActionStats(), 0, 0, 0, 0);
InetSocketAddress address = AddressUtil.parseAddress(tserverAddress, -1);
TabletStats historical =
new TabletStats(null, new ActionStats(), new ActionStats(), new ActionStats(), 0, 0, 0, 0);
List<TabletStats> tsStats = new ArrayList<TabletStats>();
try {
TabletClientService.Client client =
ThriftUtil.getClient(
new TabletClientService.Client.Factory(), address, Monitor.getSystemConfiguration());
try {
for (String tableId : Monitor.getMmi().tableMap.keySet()) {
tsStats.addAll(
client.getTabletStats(
Tracer.traceInfo(),
SystemCredentials.get().toThrift(Monitor.getInstance()),
tableId));
}
historical =
client.getHistoricalStats(
Tracer.traceInfo(), SystemCredentials.get().toThrift(Monitor.getInstance()));
} finally {
ThriftUtil.returnClient(client);
}
} catch (Exception e) {
banner(sb, "error", "No Such Tablet ServerAvailable");
log.error(e, e);
return;
}
Table perTabletResults = new Table("perTabletResults", "Detailed Current Operations");
perTabletResults.setSubCaption("Per-tablet Details");
perTabletResults.addSortableColumn("Table", new TableLinkType(), null);
perTabletResults.addSortableColumn("Tablet");
perTabletResults.addSortableColumn("Entries", new NumberType<Long>(), null);
perTabletResults.addSortableColumn("Ingest", new NumberType<Long>(), null);
perTabletResults.addSortableColumn("Query", new NumberType<Long>(), null);
perTabletResults.addSortableColumn("Minor Avg", new SecondType(), null);
perTabletResults.addSortableColumn("Minor Std Dev", new SecondType(), null);
perTabletResults.addSortableColumn("Minor Avg e/s", new NumberType<Double>(), null);
perTabletResults.addSortableColumn("Major Avg", new SecondType(), null);
perTabletResults.addSortableColumn("Major Std Dev", new SecondType(), null);
perTabletResults.addSortableColumn("Major Avg e/s", new NumberType<Double>(), null);
for (TabletStats info : tsStats) {
if (info.extent == null) {
historical = info;
continue;
}
total.numEntries += info.numEntries;
TabletStatsKeeper.update(total.minors, info.minors);
TabletStatsKeeper.update(total.majors, info.majors);
KeyExtent extent = new KeyExtent(info.extent);
String tableId = extent.getTableId().toString();
MessageDigest digester = MessageDigest.getInstance("MD5");
if (extent.getEndRow() != null && extent.getEndRow().getLength() > 0) {
digester.update(extent.getEndRow().getBytes(), 0, extent.getEndRow().getLength());
}
String obscuredExtent = new String(Base64.encodeBase64(digester.digest()));
String displayExtent = String.format("<code>[%s]</code>", obscuredExtent);
TableRow row = perTabletResults.prepareRow();
row.add(tableId);
row.add(displayExtent);
row.add(info.numEntries);
row.add(info.ingestRate);
row.add(info.queryRate);
row.add(info.minors.num != 0 ? info.minors.elapsed / info.minors.num : null);
row.add(stddev(info.minors.elapsed, info.minors.num, info.minors.sumDev));
row.add(info.minors.elapsed != 0 ? info.minors.count / info.minors.elapsed : null);
row.add(info.majors.num != 0 ? info.majors.elapsed / info.majors.num : null);
row.add(stddev(info.majors.elapsed, info.majors.num, info.majors.sumDev));
row.add(info.majors.elapsed != 0 ? info.majors.count / info.majors.elapsed : null);
perTabletResults.addRow(row);
}
// Calculate current averages oldServer adding in historical data
if (total.minors.num != 0) currentMinorAvg = (long) (total.minors.elapsed / total.minors.num);
if (total.minors.elapsed != 0 && total.minors.num != 0)
currentMinorStdDev = stddev(total.minors.elapsed, total.minors.num, total.minors.sumDev);
if (total.majors.num != 0) currentMajorAvg = total.majors.elapsed / total.majors.num;
if (total.majors.elapsed != 0
&& total.majors.num != 0
&& total.majors.elapsed > total.majors.num)
currentMajorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);
// After these += operations, these variables are now total for current
// tablets and historical tablets
TabletStatsKeeper.update(total.minors, historical.minors);
TabletStatsKeeper.update(total.majors, historical.majors);
totalElapsedForAll += total.majors.elapsed + historical.splits.elapsed + total.minors.elapsed;
minorStdDev = stddev(total.minors.elapsed, total.minors.num, total.minors.sumDev);
minorQueueStdDev = stddev(total.minors.queueTime, total.minors.num, total.minors.queueSumDev);
majorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);
majorQueueStdDev = stddev(total.majors.queueTime, total.majors.num, total.majors.queueSumDev);
splitStdDev =
stddev(historical.splits.num, historical.splits.elapsed, historical.splits.sumDev);
doDetailTable(req, sb, address, tsStats.size(), total, historical);
doAllTimeTable(
req,
sb,
total,
historical,
majorQueueStdDev,
minorQueueStdDev,
totalElapsedForAll,
splitStdDev,
majorStdDev,
minorStdDev);
doCurrentTabletOps(
req, sb, currentMinorAvg, currentMinorStdDev, currentMajorAvg, currentMajorStdDev);
perTabletResults.generate(req, sb);
}
static void doTserverList(
HttpServletRequest req,
StringBuilder sb,
List<TabletServerStatus> tservers,
String tableId,
Table tServerList) {
int guessHighLoad = ManagementFactory.getOperatingSystemMXBean().getAvailableProcessors();
long now = System.currentTimeMillis();
double avgLastContact = 0.;
for (TabletServerStatus status : tservers) {
avgLastContact += (now - status.lastContact);
}
final long MINUTES = 3 * 60 * 1000;
tServerList.addSortableColumn("Server", new TServerLinkType(), null);
tServerList.addSortableColumn(
"Hosted Tablets", new NumberType<Integer>(0, Integer.MAX_VALUE), null);
tServerList.addSortableColumn(
"Last Contact",
new DurationType(0l, (long) Math.min(avgLastContact * 4, MINUTES)),
null);
tServerList.addSortableColumn(
"Entries", new NumberType<Long>(), "The number of key/value pairs.");
tServerList.addSortableColumn(
"Ingest",
new NumberType<Long>(),
"The number of key/value pairs inserted. (Note that deletes are also 'inserted')");
tServerList.addSortableColumn(
"Query",
new NumberType<Long>(),
"The number of key/value pairs returned to clients. (Not the number of scans)");
tServerList.addSortableColumn(
"Hold Time",
new DurationType(),
"The amount of time ingest is suspended waiting for data to be written to disk.");
tServerList.addSortableColumn(
"Running<br />Scans",
new CompactionsType("scans"),
"The number of scans running and queued on this tablet server.");
tServerList.addSortableColumn(
"Minor<br />Compactions",
new CompactionsType("minor"),
"The number of minor compactions running and (queued waiting for resources). Minor compactions are the operations where entries are flushed from memory to disk.");
tServerList.addSortableColumn(
"Major<br />Compactions",
new CompactionsType("major"),
"The number of major compactions running and (queued waiting for resources). "
+ "Major compactions are the operations where many smaller files are grouped into a larger file, eliminating duplicates and cleaning up deletes.");
tServerList.addSortableColumn(
"Index Cache<br />Hit Rate", new PercentageType(), "The recent index cache hit rate.");
tServerList.addSortableColumn(
"Data Cache<br />Hit Rate", new PercentageType(), "The recent data cache hit rate.");
tServerList.addSortableColumn(
"OS Load",
new NumberType<Double>(0., guessHighLoad * 1., 0., guessHighLoad * 3.),
"The Unix one minute load average. The average number of processes in the run queue over a one minute interval.");
log.debug("tableId: " + tableId);
for (TabletServerStatus status : tservers) {
if (status == null) status = NO_STATUS;
TableInfo summary = Monitor.summarizeTableStats(status);
if (tableId != null) summary = status.tableMap.get(tableId);
if (summary == null) continue;
TableRow row = tServerList.prepareRow();
row.add(status); // add for server name
row.add(summary.tablets);
row.add(now - status.lastContact);
row.add(summary.recs);
row.add(summary.ingestRate);
row.add(summary.queryRate);
row.add(status.holdTime);
row.add(summary); // add for scans
row.add(summary); // add for minor compactions
row.add(summary); // add for major compactions
double indexCacheHitRate =
status.indexCacheHits / (double) Math.max(status.indexCacheRequest, 1);
row.add(indexCacheHitRate);
double dataCacheHitRate =
status.dataCacheHits / (double) Math.max(status.dataCacheRequest, 1);
row.add(dataCacheHitRate);
row.add(status.osLoad);
tServerList.addRow(row);
}
tServerList.generate(req, sb);
}