private static GCStatus fetchGcStatus() {
GCStatus result = null;
HostAndPort address = null;
try {
// Read the gc location from its lock
ZooReaderWriter zk = ZooReaderWriter.getInstance();
String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
List<String> locks = zk.getChildren(path, null);
if (locks != null && locks.size() > 0) {
Collections.sort(locks);
address =
new ServerServices(
new String(zk.getData(path + "/" + locks.get(0), null), StandardCharsets.UTF_8))
.getAddress(Service.GC_CLIENT);
GCMonitorService.Client client =
ThriftUtil.getClient(
new GCMonitorService.Client.Factory(), address, config.getConfiguration());
try {
result = client.getStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
} finally {
ThriftUtil.returnClient(client);
}
}
} catch (Exception ex) {
log.warn("Unable to contact the garbage collector at " + address, ex);
}
return result;
}
public static void removeUnusedWALEntries(
KeyExtent extent, List<LogEntry> logEntries, ZooLock zooLock) {
if (extent.isRootTablet()) {
for (LogEntry entry : logEntries) {
String root = getZookeeperLogLocation();
while (true) {
try {
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
if (zoo.isLockHeld(zooLock.getLockID()))
zoo.recursiveDelete(root + "/" + entry.filename, NodeMissingPolicy.SKIP);
break;
} catch (Exception e) {
log.error(e, e);
}
UtilWaitThread.sleep(1000);
}
}
} else {
Mutation m = new Mutation(extent.getMetadataEntry());
for (LogEntry entry : logEntries) {
m.putDelete(LogColumnFamily.NAME, new Text(entry.toString()));
}
update(SystemCredentials.get(), zooLock, m, extent);
}
}
public static void removeBulkLoadInProgressFlag(String path) {
Mutation m = new Mutation(MetadataSchema.BlipSection.getRowPrefix() + path);
m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
// new KeyExtent is only added to force update to write to the metadata table, not the root
// table
// because bulk loads aren't supported to the metadata table
update(SystemCredentials.get(), m, new KeyExtent(new Text("anythingNotMetadata"), null, null));
}
@Before
public void setUp() {
volMgr = createMock(VolumeManager.class);
instance = createMock(Instance.class);
SiteConfiguration siteConfig = EasyMock.createMock(SiteConfiguration.class);
expect(instance.getInstanceID()).andReturn("mock").anyTimes();
expect(instance.getZooKeepers()).andReturn("localhost").anyTimes();
expect(instance.getZooKeepersSessionTimeOut()).andReturn(30000).anyTimes();
opts = new Opts();
systemConfig = createSystemConfig();
ServerConfigurationFactory factory = createMock(ServerConfigurationFactory.class);
expect(factory.getInstance()).andReturn(instance).anyTimes();
expect(factory.getConfiguration()).andReturn(systemConfig).anyTimes();
expect(factory.getSiteConfiguration()).andReturn(siteConfig).anyTimes();
// Just make the SiteConfiguration delegate to our AccumuloConfiguration
// Presently, we only need get(Property) and iterator().
EasyMock.expect(siteConfig.get(EasyMock.anyObject(Property.class)))
.andAnswer(
new IAnswer<String>() {
@Override
public String answer() {
Object[] args = EasyMock.getCurrentArguments();
return systemConfig.get((Property) args[0]);
}
})
.anyTimes();
EasyMock.expect(siteConfig.getBoolean(EasyMock.anyObject(Property.class)))
.andAnswer(
new IAnswer<Boolean>() {
@Override
public Boolean answer() {
Object[] args = EasyMock.getCurrentArguments();
return systemConfig.getBoolean((Property) args[0]);
}
})
.anyTimes();
EasyMock.expect(siteConfig.iterator())
.andAnswer(
new IAnswer<Iterator<Entry<String, String>>>() {
@Override
public Iterator<Entry<String, String>> answer() {
return systemConfig.iterator();
}
})
.anyTimes();
replay(instance, factory, siteConfig);
credentials = SystemCredentials.get(instance);
gc = new SimpleGarbageCollector(opts, volMgr, factory);
}
@Override
protected void pageBody(HttpServletRequest req, HttpServletResponse response, StringBuilder sb)
throws Exception {
String tserverAddress = req.getParameter("s");
// Check to make sure tserver is a known address
boolean tserverExists = false;
if (tserverAddress != null && tserverAddress.isEmpty() == false) {
for (TabletServerStatus ts : Monitor.getMmi().getTServerInfo()) {
if (tserverAddress.equals(ts.getName())) {
tserverExists = true;
break;
}
}
}
if (tserverAddress == null || tserverAddress.isEmpty() || tserverExists == false) {
doBadTserverList(req, sb);
doDeadTserverList(req, sb);
ArrayList<TabletServerStatus> tservers = new ArrayList<TabletServerStatus>();
if (Monitor.getMmi() != null) tservers.addAll(Monitor.getMmi().tServerInfo);
Table tServerList = new Table("tservers", "Tablet Servers");
tServerList.setSubCaption(
"Click on the <span style='color: #0000ff;'>server address</span> to view detailed performance statistics for that server.");
doTserverList(req, sb, tservers, null, tServerList);
return;
}
double totalElapsedForAll = 0;
double splitStdDev = 0;
double minorStdDev = 0;
double minorQueueStdDev = 0;
double majorStdDev = 0;
double majorQueueStdDev = 0;
double currentMinorAvg = 0;
double currentMajorAvg = 0;
double currentMinorStdDev = 0;
double currentMajorStdDev = 0;
TabletStats total =
new TabletStats(null, new ActionStats(), new ActionStats(), new ActionStats(), 0, 0, 0, 0);
InetSocketAddress address = AddressUtil.parseAddress(tserverAddress, -1);
TabletStats historical =
new TabletStats(null, new ActionStats(), new ActionStats(), new ActionStats(), 0, 0, 0, 0);
List<TabletStats> tsStats = new ArrayList<TabletStats>();
try {
TabletClientService.Client client =
ThriftUtil.getClient(
new TabletClientService.Client.Factory(), address, Monitor.getSystemConfiguration());
try {
for (String tableId : Monitor.getMmi().tableMap.keySet()) {
tsStats.addAll(
client.getTabletStats(
Tracer.traceInfo(),
SystemCredentials.get().toThrift(Monitor.getInstance()),
tableId));
}
historical =
client.getHistoricalStats(
Tracer.traceInfo(), SystemCredentials.get().toThrift(Monitor.getInstance()));
} finally {
ThriftUtil.returnClient(client);
}
} catch (Exception e) {
banner(sb, "error", "No Such Tablet ServerAvailable");
log.error(e, e);
return;
}
Table perTabletResults = new Table("perTabletResults", "Detailed Current Operations");
perTabletResults.setSubCaption("Per-tablet Details");
perTabletResults.addSortableColumn("Table", new TableLinkType(), null);
perTabletResults.addSortableColumn("Tablet");
perTabletResults.addSortableColumn("Entries", new NumberType<Long>(), null);
perTabletResults.addSortableColumn("Ingest", new NumberType<Long>(), null);
perTabletResults.addSortableColumn("Query", new NumberType<Long>(), null);
perTabletResults.addSortableColumn("Minor Avg", new SecondType(), null);
perTabletResults.addSortableColumn("Minor Std Dev", new SecondType(), null);
perTabletResults.addSortableColumn("Minor Avg e/s", new NumberType<Double>(), null);
perTabletResults.addSortableColumn("Major Avg", new SecondType(), null);
perTabletResults.addSortableColumn("Major Std Dev", new SecondType(), null);
perTabletResults.addSortableColumn("Major Avg e/s", new NumberType<Double>(), null);
for (TabletStats info : tsStats) {
if (info.extent == null) {
historical = info;
continue;
}
total.numEntries += info.numEntries;
TabletStatsKeeper.update(total.minors, info.minors);
TabletStatsKeeper.update(total.majors, info.majors);
KeyExtent extent = new KeyExtent(info.extent);
String tableId = extent.getTableId().toString();
MessageDigest digester = MessageDigest.getInstance("MD5");
if (extent.getEndRow() != null && extent.getEndRow().getLength() > 0) {
digester.update(extent.getEndRow().getBytes(), 0, extent.getEndRow().getLength());
}
String obscuredExtent = new String(Base64.encodeBase64(digester.digest()));
String displayExtent = String.format("<code>[%s]</code>", obscuredExtent);
TableRow row = perTabletResults.prepareRow();
row.add(tableId);
row.add(displayExtent);
row.add(info.numEntries);
row.add(info.ingestRate);
row.add(info.queryRate);
row.add(info.minors.num != 0 ? info.minors.elapsed / info.minors.num : null);
row.add(stddev(info.minors.elapsed, info.minors.num, info.minors.sumDev));
row.add(info.minors.elapsed != 0 ? info.minors.count / info.minors.elapsed : null);
row.add(info.majors.num != 0 ? info.majors.elapsed / info.majors.num : null);
row.add(stddev(info.majors.elapsed, info.majors.num, info.majors.sumDev));
row.add(info.majors.elapsed != 0 ? info.majors.count / info.majors.elapsed : null);
perTabletResults.addRow(row);
}
// Calculate current averages oldServer adding in historical data
if (total.minors.num != 0) currentMinorAvg = (long) (total.minors.elapsed / total.minors.num);
if (total.minors.elapsed != 0 && total.minors.num != 0)
currentMinorStdDev = stddev(total.minors.elapsed, total.minors.num, total.minors.sumDev);
if (total.majors.num != 0) currentMajorAvg = total.majors.elapsed / total.majors.num;
if (total.majors.elapsed != 0
&& total.majors.num != 0
&& total.majors.elapsed > total.majors.num)
currentMajorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);
// After these += operations, these variables are now total for current
// tablets and historical tablets
TabletStatsKeeper.update(total.minors, historical.minors);
TabletStatsKeeper.update(total.majors, historical.majors);
totalElapsedForAll += total.majors.elapsed + historical.splits.elapsed + total.minors.elapsed;
minorStdDev = stddev(total.minors.elapsed, total.minors.num, total.minors.sumDev);
minorQueueStdDev = stddev(total.minors.queueTime, total.minors.num, total.minors.queueSumDev);
majorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);
majorQueueStdDev = stddev(total.majors.queueTime, total.majors.num, total.majors.queueSumDev);
splitStdDev =
stddev(historical.splits.num, historical.splits.elapsed, historical.splits.sumDev);
doDetailTable(req, sb, address, tsStats.size(), total, historical);
doAllTimeTable(
req,
sb,
total,
historical,
majorQueueStdDev,
minorQueueStdDev,
totalElapsedForAll,
splitStdDev,
majorStdDev,
minorStdDev);
doCurrentTabletOps(
req, sb, currentMinorAvg, currentMinorStdDev, currentMajorAvg, currentMajorStdDev);
perTabletResults.generate(req, sb);
}
public static void fetchData() {
double totalIngestRate = 0.;
double totalIngestByteRate = 0.;
double totalQueryRate = 0.;
double totalQueryByteRate = 0.;
double totalScanRate = 0.;
long totalEntries = 0;
int totalTabletCount = 0;
int onlineTabletCount = 0;
long totalHoldTime = 0;
long totalLookups = 0;
boolean retry = true;
// only recalc every so often
long currentTime = System.currentTimeMillis();
if (currentTime - lastRecalc < REFRESH_TIME * 1000) return;
synchronized (Monitor.class) {
if (fetching) return;
fetching = true;
}
try {
while (retry) {
MasterClientService.Iface client = null;
try {
client = MasterClient.getConnection(HdfsZooInstance.getInstance());
if (client != null) {
mmi =
client.getMasterStats(
Tracer.traceInfo(),
SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()));
retry = false;
} else {
mmi = null;
}
Monitor.gcStatus = fetchGcStatus();
} catch (Exception e) {
mmi = null;
log.info("Error fetching stats: " + e);
} finally {
if (client != null) {
MasterClient.close(client);
}
}
if (mmi == null) UtilWaitThread.sleep(1000);
}
if (mmi != null) {
int majorCompactions = 0;
int minorCompactions = 0;
lookupRateTracker.startingUpdates();
indexCacheHitTracker.startingUpdates();
indexCacheRequestTracker.startingUpdates();
dataCacheHitTracker.startingUpdates();
dataCacheRequestTracker.startingUpdates();
for (TabletServerStatus server : mmi.tServerInfo) {
TableInfo summary = TableInfoUtil.summarizeTableStats(server);
totalIngestRate += summary.ingestRate;
totalIngestByteRate += summary.ingestByteRate;
totalQueryRate += summary.queryRate;
totalScanRate += summary.scanRate;
totalQueryByteRate += summary.queryByteRate;
totalEntries += summary.recs;
totalHoldTime += server.holdTime;
totalLookups += server.lookups;
majorCompactions += summary.majors.running;
minorCompactions += summary.minors.running;
lookupRateTracker.updateTabletServer(server.name, server.lastContact, server.lookups);
indexCacheHitTracker.updateTabletServer(
server.name, server.lastContact, server.indexCacheHits);
indexCacheRequestTracker.updateTabletServer(
server.name, server.lastContact, server.indexCacheRequest);
dataCacheHitTracker.updateTabletServer(
server.name, server.lastContact, server.dataCacheHits);
dataCacheRequestTracker.updateTabletServer(
server.name, server.lastContact, server.dataCacheRequest);
}
lookupRateTracker.finishedUpdating();
indexCacheHitTracker.finishedUpdating();
indexCacheRequestTracker.finishedUpdating();
dataCacheHitTracker.finishedUpdating();
dataCacheRequestTracker.finishedUpdating();
int totalTables = 0;
for (TableInfo tInfo : mmi.tableMap.values()) {
totalTabletCount += tInfo.tablets;
onlineTabletCount += tInfo.onlineTablets;
totalTables++;
}
Monitor.totalIngestRate = totalIngestRate;
Monitor.totalTables = totalTables;
totalIngestByteRate = totalIngestByteRate / 1000000.0;
Monitor.totalIngestByteRate = totalIngestByteRate;
Monitor.totalQueryRate = totalQueryRate;
Monitor.totalScanRate = totalScanRate;
totalQueryByteRate = totalQueryByteRate / 1000000.0;
Monitor.totalQueryByteRate = totalQueryByteRate;
Monitor.totalEntries = totalEntries;
Monitor.totalTabletCount = totalTabletCount;
Monitor.onlineTabletCount = onlineTabletCount;
Monitor.totalHoldTime = totalHoldTime;
Monitor.totalLookups = totalLookups;
ingestRateOverTime.add(new Pair<Long, Double>(currentTime, totalIngestRate));
ingestByteRateOverTime.add(new Pair<Long, Double>(currentTime, totalIngestByteRate));
double totalLoad = 0.;
for (TabletServerStatus status : mmi.tServerInfo) {
if (status != null) totalLoad += status.osLoad;
}
loadOverTime.add(new Pair<Long, Double>(currentTime, totalLoad));
minorCompactionsOverTime.add(new Pair<Long, Integer>(currentTime, minorCompactions));
majorCompactionsOverTime.add(new Pair<Long, Integer>(currentTime, majorCompactions));
lookupsOverTime.add(new Pair<Long, Double>(currentTime, lookupRateTracker.calculateRate()));
queryRateOverTime.add(new Pair<Long, Integer>(currentTime, (int) totalQueryRate));
queryByteRateOverTime.add(new Pair<Long, Double>(currentTime, totalQueryByteRate));
scanRateOverTime.add(new Pair<Long, Integer>(currentTime, (int) totalScanRate));
calcCacheHitRate(
indexCacheHitRateOverTime, currentTime, indexCacheHitTracker, indexCacheRequestTracker);
calcCacheHitRate(
dataCacheHitRateOverTime, currentTime, dataCacheHitTracker, dataCacheRequestTracker);
}
try {
Monitor.problemSummary = ProblemReports.getInstance().summarize();
Monitor.problemException = null;
} catch (Exception e) {
log.info("Failed to obtain problem reports ", e);
Monitor.problemSummary = Collections.emptyMap();
Monitor.problemException = e;
}
} finally {
synchronized (Monitor.class) {
fetching = false;
lastRecalc = currentTime;
}
}
}
public static void chopped(KeyExtent extent, ZooLock zooLock) {
Mutation m = new Mutation(extent.getMetadataEntry());
ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("chopped".getBytes()));
update(SystemCredentials.get(), zooLock, m, extent);
}
public static void cloneTable(
Instance instance, String srcTableId, String tableId, VolumeManager volumeManager)
throws Exception {
Connector conn =
instance.getConnector(
SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken());
BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
while (true) {
try {
initializeClone(srcTableId, tableId, conn, bw);
// the following loop looks changes in the file that occurred during the copy.. if files
// were dereferenced then they could have been GCed
while (true) {
int rewrites = checkClone(srcTableId, tableId, conn, bw);
if (rewrites == 0) break;
}
bw.flush();
break;
} catch (TabletIterator.TabletDeletedException tde) {
// tablets were merged in the src table
bw.flush();
// delete what we have cloned and try again
deleteTable(tableId, false, SystemCredentials.get(), null);
log.debug(
"Tablets merged in table " + srcTableId + " while attempting to clone, trying again");
UtilWaitThread.sleep(100);
}
}
// delete the clone markers and create directory entries
Scanner mscanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
mscanner.fetchColumnFamily(ClonedColumnFamily.NAME);
int dirCount = 0;
for (Entry<Key, Value> entry : mscanner) {
Key k = entry.getKey();
Mutation m = new Mutation(k.getRow());
m.putDelete(k.getColumnFamily(), k.getColumnQualifier());
String dir =
volumeManager.choose(ServerConstants.getTablesDirs())
+ "/"
+ tableId
+ new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, "/c-".getBytes()));
TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes()));
bw.addMutation(m);
}
bw.close();
}
public static void addDeleteEntry(String tableId, String path) throws IOException {
update(
SystemCredentials.get(),
createDeleteMutation(tableId, path),
new KeyExtent(new Text(tableId), null, null));
}