@Override
public CompactionStats call() {
final String outputFileName = getOutputFile();
log.debug("Begin minor compaction " + outputFileName + " " + getExtent());
// output to new MapFile with a temporary name
int sleepTime = 100;
double growthFactor = 4;
int maxSleepTime = 1000 * 60 * 3; // 3 minutes
boolean reportedProblem = false;
runningCompactions.add(this);
try {
do {
try {
CompactionStats ret = super.call();
// log.debug(String.format("MinC %,d recs in | %,d recs out | %,d recs/sec | %6.3f secs |
// %,d bytes ",map.size(), entriesCompacted,
// (int)(map.size()/((t2 - t1)/1000.0)), (t2 - t1)/1000.0, estimatedSizeInBytes()));
if (reportedProblem) {
ProblemReports.getInstance(tabletServer)
.deleteProblemReport(
getExtent().getTableId(), ProblemType.FILE_WRITE, outputFileName);
}
return ret;
} catch (IOException e) {
log.warn("MinC failed ({}) to create {} retrying ...", e.getMessage(), outputFileName);
ProblemReports.getInstance(tabletServer)
.report(
new ProblemReport(
getExtent().getTableId(), ProblemType.FILE_WRITE, outputFileName, e));
reportedProblem = true;
} catch (RuntimeException e) {
// if this is coming from a user iterator, it is possible that the user could change the
// iterator config and that the
// minor compaction would succeed
log.warn("MinC failed ({}) to create {} retrying ...", e.getMessage(), outputFileName, e);
ProblemReports.getInstance(tabletServer)
.report(
new ProblemReport(
getExtent().getTableId(), ProblemType.FILE_WRITE, outputFileName, e));
reportedProblem = true;
} catch (CompactionCanceledException e) {
throw new IllegalStateException(e);
}
Random random = new Random();
int sleep = sleepTime + random.nextInt(sleepTime);
log.debug("MinC failed sleeping " + sleep + " ms before retrying");
sleepUninterruptibly(sleep, TimeUnit.MILLISECONDS);
sleepTime = (int) Math.round(Math.min(maxSleepTime, sleepTime * growthFactor));
// clean up
try {
if (getFileSystem().exists(new Path(outputFileName))) {
getFileSystem().deleteRecursively(new Path(outputFileName));
}
} catch (IOException e) {
log.warn("Failed to delete failed MinC file {} {}", outputFileName, e.getMessage());
}
if (isTableDeleting()) return new CompactionStats(0, 0);
} while (true);
} finally {
thread = null;
runningCompactions.remove(this);
}
}
public static void fetchData() {
double totalIngestRate = 0.;
double totalIngestByteRate = 0.;
double totalQueryRate = 0.;
double totalQueryByteRate = 0.;
double totalScanRate = 0.;
long totalEntries = 0;
int totalTabletCount = 0;
int onlineTabletCount = 0;
long totalHoldTime = 0;
long totalLookups = 0;
boolean retry = true;
// only recalc every so often
long currentTime = System.currentTimeMillis();
if (currentTime - lastRecalc < REFRESH_TIME * 1000) return;
synchronized (Monitor.class) {
if (fetching) return;
fetching = true;
}
try {
while (retry) {
MasterClientService.Iface client = null;
try {
client = MasterClient.getConnection(HdfsZooInstance.getInstance());
if (client != null) {
mmi =
client.getMasterStats(
Tracer.traceInfo(),
SystemCredentials.get().toThrift(HdfsZooInstance.getInstance()));
retry = false;
} else {
mmi = null;
}
Monitor.gcStatus = fetchGcStatus();
} catch (Exception e) {
mmi = null;
log.info("Error fetching stats: " + e);
} finally {
if (client != null) {
MasterClient.close(client);
}
}
if (mmi == null) UtilWaitThread.sleep(1000);
}
if (mmi != null) {
int majorCompactions = 0;
int minorCompactions = 0;
lookupRateTracker.startingUpdates();
indexCacheHitTracker.startingUpdates();
indexCacheRequestTracker.startingUpdates();
dataCacheHitTracker.startingUpdates();
dataCacheRequestTracker.startingUpdates();
for (TabletServerStatus server : mmi.tServerInfo) {
TableInfo summary = TableInfoUtil.summarizeTableStats(server);
totalIngestRate += summary.ingestRate;
totalIngestByteRate += summary.ingestByteRate;
totalQueryRate += summary.queryRate;
totalScanRate += summary.scanRate;
totalQueryByteRate += summary.queryByteRate;
totalEntries += summary.recs;
totalHoldTime += server.holdTime;
totalLookups += server.lookups;
majorCompactions += summary.majors.running;
minorCompactions += summary.minors.running;
lookupRateTracker.updateTabletServer(server.name, server.lastContact, server.lookups);
indexCacheHitTracker.updateTabletServer(
server.name, server.lastContact, server.indexCacheHits);
indexCacheRequestTracker.updateTabletServer(
server.name, server.lastContact, server.indexCacheRequest);
dataCacheHitTracker.updateTabletServer(
server.name, server.lastContact, server.dataCacheHits);
dataCacheRequestTracker.updateTabletServer(
server.name, server.lastContact, server.dataCacheRequest);
}
lookupRateTracker.finishedUpdating();
indexCacheHitTracker.finishedUpdating();
indexCacheRequestTracker.finishedUpdating();
dataCacheHitTracker.finishedUpdating();
dataCacheRequestTracker.finishedUpdating();
int totalTables = 0;
for (TableInfo tInfo : mmi.tableMap.values()) {
totalTabletCount += tInfo.tablets;
onlineTabletCount += tInfo.onlineTablets;
totalTables++;
}
Monitor.totalIngestRate = totalIngestRate;
Monitor.totalTables = totalTables;
totalIngestByteRate = totalIngestByteRate / 1000000.0;
Monitor.totalIngestByteRate = totalIngestByteRate;
Monitor.totalQueryRate = totalQueryRate;
Monitor.totalScanRate = totalScanRate;
totalQueryByteRate = totalQueryByteRate / 1000000.0;
Monitor.totalQueryByteRate = totalQueryByteRate;
Monitor.totalEntries = totalEntries;
Monitor.totalTabletCount = totalTabletCount;
Monitor.onlineTabletCount = onlineTabletCount;
Monitor.totalHoldTime = totalHoldTime;
Monitor.totalLookups = totalLookups;
ingestRateOverTime.add(new Pair<Long, Double>(currentTime, totalIngestRate));
ingestByteRateOverTime.add(new Pair<Long, Double>(currentTime, totalIngestByteRate));
double totalLoad = 0.;
for (TabletServerStatus status : mmi.tServerInfo) {
if (status != null) totalLoad += status.osLoad;
}
loadOverTime.add(new Pair<Long, Double>(currentTime, totalLoad));
minorCompactionsOverTime.add(new Pair<Long, Integer>(currentTime, minorCompactions));
majorCompactionsOverTime.add(new Pair<Long, Integer>(currentTime, majorCompactions));
lookupsOverTime.add(new Pair<Long, Double>(currentTime, lookupRateTracker.calculateRate()));
queryRateOverTime.add(new Pair<Long, Integer>(currentTime, (int) totalQueryRate));
queryByteRateOverTime.add(new Pair<Long, Double>(currentTime, totalQueryByteRate));
scanRateOverTime.add(new Pair<Long, Integer>(currentTime, (int) totalScanRate));
calcCacheHitRate(
indexCacheHitRateOverTime, currentTime, indexCacheHitTracker, indexCacheRequestTracker);
calcCacheHitRate(
dataCacheHitRateOverTime, currentTime, dataCacheHitTracker, dataCacheRequestTracker);
}
try {
Monitor.problemSummary = ProblemReports.getInstance().summarize();
Monitor.problemException = null;
} catch (Exception e) {
log.info("Failed to obtain problem reports ", e);
Monitor.problemSummary = Collections.emptyMap();
Monitor.problemException = e;
}
} finally {
synchronized (Monitor.class) {
fetching = false;
lastRecalc = currentTime;
}
}
}