private static GCStatus fetchGcStatus() {
GCStatus result = null;
HostAndPort address = null;
try {
// Read the gc location from its lock
ZooReaderWriter zk = ZooReaderWriter.getInstance();
String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
List<String> locks = zk.getChildren(path, null);
if (locks != null && locks.size() > 0) {
Collections.sort(locks);
address =
new ServerServices(
new String(zk.getData(path + "/" + locks.get(0), null), StandardCharsets.UTF_8))
.getAddress(Service.GC_CLIENT);
GCMonitorService.Client client =
ThriftUtil.getClient(
new GCMonitorService.Client.Factory(), address, config.getConfiguration());
try {
result = client.getStatus(Tracer.traceInfo(), SystemCredentials.get().toThrift(instance));
} finally {
ThriftUtil.returnClient(client);
}
}
} catch (Exception ex) {
log.warn("Unable to contact the garbage collector at " + address, ex);
}
return result;
}
public static ServerAddress startThreadPoolServer(
InetSocketAddress address,
TProcessor processor,
String serverName,
String threadName,
int numThreads)
throws TTransportException {
// if port is zero, then we must bind to get the port number
ServerSocket sock;
try {
sock = ServerSocketChannel.open().socket();
sock.setReuseAddress(true);
sock.bind(address);
address = new InetSocketAddress(address.getHostName(), sock.getLocalPort());
} catch (IOException ex) {
throw new TTransportException(ex);
}
TServerTransport transport = new TBufferedServerSocket(sock, 32 * 1024);
TThreadPoolServer.Args options = new TThreadPoolServer.Args(transport);
options.protocolFactory(ThriftUtil.protocolFactory());
options.transportFactory(ThriftUtil.transportFactory());
options.processorFactory(new ClientInfoProcessorFactory(processor));
return new ServerAddress(new TThreadPoolServer(options), address);
}
public static MasterClientService.Client getConnection(Instance instance) {
List<String> locations = instance.getMasterLocations();
if (locations.size() == 0) {
log.debug("No masters...");
return null;
}
String master = locations.get(0);
if (master.endsWith(":0")) return null;
try {
// Master requests can take a long time: don't ever time out
MasterClientService.Client client =
ThriftUtil.getClientNoTimeout(
new MasterClientService.Client.Factory(),
master,
ServerConfigurationUtil.getConfiguration(instance));
return client;
} catch (TTransportException tte) {
if (tte.getCause().getClass().equals(UnknownHostException.class)) {
// do not expect to recover from this
throw new RuntimeException(tte);
}
log.debug("Failed to connect to master=" + master + ", will retry... ", tte);
return null;
}
}
@Override
protected void pageBody(HttpServletRequest req, HttpServletResponse response, StringBuilder sb)
throws Exception {
String tserverAddress = req.getParameter("s");
// Check to make sure tserver is a known address
boolean tserverExists = false;
if (tserverAddress != null && tserverAddress.isEmpty() == false) {
for (TabletServerStatus ts : Monitor.getMmi().getTServerInfo()) {
if (tserverAddress.equals(ts.getName())) {
tserverExists = true;
break;
}
}
}
if (tserverAddress == null || tserverAddress.isEmpty() || tserverExists == false) {
doBadTserverList(req, sb);
doDeadTserverList(req, sb);
ArrayList<TabletServerStatus> tservers = new ArrayList<TabletServerStatus>();
if (Monitor.getMmi() != null) tservers.addAll(Monitor.getMmi().tServerInfo);
Table tServerList = new Table("tservers", "Tablet Servers");
tServerList.setSubCaption(
"Click on the <span style='color: #0000ff;'>server address</span> to view detailed performance statistics for that server.");
doTserverList(req, sb, tservers, null, tServerList);
return;
}
double totalElapsedForAll = 0;
double splitStdDev = 0;
double minorStdDev = 0;
double minorQueueStdDev = 0;
double majorStdDev = 0;
double majorQueueStdDev = 0;
double currentMinorAvg = 0;
double currentMajorAvg = 0;
double currentMinorStdDev = 0;
double currentMajorStdDev = 0;
TabletStats total =
new TabletStats(null, new ActionStats(), new ActionStats(), new ActionStats(), 0, 0, 0, 0);
InetSocketAddress address = AddressUtil.parseAddress(tserverAddress, -1);
TabletStats historical =
new TabletStats(null, new ActionStats(), new ActionStats(), new ActionStats(), 0, 0, 0, 0);
List<TabletStats> tsStats = new ArrayList<TabletStats>();
try {
TabletClientService.Client client =
ThriftUtil.getClient(
new TabletClientService.Client.Factory(), address, Monitor.getSystemConfiguration());
try {
for (String tableId : Monitor.getMmi().tableMap.keySet()) {
tsStats.addAll(
client.getTabletStats(
Tracer.traceInfo(),
SystemCredentials.get().toThrift(Monitor.getInstance()),
tableId));
}
historical =
client.getHistoricalStats(
Tracer.traceInfo(), SystemCredentials.get().toThrift(Monitor.getInstance()));
} finally {
ThriftUtil.returnClient(client);
}
} catch (Exception e) {
banner(sb, "error", "No Such Tablet ServerAvailable");
log.error(e, e);
return;
}
Table perTabletResults = new Table("perTabletResults", "Detailed Current Operations");
perTabletResults.setSubCaption("Per-tablet Details");
perTabletResults.addSortableColumn("Table", new TableLinkType(), null);
perTabletResults.addSortableColumn("Tablet");
perTabletResults.addSortableColumn("Entries", new NumberType<Long>(), null);
perTabletResults.addSortableColumn("Ingest", new NumberType<Long>(), null);
perTabletResults.addSortableColumn("Query", new NumberType<Long>(), null);
perTabletResults.addSortableColumn("Minor Avg", new SecondType(), null);
perTabletResults.addSortableColumn("Minor Std Dev", new SecondType(), null);
perTabletResults.addSortableColumn("Minor Avg e/s", new NumberType<Double>(), null);
perTabletResults.addSortableColumn("Major Avg", new SecondType(), null);
perTabletResults.addSortableColumn("Major Std Dev", new SecondType(), null);
perTabletResults.addSortableColumn("Major Avg e/s", new NumberType<Double>(), null);
for (TabletStats info : tsStats) {
if (info.extent == null) {
historical = info;
continue;
}
total.numEntries += info.numEntries;
TabletStatsKeeper.update(total.minors, info.minors);
TabletStatsKeeper.update(total.majors, info.majors);
KeyExtent extent = new KeyExtent(info.extent);
String tableId = extent.getTableId().toString();
MessageDigest digester = MessageDigest.getInstance("MD5");
if (extent.getEndRow() != null && extent.getEndRow().getLength() > 0) {
digester.update(extent.getEndRow().getBytes(), 0, extent.getEndRow().getLength());
}
String obscuredExtent = new String(Base64.encodeBase64(digester.digest()));
String displayExtent = String.format("<code>[%s]</code>", obscuredExtent);
TableRow row = perTabletResults.prepareRow();
row.add(tableId);
row.add(displayExtent);
row.add(info.numEntries);
row.add(info.ingestRate);
row.add(info.queryRate);
row.add(info.minors.num != 0 ? info.minors.elapsed / info.minors.num : null);
row.add(stddev(info.minors.elapsed, info.minors.num, info.minors.sumDev));
row.add(info.minors.elapsed != 0 ? info.minors.count / info.minors.elapsed : null);
row.add(info.majors.num != 0 ? info.majors.elapsed / info.majors.num : null);
row.add(stddev(info.majors.elapsed, info.majors.num, info.majors.sumDev));
row.add(info.majors.elapsed != 0 ? info.majors.count / info.majors.elapsed : null);
perTabletResults.addRow(row);
}
// Calculate current averages oldServer adding in historical data
if (total.minors.num != 0) currentMinorAvg = (long) (total.minors.elapsed / total.minors.num);
if (total.minors.elapsed != 0 && total.minors.num != 0)
currentMinorStdDev = stddev(total.minors.elapsed, total.minors.num, total.minors.sumDev);
if (total.majors.num != 0) currentMajorAvg = total.majors.elapsed / total.majors.num;
if (total.majors.elapsed != 0
&& total.majors.num != 0
&& total.majors.elapsed > total.majors.num)
currentMajorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);
// After these += operations, these variables are now total for current
// tablets and historical tablets
TabletStatsKeeper.update(total.minors, historical.minors);
TabletStatsKeeper.update(total.majors, historical.majors);
totalElapsedForAll += total.majors.elapsed + historical.splits.elapsed + total.minors.elapsed;
minorStdDev = stddev(total.minors.elapsed, total.minors.num, total.minors.sumDev);
minorQueueStdDev = stddev(total.minors.queueTime, total.minors.num, total.minors.queueSumDev);
majorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);
majorQueueStdDev = stddev(total.majors.queueTime, total.majors.num, total.majors.queueSumDev);
splitStdDev =
stddev(historical.splits.num, historical.splits.elapsed, historical.splits.sumDev);
doDetailTable(req, sb, address, tsStats.size(), total, historical);
doAllTimeTable(
req,
sb,
total,
historical,
majorQueueStdDev,
minorQueueStdDev,
totalElapsedForAll,
splitStdDev,
majorStdDev,
minorStdDev);
doCurrentTabletOps(
req, sb, currentMinorAvg, currentMinorStdDev, currentMajorAvg, currentMajorStdDev);
perTabletResults.generate(req, sb);
}
private MutationSet sendMutationsToTabletServer(
String location, Map<KeyExtent, List<Mutation>> tabMuts, TimeoutTracker timeoutTracker)
throws IOException, AccumuloSecurityException, AccumuloServerException {
if (tabMuts.size() == 0) {
return new MutationSet();
}
TInfo tinfo = Tracer.traceInfo();
// TODO remove this
TTransport transport = null;
timeoutTracker.startingWrite();
try {
TabletClientService.Iface client;
if (timeoutTracker.getTimeOut()
< instance.getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT))
client = ThriftUtil.getTServerClient(location, timeoutTracker.getTimeOut());
else client = ThriftUtil.getTServerClient(location, instance.getConfiguration());
try {
MutationSet allFailures = new MutationSet();
if (tabMuts.size() == 1 && tabMuts.values().iterator().next().size() == 1) {
Entry<KeyExtent, List<Mutation>> entry = tabMuts.entrySet().iterator().next();
try {
client.update(
tinfo,
credentials.toThrift(instance),
entry.getKey().toThrift(),
entry.getValue().get(0).toThrift());
} catch (NotServingTabletException e) {
allFailures.addAll(entry.getKey().getTableId().toString(), entry.getValue());
TabletLocator.getLocator(instance, new Text(entry.getKey().getTableId()))
.invalidateCache(entry.getKey());
} catch (ConstraintViolationException e) {
updatedConstraintViolations(
Translator.translate(e.violationSummaries, Translator.TCVST));
}
timeoutTracker.madeProgress();
} else {
long usid = client.startUpdate(tinfo, credentials.toThrift(instance));
List<TMutation> updates = new ArrayList<TMutation>();
for (Entry<KeyExtent, List<Mutation>> entry : tabMuts.entrySet()) {
long size = 0;
Iterator<Mutation> iter = entry.getValue().iterator();
while (iter.hasNext()) {
while (size < MUTATION_BATCH_SIZE && iter.hasNext()) {
Mutation mutation = iter.next();
updates.add(mutation.toThrift());
size += mutation.numBytes();
}
client.applyUpdates(tinfo, usid, entry.getKey().toThrift(), updates);
updates.clear();
size = 0;
}
}
UpdateErrors updateErrors = client.closeUpdate(tinfo, usid);
Map<KeyExtent, Long> failures =
Translator.translate(updateErrors.failedExtents, Translator.TKET);
updatedConstraintViolations(
Translator.translate(updateErrors.violationSummaries, Translator.TCVST));
updateAuthorizationFailures(
Translator.translate(updateErrors.authorizationFailures, Translator.TKET));
long totalCommitted = 0;
for (Entry<KeyExtent, Long> entry : failures.entrySet()) {
KeyExtent failedExtent = entry.getKey();
int numCommitted = (int) (long) entry.getValue();
totalCommitted += numCommitted;
String table = failedExtent.getTableId().toString();
TabletLocator.getLocator(instance, new Text(table)).invalidateCache(failedExtent);
ArrayList<Mutation> mutations = (ArrayList<Mutation>) tabMuts.get(failedExtent);
allFailures.addAll(table, mutations.subList(numCommitted, mutations.size()));
}
if (failures.keySet().containsAll(tabMuts.keySet()) && totalCommitted == 0) {
// nothing was successfully written
timeoutTracker.wroteNothing();
} else {
// successfully wrote something to tablet server
timeoutTracker.madeProgress();
}
}
return allFailures;
} finally {
ThriftUtil.returnClient((TServiceClient) client);
}
} catch (TTransportException e) {
timeoutTracker.errorOccured(e);
throw new IOException(e);
} catch (TApplicationException tae) {
updateServerErrors(location, tae);
throw new AccumuloServerException(location, tae);
} catch (ThriftSecurityException e) {
updateAuthorizationFailures(tabMuts.keySet(), e.code);
throw new AccumuloSecurityException(e.user, e.code, e);
} catch (NoSuchScanIDException e) {
throw new IOException(e);
} catch (TException e) {
throw new IOException(e);
} finally {
ThriftTransportPool.getInstance().returnTransport(transport);
}
}
public static ServerAddress startHsHaServer(
InetSocketAddress address,
TProcessor processor,
final String serverName,
String threadName,
final int numThreads,
long timeBetweenThreadChecks,
long maxMessageSize)
throws TTransportException {
TNonblockingServerSocket transport = new TNonblockingServerSocket(address);
// check for the special "bind to everything address"
if (address.getAddress().getHostAddress().equals("0.0.0.0")) {
// can't get the address from the bind, so we'll do our best to invent our hostname
try {
address =
new InetSocketAddress(InetAddress.getLocalHost().getHostName(), address.getPort());
} catch (UnknownHostException e) {
throw new TTransportException(e);
}
}
THsHaServer.Args options = new THsHaServer.Args(transport);
options.protocolFactory(ThriftUtil.protocolFactory());
options.transportFactory(ThriftUtil.transportFactory(maxMessageSize));
options.stopTimeoutVal(5);
/*
* Create our own very special thread pool.
*/
final ThreadPoolExecutor pool = new SimpleThreadPool(numThreads, "ClientPool");
// periodically adjust the number of threads we need by checking how busy our threads are
SimpleTimer.getInstance()
.schedule(
new Runnable() {
@Override
public void run() {
if (pool.getCorePoolSize() <= pool.getActiveCount()) {
int larger = pool.getCorePoolSize() + Math.min(pool.getQueue().size(), 2);
log.info("Increasing server thread pool size on " + serverName + " to " + larger);
pool.setMaximumPoolSize(larger);
pool.setCorePoolSize(larger);
} else {
if (pool.getCorePoolSize() > pool.getActiveCount() + 3) {
int smaller = Math.max(numThreads, pool.getCorePoolSize() - 1);
if (smaller != pool.getCorePoolSize()) {
// there is a race condition here... the active count could be higher by the
// time
// we decrease the core pool size... so the active count could end up higher
// than
// the core pool size, in which case everything will be queued... the increase
// case
// should handle this and prevent deadlock
log.info(
"Decreasing server thread pool size on " + serverName + " to " + smaller);
pool.setCorePoolSize(smaller);
}
}
}
}
},
timeBetweenThreadChecks,
timeBetweenThreadChecks);
options.executorService(pool);
options.processorFactory(new TProcessorFactory(processor));
return new ServerAddress(new THsHaServer(options), address);
}