protected Scanner getScanner(StringBuilder sb)
throws AccumuloException, AccumuloSecurityException {
AccumuloConfiguration conf = Monitor.getSystemConfiguration();
String principal = conf.get(Property.TRACE_USER);
AuthenticationToken at;
Map<String, String> loginMap =
conf.getAllPropertiesWithPrefix(Property.TRACE_TOKEN_PROPERTY_PREFIX);
if (loginMap.isEmpty()) {
Property p = Property.TRACE_PASSWORD;
at = new PasswordToken(conf.get(p).getBytes(StandardCharsets.UTF_8));
} else {
Properties props = new Properties();
int prefixLength = Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey().length();
for (Entry<String, String> entry : loginMap.entrySet()) {
props.put(entry.getKey().substring(prefixLength), entry.getValue());
}
AuthenticationToken token =
Property.createInstanceFromPropertyName(
conf, Property.TRACE_TOKEN_TYPE, AuthenticationToken.class, new PasswordToken());
token.init(props);
at = token;
}
String table = conf.get(Property.TRACE_TABLE);
try {
Connector conn = HdfsZooInstance.getInstance().getConnector(principal, at);
if (!conn.tableOperations().exists(table)) {
return new NullScanner();
}
Scanner scanner =
conn.createScanner(table, conn.securityOperations().getUserAuthorizations(principal));
return scanner;
} catch (AccumuloSecurityException ex) {
sb.append(
"<h2>Unable to read trace table: check trace username and password configuration.</h2>\n");
return null;
} catch (TableNotFoundException ex) {
return new NullScanner();
}
}
public TabletServerResourceManager(TabletServer tserver, VolumeManager fs) {
this.tserver = tserver;
this.conf = tserver.getServerConfigurationFactory();
this.fs = fs;
final AccumuloConfiguration acuConf = conf.getConfiguration();
long maxMemory = acuConf.getMemoryInBytes(Property.TSERV_MAXMEM);
boolean usingNativeMap =
acuConf.getBoolean(Property.TSERV_NATIVEMAP_ENABLED) && NativeMap.isLoaded();
long blockSize = acuConf.getMemoryInBytes(Property.TSERV_DEFAULT_BLOCKSIZE);
long dCacheSize = acuConf.getMemoryInBytes(Property.TSERV_DATACACHE_SIZE);
long iCacheSize = acuConf.getMemoryInBytes(Property.TSERV_INDEXCACHE_SIZE);
long totalQueueSize = acuConf.getMemoryInBytes(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX);
_iCache = new LruBlockCache(iCacheSize, blockSize);
_dCache = new LruBlockCache(dCacheSize, blockSize);
Runtime runtime = Runtime.getRuntime();
if (usingNativeMap) {
// Still check block cache sizes when using native maps.
if (dCacheSize + iCacheSize + totalQueueSize > runtime.maxMemory()) {
throw new IllegalArgumentException(
String.format(
"Block cache sizes %,d and mutation queue size %,d is too large for this JVM configuration %,d",
dCacheSize + iCacheSize, totalQueueSize, runtime.maxMemory()));
}
} else if (maxMemory + dCacheSize + iCacheSize + totalQueueSize > runtime.maxMemory()) {
throw new IllegalArgumentException(
String.format(
"Maximum tablet server map memory %,d block cache sizes %,d and mutation queue size %,d is too large for this JVM configuration %,d",
maxMemory, dCacheSize + iCacheSize, totalQueueSize, runtime.maxMemory()));
}
runtime.gc();
// totalMemory - freeMemory = memory in use
// maxMemory - memory in use = max available memory
if (!usingNativeMap
&& maxMemory > runtime.maxMemory() - (runtime.totalMemory() - runtime.freeMemory())) {
log.warn("In-memory map may not fit into local memory space.");
}
minorCompactionThreadPool = createEs(Property.TSERV_MINC_MAXCONCURRENT, "minor compactor");
// make this thread pool have a priority queue... and execute tablets with the most
// files first!
majorCompactionThreadPool =
createEs(
Property.TSERV_MAJC_MAXCONCURRENT,
"major compactor",
new CompactionQueue().asBlockingQueueOfRunnable());
rootMajorCompactionThreadPool = createEs(0, 1, 300, "md root major compactor");
defaultMajorCompactionThreadPool = createEs(0, 1, 300, "md major compactor");
splitThreadPool = createEs(1, "splitter");
defaultSplitThreadPool = createEs(0, 1, 60, "md splitter");
defaultMigrationPool = createEs(0, 1, 60, "metadata tablet migration");
migrationPool = createEs(Property.TSERV_MIGRATE_MAXCONCURRENT, "tablet migration");
// not sure if concurrent assignments can run safely... even if they could there is probably no
// benefit at startup because
// individual tablet servers are already running assignments concurrently... having each
// individual tablet server run
// concurrent assignments would put more load on the metadata table at startup
assignmentPool = createEs(Property.TSERV_ASSIGNMENT_MAXCONCURRENT, "tablet assignment");
assignMetaDataPool = createEs(0, 1, 60, "metadata tablet assignment");
activeAssignments = new ConcurrentHashMap<KeyExtent, RunnableStartedAt>();
readAheadThreadPool = createEs(Property.TSERV_READ_AHEAD_MAXCONCURRENT, "tablet read ahead");
defaultReadAheadThreadPool =
createEs(Property.TSERV_METADATA_READ_AHEAD_MAXCONCURRENT, "metadata tablets read ahead");
int maxOpenFiles = acuConf.getCount(Property.TSERV_SCAN_MAX_OPENFILES);
fileManager = new FileManager(tserver, fs, maxOpenFiles, _dCache, _iCache);
memoryManager =
Property.createInstanceFromPropertyName(
acuConf, Property.TSERV_MEM_MGMT, MemoryManager.class, new LargestFirstMemoryManager());
memoryManager.init(tserver.getServerConfigurationFactory());
memMgmt = new MemoryManagementFramework();
memMgmt.startThreads();
SimpleTimer timer = SimpleTimer.getInstance(tserver.getConfiguration());
// We can use the same map for both metadata and normal assignments since the keyspace (extent)
// is guaranteed to be unique. Schedule the task once, the task will reschedule itself.
timer.schedule(new AssignmentWatcher(acuConf, activeAssignments, timer), 5000);
}