private LlapIoImpl(Configuration conf) throws IOException {
String ioMode = HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_IO_MEMORY_MODE);
boolean useLowLevelCache = LlapIoImpl.MODE_CACHE.equalsIgnoreCase(ioMode),
useAllocOnly = !useLowLevelCache && LlapIoImpl.MODE_ALLOCATOR.equalsIgnoreCase(ioMode);
LOG.info("Initializing LLAP IO in {} mode", ioMode);
String displayName = "LlapDaemonCacheMetrics-" + MetricsUtils.getHostName();
String sessionId = conf.get("llap.daemon.metrics.sessionid");
this.cacheMetrics = LlapDaemonCacheMetrics.create(displayName, sessionId);
displayName = "LlapDaemonQueueMetrics-" + MetricsUtils.getHostName();
int[] intervals =
conf.getInts(String.valueOf(HiveConf.ConfVars.LLAP_QUEUE_METRICS_PERCENTILE_INTERVALS));
this.queueMetrics = LlapDaemonQueueMetrics.create(displayName, sessionId, intervals);
LOG.info(
"Started llap daemon metrics with displayName: {} sessionId: {}", displayName, sessionId);
OrcMetadataCache metadataCache = null;
LowLevelCacheImpl orcCache = null;
BufferUsageManager bufferManager = null;
if (useLowLevelCache) {
// Memory manager uses cache policy to trigger evictions, so create the policy first.
boolean useLrfu = HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_USE_LRFU);
LowLevelCachePolicy cachePolicy =
useLrfu ? new LowLevelLrfuCachePolicy(conf) : new LowLevelFifoCachePolicy(conf);
// Allocator uses memory manager to request memory, so create the manager next.
LowLevelCacheMemoryManager memManager =
new LowLevelCacheMemoryManager(conf, cachePolicy, cacheMetrics);
// Cache uses allocator to allocate and deallocate, create allocator and then caches.
EvictionAwareAllocator allocator = new BuddyAllocator(conf, memManager, cacheMetrics);
this.allocator = allocator;
orcCache = new LowLevelCacheImpl(cacheMetrics, cachePolicy, allocator, true);
metadataCache = new OrcMetadataCache(memManager, cachePolicy);
// And finally cache policy uses cache to notify it of eviction. The cycle is complete!
cachePolicy.setEvictionListener(new EvictionDispatcher(orcCache, metadataCache));
cachePolicy.setParentDebugDumper(orcCache);
orcCache.init(); // Start the cache threads.
bufferManager = orcCache; // Cache also serves as buffer manager.
} else {
if (useAllocOnly) {
LowLevelCacheMemoryManager memManager =
new LowLevelCacheMemoryManager(conf, null, cacheMetrics);
allocator = new BuddyAllocator(conf, memManager, cacheMetrics);
} else {
allocator = new SimpleAllocator(conf);
}
bufferManager = new SimpleBufferManager(allocator, cacheMetrics);
}
// IO thread pool. Listening is used for unhandled errors for now (TODO: remove?)
int numThreads = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_IO_THREADPOOL_SIZE);
executor =
MoreExecutors.listeningDecorator(
Executors.newFixedThreadPool(
numThreads,
new ThreadFactoryBuilder()
.setNameFormat("IO-Elevator-Thread-%d")
.setDaemon(true)
.build()));
// TODO: this should depends on input format and be in a map, or something.
this.cvp =
new OrcColumnVectorProducer(
metadataCache, orcCache, bufferManager, conf, cacheMetrics, queueMetrics);
LOG.info("LLAP IO initialized");
registerMXBeans();
}
public LlapDaemon(
Configuration daemonConf,
int numExecutors,
long executorMemoryBytes,
boolean ioEnabled,
boolean isDirectCache,
long ioMemoryBytes,
String[] localDirs,
int rpcPort,
int shufflePort) {
super("LlapDaemon");
printAsciiArt();
Preconditions.checkArgument(numExecutors > 0);
Preconditions.checkArgument(
rpcPort == 0 || (rpcPort > 1024 && rpcPort < 65536),
"RPC Port must be between 1025 and 65535, or 0 automatic selection");
Preconditions.checkArgument(
localDirs != null && localDirs.length > 0, "Work dirs must be specified");
Preconditions.checkArgument(
shufflePort == 0 || (shufflePort > 1024 && shufflePort < 65536),
"Shuffle Port must be betwee 1024 and 65535, or 0 for automatic selection");
this.maxJvmMemory = getTotalHeapSize();
this.llapIoEnabled = ioEnabled;
this.executorMemoryPerInstance = executorMemoryBytes;
this.ioMemoryPerInstance = ioMemoryBytes;
this.numExecutors = numExecutors;
this.localDirs = localDirs;
int waitQueueSize =
daemonConf.getInt(
LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE,
LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE_DEFAULT);
boolean enablePreemption =
daemonConf.getBoolean(
LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_ENABLE_PREEMPTION,
LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_ENABLE_PREEMPTION_DEFAULT);
LOG.info(
"Attempting to start LlapDaemonConf with the following configuration: "
+ "numExecutors="
+ numExecutors
+ ", rpcListenerPort="
+ rpcPort
+ ", workDirs="
+ Arrays.toString(localDirs)
+ ", shufflePort="
+ shufflePort
+ ", executorMemory="
+ executorMemoryBytes
+ ", llapIoEnabled="
+ ioEnabled
+ ", llapIoCacheIsDirect="
+ isDirectCache
+ ", llapIoCacheSize="
+ ioMemoryBytes
+ ", jvmAvailableMemory="
+ maxJvmMemory
+ ", waitQueueSize= "
+ waitQueueSize
+ ", enablePreemption= "
+ enablePreemption);
long memRequired =
executorMemoryBytes + (ioEnabled && isDirectCache == false ? ioMemoryBytes : 0);
// TODO: this check is somewhat bogus as the maxJvmMemory != Xmx parameters (see annotation in
// LlapServiceDriver)
Preconditions.checkState(
maxJvmMemory >= memRequired,
"Invalid configuration. Xmx value too small. maxAvailable="
+ maxJvmMemory
+ ", configured(exec + io if enabled)="
+ memRequired);
this.shuffleHandlerConf = new Configuration(daemonConf);
this.shuffleHandlerConf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, shufflePort);
this.shuffleHandlerConf.set(
ShuffleHandler.SHUFFLE_HANDLER_LOCAL_DIRS, StringUtils.arrayToString(localDirs));
this.shuffleHandlerConf.setBoolean(
ShuffleHandler.SHUFFLE_DIR_WATCHER_ENABLED,
daemonConf.getBoolean(
LlapConfiguration.LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED,
LlapConfiguration.LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED_DEFAULT));
// Less frequently set parameter, not passing in as a param.
int numHandlers =
daemonConf.getInt(
LlapConfiguration.LLAP_DAEMON_RPC_NUM_HANDLERS,
LlapConfiguration.LLAP_DAEMON_RPC_NUM_HANDLERS_DEFAULT);
// Initialize the metrics system
LlapMetricsSystem.initialize("LlapDaemon");
this.pauseMonitor = new JvmPauseMonitor(daemonConf);
pauseMonitor.start();
String displayName = "LlapDaemonExecutorMetrics-" + MetricsUtils.getHostName();
String sessionId = MetricsUtils.getUUID();
daemonConf.set("llap.daemon.metrics.sessionid", sessionId);
this.metrics = LlapDaemonExecutorMetrics.create(displayName, sessionId, numExecutors);
metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
this.llapDaemonInfoBean = MBeans.register("LlapDaemon", "LlapDaemonInfo", this);
LOG.info(
"Started LlapMetricsSystem with displayName: " + displayName + " sessionId: " + sessionId);
this.amReporter = new AMReporter(address, new QueryFailedHandlerProxy(), daemonConf);
this.server = new LlapDaemonProtocolServerImpl(numHandlers, this, address, rpcPort);
this.containerRunner =
new ContainerRunnerImpl(
daemonConf,
numExecutors,
waitQueueSize,
enablePreemption,
localDirs,
this.shufflePort,
address,
executorMemoryBytes,
metrics,
amReporter);
addIfService(containerRunner);
this.registry = new LlapRegistryService(true);
addIfService(registry);
if (HiveConf.getBoolVar(daemonConf, HiveConf.ConfVars.HIVE_IN_TEST)) {
this.webServices = null;
} else {
this.webServices = new LlapWebServices();
addIfService(webServices);
}
// Bring up the server only after all other components have started.
addIfService(server);
// AMReporter after the server so that it gets the correct address. It knows how to deal with
// requests before it is started.
addIfService(amReporter);
}