Example #1
0
  private LlapIoImpl(Configuration conf) throws IOException {
    String ioMode = HiveConf.getVar(conf, HiveConf.ConfVars.LLAP_IO_MEMORY_MODE);
    boolean useLowLevelCache = LlapIoImpl.MODE_CACHE.equalsIgnoreCase(ioMode),
        useAllocOnly = !useLowLevelCache && LlapIoImpl.MODE_ALLOCATOR.equalsIgnoreCase(ioMode);
    LOG.info("Initializing LLAP IO in {} mode", ioMode);

    String displayName = "LlapDaemonCacheMetrics-" + MetricsUtils.getHostName();
    String sessionId = conf.get("llap.daemon.metrics.sessionid");
    this.cacheMetrics = LlapDaemonCacheMetrics.create(displayName, sessionId);

    displayName = "LlapDaemonQueueMetrics-" + MetricsUtils.getHostName();
    int[] intervals =
        conf.getInts(String.valueOf(HiveConf.ConfVars.LLAP_QUEUE_METRICS_PERCENTILE_INTERVALS));
    this.queueMetrics = LlapDaemonQueueMetrics.create(displayName, sessionId, intervals);

    LOG.info(
        "Started llap daemon metrics with displayName: {} sessionId: {}", displayName, sessionId);

    OrcMetadataCache metadataCache = null;
    LowLevelCacheImpl orcCache = null;
    BufferUsageManager bufferManager = null;
    if (useLowLevelCache) {
      // Memory manager uses cache policy to trigger evictions, so create the policy first.
      boolean useLrfu = HiveConf.getBoolVar(conf, HiveConf.ConfVars.LLAP_USE_LRFU);
      LowLevelCachePolicy cachePolicy =
          useLrfu ? new LowLevelLrfuCachePolicy(conf) : new LowLevelFifoCachePolicy(conf);
      // Allocator uses memory manager to request memory, so create the manager next.
      LowLevelCacheMemoryManager memManager =
          new LowLevelCacheMemoryManager(conf, cachePolicy, cacheMetrics);
      // Cache uses allocator to allocate and deallocate, create allocator and then caches.
      EvictionAwareAllocator allocator = new BuddyAllocator(conf, memManager, cacheMetrics);
      this.allocator = allocator;
      orcCache = new LowLevelCacheImpl(cacheMetrics, cachePolicy, allocator, true);
      metadataCache = new OrcMetadataCache(memManager, cachePolicy);
      // And finally cache policy uses cache to notify it of eviction. The cycle is complete!
      cachePolicy.setEvictionListener(new EvictionDispatcher(orcCache, metadataCache));
      cachePolicy.setParentDebugDumper(orcCache);
      orcCache.init(); // Start the cache threads.
      bufferManager = orcCache; // Cache also serves as buffer manager.
    } else {
      if (useAllocOnly) {
        LowLevelCacheMemoryManager memManager =
            new LowLevelCacheMemoryManager(conf, null, cacheMetrics);
        allocator = new BuddyAllocator(conf, memManager, cacheMetrics);
      } else {
        allocator = new SimpleAllocator(conf);
      }
      bufferManager = new SimpleBufferManager(allocator, cacheMetrics);
    }
    // IO thread pool. Listening is used for unhandled errors for now (TODO: remove?)
    int numThreads = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_IO_THREADPOOL_SIZE);
    executor =
        MoreExecutors.listeningDecorator(
            Executors.newFixedThreadPool(
                numThreads,
                new ThreadFactoryBuilder()
                    .setNameFormat("IO-Elevator-Thread-%d")
                    .setDaemon(true)
                    .build()));

    // TODO: this should depends on input format and be in a map, or something.
    this.cvp =
        new OrcColumnVectorProducer(
            metadataCache, orcCache, bufferManager, conf, cacheMetrics, queueMetrics);
    LOG.info("LLAP IO initialized");

    registerMXBeans();
  }
Example #2
0
  public LlapDaemon(
      Configuration daemonConf,
      int numExecutors,
      long executorMemoryBytes,
      boolean ioEnabled,
      boolean isDirectCache,
      long ioMemoryBytes,
      String[] localDirs,
      int rpcPort,
      int shufflePort) {
    super("LlapDaemon");

    printAsciiArt();

    Preconditions.checkArgument(numExecutors > 0);
    Preconditions.checkArgument(
        rpcPort == 0 || (rpcPort > 1024 && rpcPort < 65536),
        "RPC Port must be between 1025 and 65535, or 0 automatic selection");
    Preconditions.checkArgument(
        localDirs != null && localDirs.length > 0, "Work dirs must be specified");
    Preconditions.checkArgument(
        shufflePort == 0 || (shufflePort > 1024 && shufflePort < 65536),
        "Shuffle Port must be betwee 1024 and 65535, or 0 for automatic selection");

    this.maxJvmMemory = getTotalHeapSize();
    this.llapIoEnabled = ioEnabled;
    this.executorMemoryPerInstance = executorMemoryBytes;
    this.ioMemoryPerInstance = ioMemoryBytes;
    this.numExecutors = numExecutors;
    this.localDirs = localDirs;

    int waitQueueSize =
        daemonConf.getInt(
            LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE,
            LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE_DEFAULT);
    boolean enablePreemption =
        daemonConf.getBoolean(
            LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_ENABLE_PREEMPTION,
            LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_ENABLE_PREEMPTION_DEFAULT);
    LOG.info(
        "Attempting to start LlapDaemonConf with the following configuration: "
            + "numExecutors="
            + numExecutors
            + ", rpcListenerPort="
            + rpcPort
            + ", workDirs="
            + Arrays.toString(localDirs)
            + ", shufflePort="
            + shufflePort
            + ", executorMemory="
            + executorMemoryBytes
            + ", llapIoEnabled="
            + ioEnabled
            + ", llapIoCacheIsDirect="
            + isDirectCache
            + ", llapIoCacheSize="
            + ioMemoryBytes
            + ", jvmAvailableMemory="
            + maxJvmMemory
            + ", waitQueueSize= "
            + waitQueueSize
            + ", enablePreemption= "
            + enablePreemption);

    long memRequired =
        executorMemoryBytes + (ioEnabled && isDirectCache == false ? ioMemoryBytes : 0);
    // TODO: this check is somewhat bogus as the maxJvmMemory != Xmx parameters (see annotation in
    // LlapServiceDriver)
    Preconditions.checkState(
        maxJvmMemory >= memRequired,
        "Invalid configuration. Xmx value too small. maxAvailable="
            + maxJvmMemory
            + ", configured(exec + io if enabled)="
            + memRequired);

    this.shuffleHandlerConf = new Configuration(daemonConf);
    this.shuffleHandlerConf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, shufflePort);
    this.shuffleHandlerConf.set(
        ShuffleHandler.SHUFFLE_HANDLER_LOCAL_DIRS, StringUtils.arrayToString(localDirs));
    this.shuffleHandlerConf.setBoolean(
        ShuffleHandler.SHUFFLE_DIR_WATCHER_ENABLED,
        daemonConf.getBoolean(
            LlapConfiguration.LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED,
            LlapConfiguration.LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED_DEFAULT));

    // Less frequently set parameter, not passing in as a param.
    int numHandlers =
        daemonConf.getInt(
            LlapConfiguration.LLAP_DAEMON_RPC_NUM_HANDLERS,
            LlapConfiguration.LLAP_DAEMON_RPC_NUM_HANDLERS_DEFAULT);

    // Initialize the metrics system
    LlapMetricsSystem.initialize("LlapDaemon");
    this.pauseMonitor = new JvmPauseMonitor(daemonConf);
    pauseMonitor.start();
    String displayName = "LlapDaemonExecutorMetrics-" + MetricsUtils.getHostName();
    String sessionId = MetricsUtils.getUUID();
    daemonConf.set("llap.daemon.metrics.sessionid", sessionId);
    this.metrics = LlapDaemonExecutorMetrics.create(displayName, sessionId, numExecutors);
    metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
    this.llapDaemonInfoBean = MBeans.register("LlapDaemon", "LlapDaemonInfo", this);
    LOG.info(
        "Started LlapMetricsSystem with displayName: " + displayName + " sessionId: " + sessionId);

    this.amReporter = new AMReporter(address, new QueryFailedHandlerProxy(), daemonConf);

    this.server = new LlapDaemonProtocolServerImpl(numHandlers, this, address, rpcPort);

    this.containerRunner =
        new ContainerRunnerImpl(
            daemonConf,
            numExecutors,
            waitQueueSize,
            enablePreemption,
            localDirs,
            this.shufflePort,
            address,
            executorMemoryBytes,
            metrics,
            amReporter);
    addIfService(containerRunner);

    this.registry = new LlapRegistryService(true);
    addIfService(registry);
    if (HiveConf.getBoolVar(daemonConf, HiveConf.ConfVars.HIVE_IN_TEST)) {
      this.webServices = null;
    } else {
      this.webServices = new LlapWebServices();
      addIfService(webServices);
    }
    // Bring up the server only after all other components have started.
    addIfService(server);
    // AMReporter after the server so that it gets the correct address. It knows how to deal with
    // requests before it is started.
    addIfService(amReporter);
  }