예제 #1
0
파일: LlapDaemon.java 프로젝트: Leolh/hive
  public void shutdown() {
    LOG.info("LlapDaemon shutdown invoked");
    if (llapDaemonInfoBean != null) {
      MBeans.unregister(llapDaemonInfoBean);
    }

    if (pauseMonitor != null) {
      pauseMonitor.stop();
    }

    if (metrics != null) {
      LlapMetricsSystem.shutdown();
    }

    LlapIoProxy.close();
  }
예제 #2
0
파일: LlapDaemon.java 프로젝트: Leolh/hive
  public LlapDaemon(
      Configuration daemonConf,
      int numExecutors,
      long executorMemoryBytes,
      boolean ioEnabled,
      boolean isDirectCache,
      long ioMemoryBytes,
      String[] localDirs,
      int rpcPort,
      int shufflePort) {
    super("LlapDaemon");

    printAsciiArt();

    Preconditions.checkArgument(numExecutors > 0);
    Preconditions.checkArgument(
        rpcPort == 0 || (rpcPort > 1024 && rpcPort < 65536),
        "RPC Port must be between 1025 and 65535, or 0 automatic selection");
    Preconditions.checkArgument(
        localDirs != null && localDirs.length > 0, "Work dirs must be specified");
    Preconditions.checkArgument(
        shufflePort == 0 || (shufflePort > 1024 && shufflePort < 65536),
        "Shuffle Port must be betwee 1024 and 65535, or 0 for automatic selection");

    this.maxJvmMemory = getTotalHeapSize();
    this.llapIoEnabled = ioEnabled;
    this.executorMemoryPerInstance = executorMemoryBytes;
    this.ioMemoryPerInstance = ioMemoryBytes;
    this.numExecutors = numExecutors;
    this.localDirs = localDirs;

    int waitQueueSize =
        daemonConf.getInt(
            LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE,
            LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE_DEFAULT);
    boolean enablePreemption =
        daemonConf.getBoolean(
            LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_ENABLE_PREEMPTION,
            LlapConfiguration.LLAP_DAEMON_TASK_SCHEDULER_ENABLE_PREEMPTION_DEFAULT);
    LOG.info(
        "Attempting to start LlapDaemonConf with the following configuration: "
            + "numExecutors="
            + numExecutors
            + ", rpcListenerPort="
            + rpcPort
            + ", workDirs="
            + Arrays.toString(localDirs)
            + ", shufflePort="
            + shufflePort
            + ", executorMemory="
            + executorMemoryBytes
            + ", llapIoEnabled="
            + ioEnabled
            + ", llapIoCacheIsDirect="
            + isDirectCache
            + ", llapIoCacheSize="
            + ioMemoryBytes
            + ", jvmAvailableMemory="
            + maxJvmMemory
            + ", waitQueueSize= "
            + waitQueueSize
            + ", enablePreemption= "
            + enablePreemption);

    long memRequired =
        executorMemoryBytes + (ioEnabled && isDirectCache == false ? ioMemoryBytes : 0);
    // TODO: this check is somewhat bogus as the maxJvmMemory != Xmx parameters (see annotation in
    // LlapServiceDriver)
    Preconditions.checkState(
        maxJvmMemory >= memRequired,
        "Invalid configuration. Xmx value too small. maxAvailable="
            + maxJvmMemory
            + ", configured(exec + io if enabled)="
            + memRequired);

    this.shuffleHandlerConf = new Configuration(daemonConf);
    this.shuffleHandlerConf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, shufflePort);
    this.shuffleHandlerConf.set(
        ShuffleHandler.SHUFFLE_HANDLER_LOCAL_DIRS, StringUtils.arrayToString(localDirs));
    this.shuffleHandlerConf.setBoolean(
        ShuffleHandler.SHUFFLE_DIR_WATCHER_ENABLED,
        daemonConf.getBoolean(
            LlapConfiguration.LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED,
            LlapConfiguration.LLAP_DAEMON_SHUFFLE_DIR_WATCHER_ENABLED_DEFAULT));

    // Less frequently set parameter, not passing in as a param.
    int numHandlers =
        daemonConf.getInt(
            LlapConfiguration.LLAP_DAEMON_RPC_NUM_HANDLERS,
            LlapConfiguration.LLAP_DAEMON_RPC_NUM_HANDLERS_DEFAULT);

    // Initialize the metrics system
    LlapMetricsSystem.initialize("LlapDaemon");
    this.pauseMonitor = new JvmPauseMonitor(daemonConf);
    pauseMonitor.start();
    String displayName = "LlapDaemonExecutorMetrics-" + MetricsUtils.getHostName();
    String sessionId = MetricsUtils.getUUID();
    daemonConf.set("llap.daemon.metrics.sessionid", sessionId);
    this.metrics = LlapDaemonExecutorMetrics.create(displayName, sessionId, numExecutors);
    metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
    this.llapDaemonInfoBean = MBeans.register("LlapDaemon", "LlapDaemonInfo", this);
    LOG.info(
        "Started LlapMetricsSystem with displayName: " + displayName + " sessionId: " + sessionId);

    this.amReporter = new AMReporter(address, new QueryFailedHandlerProxy(), daemonConf);

    this.server = new LlapDaemonProtocolServerImpl(numHandlers, this, address, rpcPort);

    this.containerRunner =
        new ContainerRunnerImpl(
            daemonConf,
            numExecutors,
            waitQueueSize,
            enablePreemption,
            localDirs,
            this.shufflePort,
            address,
            executorMemoryBytes,
            metrics,
            amReporter);
    addIfService(containerRunner);

    this.registry = new LlapRegistryService(true);
    addIfService(registry);
    if (HiveConf.getBoolVar(daemonConf, HiveConf.ConfVars.HIVE_IN_TEST)) {
      this.webServices = null;
    } else {
      this.webServices = new LlapWebServices();
      addIfService(webServices);
    }
    // Bring up the server only after all other components have started.
    addIfService(server);
    // AMReporter after the server so that it gets the correct address. It knows how to deal with
    // requests before it is started.
    addIfService(amReporter);
  }