Example #1
0
  public ATSHook() {
    synchronized (LOCK) {
      if (executor == null) {

        executor =
            Executors.newSingleThreadExecutor(
                new ThreadFactoryBuilder().setDaemon(true).setNameFormat("ATS Logger %d").build());

        YarnConfiguration yarnConf = new YarnConfiguration();
        timelineClient = TimelineClient.createTimelineClient();
        timelineClient.init(yarnConf);
        timelineClient.start();

        ShutdownHookManager.addShutdownHook(
            new Runnable() {
              @Override
              public void run() {
                try {
                  executor.shutdown();
                  executor.awaitTermination(WAIT_TIME, TimeUnit.SECONDS);
                  executor = null;
                } catch (InterruptedException ie) {
                  /* ignore */
                }
                timelineClient.stop();
              }
            });
      }
    }

    LOG.info("Created ATS Hook");
  }
Example #2
0
  public static void main(String[] args) throws Exception {
    Thread.setDefaultUncaughtExceptionHandler(new LlapDaemonUncaughtExceptionHandler());
    LlapDaemon llapDaemon = null;
    try {
      // Cache settings will need to be setup in llap-daemon-site.xml - since the daemons don't read
      // hive-site.xml
      // Ideally, these properties should be part of LlapDameonConf rather than HiveConf
      LlapConfiguration daemonConf = new LlapConfiguration();
      int numExecutors =
          daemonConf.getInt(
              LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS,
              LlapConfiguration.LLAP_DAEMON_NUM_EXECUTORS_DEFAULT);

      String[] localDirs = daemonConf.getTrimmedStrings(LlapConfiguration.LLAP_DAEMON_WORK_DIRS);
      int rpcPort =
          daemonConf.getInt(
              LlapConfiguration.LLAP_DAEMON_RPC_PORT,
              LlapConfiguration.LLAP_DAEMON_RPC_PORT_DEFAULT);
      int shufflePort =
          daemonConf.getInt(
              ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, ShuffleHandler.DEFAULT_SHUFFLE_PORT);
      long executorMemoryBytes =
          daemonConf.getInt(
                  LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB,
                  LlapConfiguration.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB_DEFAULT)
              * 1024l
              * 1024l;
      long cacheMemoryBytes =
          HiveConf.getLongVar(daemonConf, HiveConf.ConfVars.LLAP_ORC_CACHE_MAX_SIZE);
      boolean isDirectCache =
          HiveConf.getBoolVar(daemonConf, HiveConf.ConfVars.LLAP_ORC_CACHE_ALLOCATE_DIRECT);
      boolean llapIoEnabled = HiveConf.getBoolVar(daemonConf, HiveConf.ConfVars.LLAP_IO_ENABLED);
      llapDaemon =
          new LlapDaemon(
              daemonConf,
              numExecutors,
              executorMemoryBytes,
              llapIoEnabled,
              isDirectCache,
              cacheMemoryBytes,
              localDirs,
              rpcPort,
              shufflePort);

      LOG.info("Adding shutdown hook for LlapDaemon");
      ShutdownHookManager.addShutdownHook(new CompositeServiceShutdownHook(llapDaemon), 1);

      llapDaemon.init(daemonConf);
      llapDaemon.start();
      LOG.info("Started LlapDaemon");
      // Relying on the RPC threads to keep the service alive.
    } catch (Throwable t) {
      // TODO Replace this with a ExceptionHandler / ShutdownHook
      LOG.warn("Failed to start LLAP Daemon with exception", t);
      if (llapDaemon != null) {
        llapDaemon.shutdown();
      }
      System.exit(-1);
    }
  }