private void createBackgroundOperationPool() {
    int poolSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS);
    LOG.info("HiveServer2: Background operation thread pool size: " + poolSize);
    int poolQueueSize = hiveConf.getIntVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_WAIT_QUEUE_SIZE);
    LOG.info("HiveServer2: Background operation thread wait queue size: " + poolQueueSize);
    long keepAliveTime =
        HiveConf.getTimeVar(
            hiveConf, ConfVars.HIVE_SERVER2_ASYNC_EXEC_KEEPALIVE_TIME, TimeUnit.SECONDS);
    LOG.info(
        "HiveServer2: Background operation thread keepalive time: " + keepAliveTime + " seconds");

    // Create a thread pool with #poolSize threads
    // Threads terminate when they are idle for more than the keepAliveTime
    // A bounded blocking queue is used to queue incoming operations, if #operations > poolSize
    String threadPoolName = "HiveServer2-Background-Pool";
    backgroundOperationPool =
        new ThreadPoolExecutor(
            poolSize,
            poolSize,
            keepAliveTime,
            TimeUnit.SECONDS,
            new LinkedBlockingQueue<Runnable>(poolQueueSize),
            new ThreadFactoryWithGarbageCleanup(threadPoolName));
    backgroundOperationPool.allowCoreThreadTimeOut(true);

    checkInterval =
        HiveConf.getTimeVar(
            hiveConf, ConfVars.HIVE_SERVER2_SESSION_CHECK_INTERVAL, TimeUnit.MILLISECONDS);
    sessionTimeout =
        HiveConf.getTimeVar(
            hiveConf, ConfVars.HIVE_SERVER2_IDLE_SESSION_TIMEOUT, TimeUnit.MILLISECONDS);
  }
Esempio n. 2
0
    public void preTest(HiveConf conf) throws Exception {

      if (zooKeeperCluster == null) {
        // create temp dir
        String tmpBaseDir = System.getProperty("test.tmp.dir");
        File tmpDir = Utilities.createTempDir(tmpBaseDir);

        zooKeeperCluster = new MiniZooKeeperCluster();
        zkPort = zooKeeperCluster.startup(tmpDir);
      }

      if (zooKeeper != null) {
        zooKeeper.close();
      }

      int sessionTimeout =
          (int)
              conf.getTimeVar(
                  HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT, TimeUnit.MILLISECONDS);
      zooKeeper =
          new ZooKeeper(
              "localhost:" + zkPort,
              sessionTimeout,
              new Watcher() {
                @Override
                public void process(WatchedEvent arg0) {}
              });

      String zkServer = "localhost";
      conf.set("hive.zookeeper.quorum", zkServer);
      conf.set("hive.zookeeper.client.port", "" + zkPort);
    }
Esempio n. 3
0
  /**
   * Send a heartbeat to the metastore for locks and transactions.
   *
   * @throws IOException
   */
  public void heartbeat() throws IOException {
    if (dontHeartbeat) return;

    if (txnMgr == null) {
      LOG.debug("txnMgr null, not heartbeating");
      dontHeartbeat = true;
      return;
    }

    if (heartbeatInterval == 0) {
      // Multiply the heartbeat interval by 1000 to convert to milliseconds,
      // but divide by 2 to give us a safety factor.
      heartbeatInterval =
          HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_TXN_TIMEOUT, TimeUnit.MILLISECONDS) / 2;
      if (heartbeatInterval == 0) {
        LOG.warn(
            HiveConf.ConfVars.HIVE_TXN_MANAGER.toString() + " not set, heartbeats won't be sent");
        dontHeartbeat = true;
        LOG.debug("heartbeat interval 0, not heartbeating");
        return;
      }
    }
    long now = System.currentTimeMillis();
    if (now - lastHeartbeat > heartbeatInterval) {
      try {
        LOG.debug("heartbeating");
        txnMgr.heartbeat();
      } catch (LockException e) {
        LOG.warn("Failed trying to heartbeat " + e.getMessage());
        throw new IOException(e);
      }
      lastHeartbeat = now;
    }
  }
Esempio n. 4
0
 public ThriftHttpServlet(
     TProcessor processor,
     TProtocolFactory protocolFactory,
     String authType,
     UserGroupInformation serviceUGI,
     UserGroupInformation httpUGI,
     HiveAuthFactory hiveAuthFactory) {
   super(processor, protocolFactory);
   this.authType = authType;
   this.serviceUGI = serviceUGI;
   this.httpUGI = httpUGI;
   this.hiveAuthFactory = hiveAuthFactory;
   this.isCookieAuthEnabled =
       hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_AUTH_ENABLED);
   // Initialize the cookie based authentication related variables.
   if (isCookieAuthEnabled) {
     // Generate the signer with secret.
     String secret = Long.toString(RAN.nextLong());
     LOG.debug("Using the random number as the secret for cookie generation " + secret);
     this.signer = new CookieSigner(secret.getBytes());
     this.cookieMaxAge =
         (int)
             hiveConf.getTimeVar(
                 ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_MAX_AGE, TimeUnit.SECONDS);
     this.cookieDomain = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_DOMAIN);
     this.cookiePath = hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_PATH);
     // always send secure cookies for SSL mode
     this.isCookieSecure = hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_USE_SSL);
     this.isHttpOnlyCookie =
         hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_IS_HTTPONLY);
   }
 }
Esempio n. 5
0
  protected RawStoreProxy(
      HiveConf hiveConf, Configuration conf, Class<? extends RawStore> rawStoreClass, int id)
      throws MetaException {
    this.conf = conf;
    this.hiveConf = hiveConf;
    this.socketTimeout =
        HiveConf.getTimeVar(
            hiveConf, HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS);

    // This has to be called before initializing the instance of RawStore
    init();

    this.base = ReflectionUtils.newInstance(rawStoreClass, conf);
  }
Esempio n. 6
0
  public QueryTracker(Configuration conf, String[] localDirsBase) {
    super("QueryTracker");
    this.localDirsBase = localDirsBase;
    try {
      localFs = FileSystem.getLocal(conf);
    } catch (IOException e) {
      throw new RuntimeException("Failed to setup local filesystem instance", e);
    }

    this.defaultDeleteDelaySeconds =
        HiveConf.getTimeVar(conf, ConfVars.LLAP_FILE_CLEANUP_DELAY_SECONDS, TimeUnit.SECONDS);

    int numCleanerThreads = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_NUM_FILE_CLEANER_THREADS);
    this.executorService =
        Executors.newScheduledThreadPool(
            numCleanerThreads,
            new ThreadFactoryBuilder()
                .setDaemon(true)
                .setNameFormat("QueryFileCleaner %d")
                .build());
  }
 @Override
 public synchronized void stop() {
   super.stop();
   shutdown = true;
   if (backgroundOperationPool != null) {
     backgroundOperationPool.shutdown();
     long timeout =
         hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT, TimeUnit.SECONDS);
     try {
       backgroundOperationPool.awaitTermination(timeout, TimeUnit.SECONDS);
     } catch (InterruptedException e) {
       LOG.warn(
           "HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT = "
               + timeout
               + " seconds has been exceeded. RUNNING background operations will be shut down",
           e);
     }
     backgroundOperationPool = null;
   }
   cleanupLoggingRootDir();
 }
Esempio n. 8
0
  /* (non-Javadoc)
   * @see org.apache.hive.service.cli.ICLIService#getOperationStatus(org.apache.hive.service.cli.OperationHandle)
   */
  @Override
  public OperationStatus getOperationStatus(OperationHandle opHandle) throws HiveSQLException {
    Operation operation = sessionManager.getOperationManager().getOperation(opHandle);
    /**
     * If this is a background operation run asynchronously, we block for a duration determined by a
     * step function, before we return However, if the background operation is complete, we return
     * immediately.
     */
    if (operation.shouldRunAsync()) {
      HiveConf conf = operation.getParentSession().getHiveConf();
      long maxTimeout =
          HiveConf.getTimeVar(
              conf, HiveConf.ConfVars.HIVE_SERVER2_LONG_POLLING_TIMEOUT, TimeUnit.MILLISECONDS);

      final long elapsed = System.currentTimeMillis() - operation.getBeginTime();
      // A step function to increase the polling timeout by 500 ms every 10 sec,
      // starting from 500 ms up to HIVE_SERVER2_LONG_POLLING_TIMEOUT
      final long timeout =
          Math.min(maxTimeout, (elapsed / TimeUnit.SECONDS.toMillis(10) + 1) * 500);

      try {
        operation.getBackgroundHandle().get(timeout, TimeUnit.MILLISECONDS);
      } catch (TimeoutException e) {
        // No Op, return to the caller since long polling timeout has expired
        LOG.trace(opHandle + ": Long polling timed out");
      } catch (CancellationException e) {
        // The background operation thread was cancelled
        LOG.trace(opHandle + ": The background operation was cancelled", e);
      } catch (ExecutionException e) {
        // The background operation thread was aborted
        LOG.warn(opHandle + ": The background operation was aborted", e);
      } catch (InterruptedException e) {
        // No op, this thread was interrupted
        // In this case, the call might return sooner than long polling timeout
      }
    }
    OperationStatus opStatus = operation.getStatus();
    LOG.debug(opHandle + ": getOperationStatus()");
    return opStatus;
  }
 public LlapTaskUmbilicalExternalClient(
     Configuration conf,
     String tokenIdentifier,
     Token<JobTokenIdentifier> sessionToken,
     LlapTaskUmbilicalExternalResponder responder,
     Token<LlapTokenIdentifier> llapToken) {
   super(LlapTaskUmbilicalExternalClient.class.getName());
   this.conf = conf;
   this.umbilical = new LlapTaskUmbilicalExternalImpl();
   this.tokenIdentifier = tokenIdentifier;
   this.sessionToken = sessionToken;
   this.responder = responder;
   this.timer = new ScheduledThreadPoolExecutor(1);
   this.connectionTimeout =
       3
           * HiveConf.getTimeVar(
               conf,
               HiveConf.ConfVars.LLAP_DAEMON_AM_LIVENESS_CONNECTION_TIMEOUT_MS,
               TimeUnit.MILLISECONDS);
   // Add support for configurable threads, however 1 should always be enough.
   this.communicator = new LlapProtocolClientProxy(1, conf, llapToken);
   this.communicator.init(conf);
 }
Esempio n. 10
0
  public Result invokeInternal(final Object proxy, final Method method, final Object[] args)
      throws Throwable {

    boolean gotNewConnectUrl = false;
    boolean reloadConf = HiveConf.getBoolVar(origConf, HiveConf.ConfVars.HMSHANDLERFORCERELOADCONF);
    long retryInterval =
        HiveConf.getTimeVar(origConf, HiveConf.ConfVars.HMSHANDLERINTERVAL, TimeUnit.MILLISECONDS);
    int retryLimit = HiveConf.getIntVar(origConf, HiveConf.ConfVars.HMSHANDLERATTEMPTS);
    long timeout =
        HiveConf.getTimeVar(
            origConf, HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS);

    Deadline.registerIfNot(timeout);

    if (reloadConf) {
      MetaStoreInit.updateConnectionURL(origConf, getActiveConf(), null, metaStoreInitData);
    }

    int retryCount = 0;
    Throwable caughtException = null;
    while (true) {
      try {
        if (reloadConf || gotNewConnectUrl) {
          baseHandler.setConf(getActiveConf());
        }
        Object object = null;
        boolean isStarted = Deadline.startTimer(method.getName());
        try {
          object = method.invoke(baseHandler, args);
        } finally {
          if (isStarted) {
            Deadline.stopTimer();
          }
        }
        return new Result(object, retryCount);

      } catch (javax.jdo.JDOException e) {
        caughtException = e;
      } catch (UndeclaredThrowableException e) {
        if (e.getCause() != null) {
          if (e.getCause() instanceof javax.jdo.JDOException) {
            // Due to reflection, the jdo exception is wrapped in
            // invocationTargetException
            caughtException = e.getCause();
          } else if (e.getCause() instanceof MetaException
              && e.getCause().getCause() != null
              && e.getCause().getCause() instanceof javax.jdo.JDOException) {
            // The JDOException may be wrapped further in a MetaException
            caughtException = e.getCause().getCause();
          } else {
            LOG.error(ExceptionUtils.getStackTrace(e.getCause()));
            throw e.getCause();
          }
        } else {
          LOG.error(ExceptionUtils.getStackTrace(e));
          throw e;
        }
      } catch (InvocationTargetException e) {
        if (e.getCause() instanceof javax.jdo.JDOException) {
          // Due to reflection, the jdo exception is wrapped in
          // invocationTargetException
          caughtException = e.getCause();
        } else if (e.getCause() instanceof NoSuchObjectException
            || e.getTargetException().getCause() instanceof NoSuchObjectException) {
          String methodName = method.getName();
          if (!methodName.startsWith("get_database")
              && !methodName.startsWith("get_table")
              && !methodName.startsWith("get_partition")
              && !methodName.startsWith("get_function")) {
            LOG.error(ExceptionUtils.getStackTrace(e.getCause()));
          }
          throw e.getCause();
        } else if (e.getCause() instanceof MetaException && e.getCause().getCause() != null) {
          if (e.getCause().getCause() instanceof javax.jdo.JDOException
              || e.getCause().getCause() instanceof NucleusException) {
            // The JDOException or the Nucleus Exception may be wrapped further in a MetaException
            caughtException = e.getCause().getCause();
          } else if (e.getCause().getCause() instanceof DeadlineException) {
            // The Deadline Exception needs no retry and be thrown immediately.
            Deadline.clear();
            LOG.error(
                "Error happens in method "
                    + method.getName()
                    + ": "
                    + ExceptionUtils.getStackTrace(e.getCause()));
            throw e.getCause();
          } else {
            LOG.error(ExceptionUtils.getStackTrace(e.getCause()));
            throw e.getCause();
          }
        } else {
          LOG.error(ExceptionUtils.getStackTrace(e.getCause()));
          throw e.getCause();
        }
      }

      if (retryCount >= retryLimit) {
        LOG.error("HMSHandler Fatal error: " + ExceptionUtils.getStackTrace(caughtException));
        // Since returning exceptions with a nested "cause" can be a problem in
        // Thrift, we are stuffing the stack trace into the message itself.
        throw new MetaException(ExceptionUtils.getStackTrace(caughtException));
      }

      assert (retryInterval >= 0);
      retryCount++;
      LOG.error(
          String.format(
                  "Retrying HMSHandler after %d ms (attempt %d of %d)",
                  retryInterval, retryCount, retryLimit)
              + " with error: "
              + ExceptionUtils.getStackTrace(caughtException));

      Thread.sleep(retryInterval);
      // If we have a connection error, the JDO connection URL hook might
      // provide us with a new URL to access the datastore.
      String lastUrl = MetaStoreInit.getConnectionURL(getActiveConf());
      gotNewConnectUrl =
          MetaStoreInit.updateConnectionURL(origConf, getActiveConf(), lastUrl, metaStoreInitData);
    }
  }