Ejemplo n.º 1
0
 @Override
 public void serviceStart() throws Exception {
   // Start the Shuffle service before the listener - until it's a service as well.
   ShuffleHandler.initializeAndStart(shuffleHandlerConf);
   LOG.info("Setting shuffle port to: " + ShuffleHandler.get().getPort());
   this.shufflePort.set(ShuffleHandler.get().getPort());
   super.serviceStart();
   LOG.info("LlapDaemon serviceStart complete");
 }
Ejemplo n.º 2
0
 /**
  * Register completion for a query
  *
  * @param queryIdentifier
  * @param deleteDelay
  */
 List<QueryFragmentInfo> queryComplete(QueryIdentifier queryIdentifier, long deleteDelay) {
   if (deleteDelay == -1) {
     deleteDelay = defaultDeleteDelaySeconds;
   }
   ReadWriteLock dagLock = getDagLock(queryIdentifier);
   dagLock.writeLock().lock();
   try {
     rememberCompletedDag(queryIdentifier);
     LOG.info(
         "Processing queryComplete for queryIdentifier={} with deleteDelay={} seconds",
         queryIdentifier,
         deleteDelay);
     QueryInfo queryInfo = queryInfoMap.remove(queryIdentifier);
     if (queryInfo == null) {
       LOG.warn("Ignoring query complete for unknown dag: {}", queryIdentifier);
       return Collections.emptyList();
     }
     String[] localDirs = queryInfo.getLocalDirsNoCreate();
     if (localDirs != null) {
       for (String localDir : localDirs) {
         cleanupDir(localDir, deleteDelay);
         ShuffleHandler.get()
             .unregisterDag(localDir, queryInfo.getAppIdString(), queryInfo.getDagIdentifier());
       }
     }
     // Clearing this before sending a kill is OK, since canFinish will change to false.
     // Ideally this should be a state machine where kills are issued to the executor,
     // and the structures are cleaned up once all tasks complete. New requests, however,
     // should not be allowed after a query complete is received.
     sourceCompletionMap.remove(queryIdentifier);
     String savedQueryId = queryIdentifierToHiveQueryId.remove(queryIdentifier);
     dagSpecificLocks.remove(queryIdentifier);
     if (savedQueryId != null) {
       ObjectCacheFactory.removeLlapQueryCache(savedQueryId);
     }
     return queryInfo.getRegisteredFragments();
   } finally {
     dagLock.writeLock().unlock();
   }
 }
Ejemplo n.º 3
0
 @Override
 public int getShufflePort() {
   return ShuffleHandler.get().getPort();
 }
Ejemplo n.º 4
0
 public void serviceStop() throws Exception {
   super.serviceStop();
   ShuffleHandler.shutdown();
   shutdown();
   LOG.info("LlapDaemon shutdown complete");
 }