@Override
public void serviceStart() throws Exception {
// Start the Shuffle service before the listener - until it's a service as well.
ShuffleHandler.initializeAndStart(shuffleHandlerConf);
LOG.info("Setting shuffle port to: " + ShuffleHandler.get().getPort());
this.shufflePort.set(ShuffleHandler.get().getPort());
super.serviceStart();
LOG.info("LlapDaemon serviceStart complete");
}
/**
* Register completion for a query
*
* @param queryIdentifier
* @param deleteDelay
*/
List<QueryFragmentInfo> queryComplete(QueryIdentifier queryIdentifier, long deleteDelay) {
if (deleteDelay == -1) {
deleteDelay = defaultDeleteDelaySeconds;
}
ReadWriteLock dagLock = getDagLock(queryIdentifier);
dagLock.writeLock().lock();
try {
rememberCompletedDag(queryIdentifier);
LOG.info(
"Processing queryComplete for queryIdentifier={} with deleteDelay={} seconds",
queryIdentifier,
deleteDelay);
QueryInfo queryInfo = queryInfoMap.remove(queryIdentifier);
if (queryInfo == null) {
LOG.warn("Ignoring query complete for unknown dag: {}", queryIdentifier);
return Collections.emptyList();
}
String[] localDirs = queryInfo.getLocalDirsNoCreate();
if (localDirs != null) {
for (String localDir : localDirs) {
cleanupDir(localDir, deleteDelay);
ShuffleHandler.get()
.unregisterDag(localDir, queryInfo.getAppIdString(), queryInfo.getDagIdentifier());
}
}
// Clearing this before sending a kill is OK, since canFinish will change to false.
// Ideally this should be a state machine where kills are issued to the executor,
// and the structures are cleaned up once all tasks complete. New requests, however,
// should not be allowed after a query complete is received.
sourceCompletionMap.remove(queryIdentifier);
String savedQueryId = queryIdentifierToHiveQueryId.remove(queryIdentifier);
dagSpecificLocks.remove(queryIdentifier);
if (savedQueryId != null) {
ObjectCacheFactory.removeLlapQueryCache(savedQueryId);
}
return queryInfo.getRegisteredFragments();
} finally {
dagLock.writeLock().unlock();
}
}
@Override
public int getShufflePort() {
return ShuffleHandler.get().getPort();
}
public void serviceStop() throws Exception {
super.serviceStop();
ShuffleHandler.shutdown();
shutdown();
LOG.info("LlapDaemon shutdown complete");
}