@Override
protected void serviceInit(Configuration conf) throws Exception {
this.conf = conf;
int serialNumberLowDigits = 3;
serialNumberFormat =
("%0" + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d");
String doneDirPrefix = null;
doneDirPrefix = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
try {
doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(new Path(doneDirPrefix));
doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf);
doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK);
mkdir(
doneDirFc,
doneDirPrefixPath,
new FsPermission(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
} catch (IOException e) {
throw new YarnRuntimeException(
"Error creating done directory: [" + doneDirPrefixPath + "]", e);
}
String intermediateDoneDirPrefix = null;
intermediateDoneDirPrefix = JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
try {
intermediateDoneDirPath =
FileContext.getFileContext(conf).makeQualified(new Path(intermediateDoneDirPrefix));
intermediateDoneDirFc = FileContext.getFileContext(intermediateDoneDirPath.toUri(), conf);
mkdir(
intermediateDoneDirFc,
intermediateDoneDirPath,
new FsPermission(JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS.toShort()));
} catch (IOException e) {
LOG.info("error creating done directory on dfs " + e);
throw new YarnRuntimeException(
"Error creating intermediate done directory: [" + intermediateDoneDirPath + "]", e);
}
this.aclsMgr = new JobACLsManager(conf);
maxHistoryAge =
conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);
jobListCache =
new JobListCache(
conf.getInt(
JHAdminConfig.MR_HISTORY_JOBLIST_CACHE_SIZE,
JHAdminConfig.DEFAULT_MR_HISTORY_JOBLIST_CACHE_SIZE),
maxHistoryAge);
serialNumberIndex =
new SerialNumberIndex(
conf.getInt(
JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));
int numMoveThreads =
conf.getInt(
JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
ThreadFactory tf =
new ThreadFactoryBuilder().setNameFormat("MoveIntermediateToDone Thread #%d").build();
moveToDoneExecutor =
new ThreadPoolExecutor(
numMoveThreads,
numMoveThreads,
1,
TimeUnit.HOURS,
new LinkedBlockingQueue<Runnable>(),
tf);
super.serviceInit(conf);
}
/**
* Returns TRUE if the history dirs were created, FALSE if they could not be created because the
* FileSystem is not reachable or in safe mode and throws and exception otherwise.
*/
@VisibleForTesting
boolean tryCreatingHistoryDirs(boolean logWait) throws IOException {
boolean succeeded = true;
String doneDirPrefix = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
try {
doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(new Path(doneDirPrefix));
doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf);
doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK);
mkdir(
doneDirFc,
doneDirPrefixPath,
new FsPermission(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
} catch (ConnectException ex) {
if (logWait) {
/* LOG.info("Waiting for FileSystem at "+doneDirPrefixPath.toUri().getAuthority()+"to be available") */
LOG.waiting_for_filesystem_available(
String.valueOf(doneDirPrefixPath.toUri().getAuthority()))
.tag("methodCall")
.info();
}
succeeded = false;
} catch (IOException e) {
if (isBecauseSafeMode(e)) {
succeeded = false;
if (logWait) {
/* LOG.info("Waiting for FileSystem at "+doneDirPrefixPath.toUri().getAuthority()+"to be out of safe mode") */
LOG.waiting_for_filesystem_out_safe_mode(
String.valueOf(doneDirPrefixPath.toUri().getAuthority()))
.tag("methodCall")
.info();
}
} else {
throw new YarnRuntimeException(
"Error creating done directory: [" + doneDirPrefixPath + "]", e);
}
}
if (succeeded) {
String intermediateDoneDirPrefix =
JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
try {
intermediateDoneDirPath =
FileContext.getFileContext(conf).makeQualified(new Path(intermediateDoneDirPrefix));
intermediateDoneDirFc = FileContext.getFileContext(intermediateDoneDirPath.toUri(), conf);
mkdir(
intermediateDoneDirFc,
intermediateDoneDirPath,
new FsPermission(JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS.toShort()));
} catch (ConnectException ex) {
succeeded = false;
if (logWait) {
/* LOG.info("Waiting for FileSystem at "+intermediateDoneDirPath.toUri().getAuthority()+"to be available") */
LOG.waiting_for_filesystem_available(
String.valueOf(intermediateDoneDirPath.toUri().getAuthority()))
.tag("methodCall")
.info();
}
} catch (IOException e) {
if (isBecauseSafeMode(e)) {
succeeded = false;
if (logWait) {
/* LOG.info("Waiting for FileSystem at "+intermediateDoneDirPath.toUri().getAuthority()+"to be out of safe mode") */
LOG.waiting_for_filesystem_out_safe_mode(
String.valueOf(intermediateDoneDirPath.toUri().getAuthority()))
.tag("methodCall")
.info();
}
} else {
throw new YarnRuntimeException(
"Error creating intermediate done directory: [" + intermediateDoneDirPath + "]", e);
}
}
}
return succeeded;
}
