/**
* Log a number of keys and values with the record. This method allows to do it in a synchronous
* fashion
*
* @param writers the writers to send the data to
* @param recordType the type to log
* @param keys keys to log
* @param values values to log
* @param sync if true - will block until the data is written
*/
private void log(
ArrayList<PrintWriter> writers,
RecordTypes recordType,
Keys[] keys,
String[] values,
boolean sync) {
StringBuffer buf = new StringBuffer(recordType.name());
buf.append(JobHistory.DELIMITER);
for (int i = 0; i < keys.length; i++) {
buf.append(keys[i]);
buf.append("=\"");
values[i] = JobHistory.escapeString(values[i]);
buf.append(values[i]);
buf.append("\"");
buf.append(JobHistory.DELIMITER);
}
buf.append(JobHistory.LINE_DELIMITER_CHAR);
for (PrintWriter out : writers) {
LogTask task = new LogTask(out, buf.toString());
if (sync) {
task.run();
} else {
fileManager.addWriteTask(task);
}
}
}
public CoronaJobHistory(JobConf conf, JobID jobId, String logPath) {
try {
this.conf = conf;
this.jobId = jobId;
if (logPath == null) {
logPath =
"file:///"
+ new File(System.getProperty("hadoop.log.dir", "/tmp")).getAbsolutePath()
+ File.separator
+ "history";
}
logDir = new Path(logPath);
logDirFs = logDir.getFileSystem(conf);
if (!logDirFs.exists(logDir)) {
LOG.info("Creating history folder at " + logDir);
if (!logDirFs.mkdirs(logDir, new FsPermission(HISTORY_DIR_PERMISSION))) {
throw new IOException("Mkdirs failed to create " + logDir.toString());
}
}
conf.set("hadoop.job.history.location", logDir.toString());
disableHistory = false;
// set the job history block size (default is 3MB)
jobHistoryBlockSize =
conf.getLong("mapred.jobtracker.job.history.block.size", 3 * 1024 * 1024);
doneDir = new Path(logDir, "done");
doneDirFs = logDirFs;
if (!doneDirFs.exists(doneDir)) {
LOG.info("Creating DONE folder at " + doneDir);
if (!doneDirFs.mkdirs(doneDir, new FsPermission(HISTORY_DIR_PERMISSION))) {
throw new IOException("Mkdirs failed to create " + doneDir);
}
}
String logFileName =
encodeJobHistoryFileName(CoronaJobHistoryFilesManager.getHistoryFilename(jobId));
logFile = new Path(logDir, logFileName);
doneFile = new Path(doneDir, logFileName);
// initialize the file manager
conf.setInt("mapred.jobtracker.historythreads.maximum", 1);
fileManager =
new CoronaJobHistoryFilesManager(
conf,
new JobHistoryObserver() {
public void historyFileCopied(JobID jobid, String historyFile) {}
},
logDir);
fileManager.setDoneDir(doneDir);
// sleeping with the past means tolerating two start methods instead of one
fileManager.start();
fileManager.startIOExecutor();
} catch (IOException e) {
LOG.error("Failed to initialize JobHistory log file", e);
disableHistory = true;
}
}
/** Move the completed job into the completed folder. */
public void markCompleted() throws IOException {
if (disableHistory) {
return;
}
fileManager.moveToDone(jobId, true, CoronaJobTracker.getMainJobID(jobId));
}
/**
* Log job submitted event to history. Creates a new file in history for the job. if history file
* creation fails, it disables history for all other events.
*
* @param jobConfPath path to job conf xml file in HDFS.
* @param submitTime time when job tracker received the job
* @throws IOException
*/
public void logSubmitted(String jobConfPath, long submitTime, String jobTrackerId)
throws IOException {
if (disableHistory) {
return;
}
// create output stream for logging in hadoop.job.history.location
int defaultBufferSize = logDirFs.getConf().getInt("io.file.buffer.size", 4096);
try {
FSDataOutputStream out = null;
PrintWriter writer = null;
// In case the old JT is still running, but we can't connect to it, we
// should ensure that it won't write to our (new JT's) job history file.
if (logDirFs.exists(logFile)) {
LOG.info("Remove the old history file " + logFile);
logDirFs.delete(logFile, true);
}
out =
logDirFs.create(
logFile,
new FsPermission(HISTORY_FILE_PERMISSION),
true,
defaultBufferSize,
logDirFs.getDefaultReplication(),
jobHistoryBlockSize,
null);
writer = new PrintWriter(out);
fileManager.addWriter(jobId, writer);
// cache it ...
fileManager.setHistoryFile(jobId, logFile);
writers = fileManager.getWriters(jobId);
if (null != writers) {
log(
writers,
RecordTypes.Meta,
new Keys[] {Keys.VERSION},
new String[] {String.valueOf(JobHistory.VERSION)});
}
String jobName = getJobName();
String user = getUserName();
// add to writer as well
log(
writers,
RecordTypes.Job,
new Keys[] {
Keys.JOBID, Keys.JOBNAME, Keys.USER, Keys.SUBMIT_TIME, Keys.JOBCONF, Keys.JOBTRACKERID
},
new String[] {
jobId.toString(), jobName, user,
String.valueOf(submitTime), jobConfPath, jobTrackerId
});
} catch (IOException e) {
// Disable history if we have errors other than in the user log.
disableHistory = true;
}
/* Storing the job conf on the log dir */
Path jobFilePath = new Path(logDir, CoronaJobHistoryFilesManager.getConfFilename(jobId));
fileManager.setConfFile(jobId, jobFilePath);
FSDataOutputStream jobFileOut = null;
try {
if (!logDirFs.exists(jobFilePath)) {
jobFileOut =
logDirFs.create(
jobFilePath,
new FsPermission(HISTORY_FILE_PERMISSION),
true,
defaultBufferSize,
logDirFs.getDefaultReplication(),
logDirFs.getDefaultBlockSize(),
null);
conf.writeXml(jobFileOut);
jobFileOut.close();
}
} catch (IOException ioe) {
LOG.error("Failed to store job conf in the log dir", ioe);
} finally {
if (jobFileOut != null) {
try {
jobFileOut.close();
} catch (IOException ie) {
LOG.info(
"Failed to close the job configuration file " + StringUtils.stringifyException(ie));
}
}
}
}
public void shutdown() {
if (fileManager != null) {
fileManager.shutdown();
}
}
