/**
* Start the child process to handle the task for us.
*
* @param conf the task's configuration
* @param recordReader the fake record reader to update progress with
* @param output the collector to send output to
* @param reporter the reporter for the task
* @param outputKeyClass the class of the output keys
* @param outputValueClass the class of the output values
* @throws IOException
* @throws InterruptedException
*/
Application(
JobConf conf,
RecordReader<FloatWritable, NullWritable> recordReader,
OutputCollector<K2, V2> output,
Reporter reporter,
Class<? extends K2> outputKeyClass,
Class<? extends V2> outputValueClass)
throws IOException, InterruptedException {
serverSocket = new ServerSocket(0);
Map<String, String> env = new HashMap<String, String>();
// add TMPDIR environment variable with the value of java.io.tmpdir
env.put("TMPDIR", System.getProperty("java.io.tmpdir"));
env.put("hadoop.pipes.command.port", Integer.toString(serverSocket.getLocalPort()));
List<String> cmd = new ArrayList<String>();
String executable = DistributedCache.getLocalCacheFiles(conf)[0].toString();
FileUtil.chmod(executable, "a+x");
cmd.add(executable);
// wrap the command in a stdout/stderr capture
TaskAttemptID taskid = TaskAttemptID.forName(conf.get("mapred.task.id"));
File stdout = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDOUT);
File stderr = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDERR);
long logLength = TaskLog.getTaskLogLength(conf);
cmd = TaskLog.captureOutAndError(cmd, stdout, stderr, logLength);
process = runClient(cmd, env);
clientSocket = serverSocket.accept();
handler = new OutputHandler<K2, V2>(output, reporter, recordReader);
K2 outputKey = (K2) ReflectionUtils.newInstance(outputKeyClass, conf);
V2 outputValue = (V2) ReflectionUtils.newInstance(outputValueClass, conf);
downlink =
new BinaryProtocol<K1, V1, K2, V2>(clientSocket, handler, outputKey, outputValue, conf);
downlink.start();
downlink.setJobConf(conf);
}
/**
* Prepare the log files for the task
*
* @param taskid
* @param isCleanup
* @return an array of files. The first file is stdout, the second is stderr.
* @throws IOException
*/
File[] prepareLogFiles(TaskAttemptID taskid, boolean isCleanup) throws IOException {
File[] logFiles = new File[2];
logFiles[0] = TaskLog.getTaskLogFile(taskid, isCleanup, TaskLog.LogName.STDOUT);
logFiles[1] = TaskLog.getTaskLogFile(taskid, isCleanup, TaskLog.LogName.STDERR);
getTracker().getTaskController().createLogDir(taskid, isCleanup);
return logFiles;
}
/**
* Prepare the log files for the task
*
* @param taskid
* @param isCleanup
* @return an array of files. The first file is stdout, the second is stderr.
* @throws IOException
*/
File[] prepareLogFiles(TaskAttemptID taskid, boolean isCleanup) throws IOException {
File[] logFiles = new File[2];
logFiles[0] = TaskLog.getTaskLogFile(taskid, isCleanup, TaskLog.LogName.STDOUT);
logFiles[1] = TaskLog.getTaskLogFile(taskid, isCleanup, TaskLog.LogName.STDERR);
File logDir = logFiles[0].getParentFile();
boolean b = logDir.mkdirs();
if (!b) {
LOG.warn("mkdirs failed. Ignoring");
} else {
Localizer.PermissionsHandler.setPermissions(
logDir, Localizer.PermissionsHandler.sevenZeroZero);
}
return logFiles;
}
/**
* Clears all the logs in userlog directory.
*
* <p>Adds the job directories for deletion with default retain hours. Deletes all other
* directories, if any. This is usually called on reinit/restart of the TaskTracker
*
* @param conf
* @throws IOException
*/
void clearOldUserLogs(Configuration conf) throws IOException {
File userLogDir = TaskLog.getUserLogDir();
if (userLogDir.exists()) {
String[] logDirs = userLogDir.list();
if (logDirs.length > 0) {
// add all the log dirs to taskLogsMnonitor.
long now = clock.getTime();
for (String logDir : logDirs) {
if (logDir.equals(logAsyncDisk.TOBEDELETED)) {
// skip this
continue;
}
JobID jobid = null;
try {
jobid = JobID.forName(logDir);
} catch (IllegalArgumentException ie) {
// if the directory is not a jobid, delete it immediately
deleteLogPath(new File(userLogDir, logDir).getAbsolutePath());
continue;
}
// add the job log directory with default retain hours, if it is not
// already added
if (!completedJobs.containsKey(jobid)) {
markJobLogsForDeletion(now, conf, jobid);
}
}
}
}
}
private File localizeJob(JobID jobid) throws IOException {
String user = UserGroupInformation.getCurrentUser().getShortUserName();
new JobLocalizer(tt.getJobConf(), user, jobid.toString()).initializeJobLogDir();
File jobUserlog = TaskLog.getJobDir(jobid);
JobConf conf = new JobConf();
// localize job log directory
tt.saveLogDir(jobid, conf);
assertTrue(jobUserlog + " directory is not created.", jobUserlog.exists());
return jobUserlog;
}
@Override
public void activateOptions() {
synchronized (this) {
if (maxEvents > 0) {
tail = new LinkedList<LoggingEvent>();
}
setFile(
TaskLog.getTaskLogFile(TaskAttemptID.forName(taskId), isCleanup, TaskLog.LogName.SYSLOG)
.toString());
setAppend(true);
super.activateOptions();
}
}
@Override
public void initializeJob(
String user,
String jobid,
Path credentials,
Path jobConf,
TaskUmbilicalProtocol taskTracker,
InetSocketAddress ttAddr)
throws IOException {
List<String> command =
new ArrayList<String>(
Arrays.asList(
taskControllerExe,
user,
localStorage.getDirsString(),
Integer.toString(Commands.INITIALIZE_JOB.getValue()),
jobid,
credentials.toUri().getPath().toString(),
jobConf.toUri().getPath().toString()));
File jvm = // use same jvm as parent
new File(new File(System.getProperty("java.home"), "bin"), "java");
command.add(jvm.toString());
command.add("-classpath");
command.add(System.getProperty("java.class.path"));
command.add("-Dhadoop.log.dir=" + TaskLog.getBaseLogDir());
command.add("-Dhadoop.root.logger=INFO,console");
command.add(JobLocalizer.class.getName()); // main of JobLocalizer
command.add(user);
command.add(jobid);
// add the task tracker's reporting address
command.add(ttAddr.getHostName());
command.add(Integer.toString(ttAddr.getPort()));
String[] commandArray = command.toArray(new String[0]);
ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
if (LOG.isDebugEnabled()) {
LOG.debug("initializeJob: " + Arrays.toString(commandArray));
}
try {
shExec.execute();
if (LOG.isDebugEnabled()) {
logOutput(shExec.getOutput());
}
} catch (ExitCodeException e) {
int exitCode = shExec.getExitCode();
logOutput(shExec.getOutput());
throw new IOException(
"Job initialization failed (" + exitCode + ") with output: " + shExec.getOutput(), e);
}
}
void processCompletedJobs() throws IOException {
long now = clock.getTime();
// iterate through completedJobs and remove old logs.
synchronized (completedJobs) {
Iterator<Entry<JobID, Long>> completedJobIter = completedJobs.entrySet().iterator();
while (completedJobIter.hasNext()) {
Entry<JobID, Long> entry = completedJobIter.next();
// see if the job is old enough
if (entry.getValue().longValue() <= now) {
// add the job logs directory to for delete
deleteLogPath(TaskLog.getJobDir(entry.getKey()).getAbsolutePath());
completedJobIter.remove();
}
}
}
}
@Override
public void truncateLogsAsUser(String user, List<Task> allAttempts) throws IOException {
Task firstTask = allAttempts.get(0);
String taskid = firstTask.getTaskID().toString();
LocalDirAllocator ldirAlloc = new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY);
String taskRanFile = TaskTracker.TT_LOG_TMP_DIR + Path.SEPARATOR + taskid;
Configuration conf = getConf();
// write the serialized task information to a file to pass to the truncater
Path taskRanFilePath = ldirAlloc.getLocalPathForWrite(taskRanFile, conf);
LocalFileSystem lfs = FileSystem.getLocal(conf);
FSDataOutputStream out = lfs.create(taskRanFilePath);
out.writeInt(allAttempts.size());
for (Task t : allAttempts) {
out.writeBoolean(t.isMapTask());
t.write(out);
}
out.close();
lfs.setPermission(taskRanFilePath, FsPermission.createImmutable((short) 0755));
List<String> command = new ArrayList<String>();
File jvm = // use same jvm as parent
new File(new File(System.getProperty("java.home"), "bin"), "java");
command.add(jvm.toString());
command.add("-Djava.library.path=" + System.getProperty("java.library.path"));
command.add("-Dhadoop.log.dir=" + TaskLog.getBaseLogDir());
command.add("-Dhadoop.root.logger=INFO,console");
command.add("-classpath");
command.add(System.getProperty("java.class.path"));
// main of TaskLogsTruncater
command.add(TaskLogsTruncater.class.getName());
command.add(taskRanFilePath.toString());
String[] taskControllerCmd = new String[4 + command.size()];
taskControllerCmd[0] = taskControllerExe;
taskControllerCmd[1] = user;
taskControllerCmd[2] = localStorage.getDirsString();
taskControllerCmd[3] = Integer.toString(Commands.RUN_COMMAND_AS_USER.getValue());
int i = 4;
for (String cmdArg : command) {
taskControllerCmd[i++] = cmdArg;
}
if (LOG.isDebugEnabled()) {
for (String cmd : taskControllerCmd) {
LOG.debug("taskctrl command = " + cmd);
}
}
ShellCommandExecutor shExec = new ShellCommandExecutor(taskControllerCmd);
try {
shExec.execute();
} catch (Exception e) {
LOG.warn(
"Exit code from "
+ taskControllerExe.toString()
+ " is : "
+ shExec.getExitCode()
+ " for truncateLogs");
LOG.warn(
"Exception thrown by "
+ taskControllerExe.toString()
+ " : "
+ StringUtils.stringifyException(e));
LOG.info("Output from LinuxTaskController's " + taskControllerExe.toString() + " follows:");
logOutput(shExec.getOutput());
lfs.delete(taskRanFilePath, false);
throw new IOException(e);
}
lfs.delete(taskRanFilePath, false);
if (LOG.isDebugEnabled()) {
LOG.info("Output from LinuxTaskController's " + taskControllerExe.toString() + " follows:");
logOutput(shExec.getOutput());
}
}
@Override
public final void run() {
String errorInfo = "Child Error";
try {
// before preparing the job localize
// all the archives
TaskAttemptID taskid = t.getTaskID();
final LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir");
final File workDir = formWorkDir(lDirAlloc, taskid, t.isTaskCleanupTask(), conf);
// We don't create any symlinks yet, so presence/absence of workDir
// actually on the file system doesn't matter.
tip.getUGI()
.doAs(
new PrivilegedExceptionAction<Void>() {
public Void run() throws IOException {
taskDistributedCacheManager =
tracker
.getTrackerDistributedCacheManager()
.newTaskDistributedCacheManager(conf);
taskDistributedCacheManager.setup(
lDirAlloc,
workDir,
TaskTracker.getPrivateDistributedCacheDir(conf.getUser()),
TaskTracker.getPublicDistributedCacheDir());
return null;
}
});
// Set up the child task's configuration. After this call, no localization
// of files should happen in the TaskTracker's process space. Any changes to
// the conf object after this will NOT be reflected to the child.
setupChildTaskConfiguration(lDirAlloc);
if (!prepare()) {
return;
}
// Accumulates class paths for child.
List<String> classPaths = getClassPaths(conf, workDir, taskDistributedCacheManager);
long logSize = TaskLog.getTaskLogLength(conf);
// Build exec child JVM args.
Vector<String> vargs = getVMArgs(taskid, workDir, classPaths, logSize);
tracker.addToMemoryManager(t.getTaskID(), t.isMapTask(), conf);
// set memory limit using ulimit if feasible and necessary ...
List<String> setup = getVMSetupCmd();
// Set up the redirection of the task's stdout and stderr streams
File[] logFiles = prepareLogFiles(taskid, t.isTaskCleanupTask());
File stdout = logFiles[0];
File stderr = logFiles[1];
List<TaskTrackerInstrumentation> ttInstrumentations =
tracker.getTaskTrackerInstrumentations();
for (TaskTrackerInstrumentation inst : ttInstrumentations) {
inst.reportTaskLaunch(taskid, stdout, stderr);
}
Map<String, String> env = new HashMap<String, String>();
errorInfo = getVMEnvironment(errorInfo, workDir, conf, env, taskid, logSize);
launchJvmAndWait(setup, vargs, stdout, stderr, logSize, workDir, env);
for (TaskTrackerInstrumentation inst : ttInstrumentations) {
inst.reportTaskEnd(t.getTaskID());
}
if (exitCodeSet) {
if (!killed && exitCode != 0) {
if (exitCode == 65) {
for (TaskTrackerInstrumentation inst : ttInstrumentations) {
inst.taskFailedPing(t.getTaskID());
}
}
throw new IOException("Task process exit with nonzero status of " + exitCode + ".");
}
}
} catch (FSError e) {
LOG.fatal("FSError", e);
try {
tracker.internalFsError(t.getTaskID(), e.getMessage());
} catch (IOException ie) {
LOG.fatal(t.getTaskID() + " reporting FSError", ie);
}
} catch (Throwable throwable) {
LOG.warn(t.getTaskID() + " : " + errorInfo, throwable);
Throwable causeThrowable = new Throwable(errorInfo, throwable);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
causeThrowable.printStackTrace(new PrintStream(baos));
try {
tracker.internalReportDiagnosticInfo(t.getTaskID(), baos.toString());
} catch (IOException e) {
LOG.warn(t.getTaskID() + " Reporting Diagnostics", e);
}
} finally {
try {
if (taskDistributedCacheManager != null) {
taskDistributedCacheManager.release();
}
} catch (IOException ie) {
LOG.warn("Error releasing caches : Cache files might not have been cleaned up");
}
// It is safe to call TaskTracker.TaskInProgress.reportTaskFinished with
// *false* since the task has either
// a) SUCCEEDED - which means commit has been done
// b) FAILED - which means we do not need to commit
tip.reportTaskFinished(false);
}
}
UserLogCleaner(Configuration conf) throws IOException {
threadSleepTime = conf.getLong(TTConfig.TT_USERLOGCLEANUP_SLEEPTIME, DEFAULT_THREAD_SLEEP_TIME);
logAsyncDisk =
new MRAsyncDiskService(FileSystem.getLocal(conf), TaskLog.getUserLogDir().toString());
setClock(new Clock());
}
@Override
public final void run() {
String errorInfo = "Child Error";
try {
// before preparing the job localize
// all the archives
TaskAttemptID taskid = t.getTaskID();
final LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir");
// simply get the location of the workDir and pass it to the child. The
// child will do the actual dir creation
final File workDir =
new File(
new Path(
localdirs[rand.nextInt(localdirs.length)],
TaskTracker.getTaskWorkDir(
t.getUser(),
taskid.getJobID().toString(),
taskid.toString(),
t.isTaskCleanupTask()))
.toString());
// Set up the child task's configuration. After this call, no localization
// of files should happen in the TaskTracker's process space. Any changes to
// the conf object after this will NOT be reflected to the child.
// setupChildTaskConfiguration(lDirAlloc);
if (!prepare()) {
return;
}
// Accumulates class paths for child.
List<String> classPaths = getClassPaths(conf, workDir, taskDistributedCacheManager);
long logSize = TaskLog.getTaskLogLength(conf);
// Build exec child JVM args.
Vector<String> vargs = getVMArgs(taskid, workDir, classPaths, logSize);
tracker.addToMemoryManager(t.getTaskID(), t.isMapTask(), conf);
// set memory limit using ulimit if feasible and necessary ...
String setup = getVMSetupCmd();
// Set up the redirection of the task's stdout and stderr streams
File[] logFiles = prepareLogFiles(taskid, t.isTaskCleanupTask());
File stdout = logFiles[0];
File stderr = logFiles[1];
tracker.getTaskTrackerInstrumentation().reportTaskLaunch(taskid, stdout, stderr);
Map<String, String> env = new HashMap<String, String>();
errorInfo = getVMEnvironment(errorInfo, workDir, conf, env, taskid, logSize);
// flatten the env as a set of export commands
List<String> setupCmds = new ArrayList<String>();
appendEnvExports(setupCmds, env);
setupCmds.add(setup);
launchJvmAndWait(setupCmds, vargs, stdout, stderr, logSize, workDir);
tracker.getTaskTrackerInstrumentation().reportTaskEnd(t.getTaskID());
if (exitCodeSet) {
if (!killed && exitCode != 0) {
if (exitCode == 65) {
tracker.getTaskTrackerInstrumentation().taskFailedPing(t.getTaskID());
}
throw new IOException("Task process exit with nonzero status of " + exitCode + ".");
}
}
} catch (FSError e) {
LOG.fatal("FSError", e);
try {
tracker.internalFsError(t.getTaskID(), e.getMessage());
} catch (IOException ie) {
LOG.fatal(t.getTaskID() + " reporting FSError", ie);
}
} catch (Throwable throwable) {
LOG.warn(t.getTaskID() + " : " + errorInfo, throwable);
Throwable causeThrowable = new Throwable(errorInfo, throwable);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
causeThrowable.printStackTrace(new PrintStream(baos));
try {
tracker.internalReportDiagnosticInfo(t.getTaskID(), baos.toString());
} catch (IOException e) {
LOG.warn(t.getTaskID() + " Reporting Diagnostics", e);
}
} finally {
// It is safe to call TaskTracker.TaskInProgress.reportTaskFinished with
// *false* since the task has either
// a) SUCCEEDED - which means commit has been done
// b) FAILED - which means we do not need to commit
tip.reportTaskFinished(false);
}
}
@SuppressWarnings("deprecation")
@Override
public void run() {
Task task = getTask();
TaskAttemptID taskid = task.getTaskID();
try {
if (!prepare()) {
return;
}
String sep = System.getProperty("path.separator");
StringBuffer classPath = new StringBuffer();
// The alternate runtime can be used to debug tasks by putting a
// custom version of the mapred libraries. This will get loaded before
// the TT's jars.
String debugRuntime = conf.get("mapred.task.debug.runtime.classpath");
if (debugRuntime != null) {
classPath.append(debugRuntime);
classPath.append(sep);
}
// start with same classpath as parent process
String systemClassPath = System.getProperty("java.class.path");
if (releasePath != null
&& !releasePath.isEmpty()
&& originalPath != null
&& !releasePath.isEmpty()) {
systemClassPath = systemClassPath.replaceAll(originalPath, releasePath);
}
classPath.append(systemClassPath);
classPath.append(sep);
// Build exec child jmv args.
Vector<String> vargs = new Vector<String>(8);
File jvm = // use same jvm as parent
new File(new File(System.getProperty("java.home"), "bin"), "java");
vargs.add(jvm.toString());
// Add child (task) java-vm options.
String javaOpts = getCJTJavaOpts(conf);
javaOpts = javaOpts.replace("@taskid@", taskid.toString());
String[] javaOptsSplit = javaOpts.split(" ");
for (int i = 0; i < javaOptsSplit.length; i++) {
vargs.add(javaOptsSplit[i]);
}
// add java.io.tmpdir given by mapred.child.tmp
String tmp = conf.get("mapred.child.tmp", "./tmp");
Path tmpDir = new Path(tmp);
// if temp directory path is not absolute
// prepend it with workDir.
if (!tmpDir.isAbsolute()) {
tmpDir = new Path(workDir.toString(), tmp);
}
FileSystem localFs = FileSystem.getLocal(conf);
if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) {
throw new IOException("Mkdirs failed to create " + tmpDir.toString());
}
vargs.add("-Djava.io.tmpdir=" + tmpDir.toString());
// Add classpath.
vargs.add("-classpath");
vargs.add(classPath.toString());
// Setup the log4j prop
long logSize = TaskLog.getTaskLogLength(conf);
vargs.add("-Dhadoop.log.dir=" + CoronaTaskTracker.jobTrackerLogDir());
boolean logToScribe = conf.getBoolean("mapred.task.log.scribe", false);
if (logToScribe) {
vargs.addAll(conf.getStringCollection("mapred.task.log.scribe.conf"));
}
String logger = logToScribe ? "INFO,TLA,scribe" : "INFO,TLA";
vargs.add("-Dhadoop.root.logger=" + logger);
vargs.add("-Dhadoop.tasklog.taskid=" + taskid);
vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize);
Path systemDirectory = tracker.systemDirectory;
if (!systemDirectory.isAbsolute()) {
systemDirectory = new Path(tracker.systemFS.getWorkingDirectory(), systemDirectory);
}
systemDirectory = systemDirectory.makeQualified(tracker.systemFS);
vargs.add("-Dmapred.system.dir=" + systemDirectory);
// Add main class and its arguments
vargs.add(CoronaJobTracker.class.getName()); // main of CJT
vargs.add(task.getJobID().toString()); // Pass job id.
vargs.add(task.getTaskID().toString()); // Pass attempt id.
vargs.add(coronaSessionInfo.getJobTrackerAddr().getHostName());
vargs.add(Integer.toString(coronaSessionInfo.getJobTrackerAddr().getPort()));
tracker.addToMemoryManager(task.getTaskID(), task.isMapTask(), conf);
// set memory limit using ulimit if feasible and necessary ...
String[] ulimitCmd = Shell.getUlimitMemoryCommand(getChildUlimit(conf));
List<String> setup = null;
if (ulimitCmd != null) {
setup = new ArrayList<String>();
for (String arg : ulimitCmd) {
setup.add(arg);
}
}
// Set up the redirection of the task's stdout and stderr streams
File stdout = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDOUT);
File stderr = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDERR);
stdout.getParentFile().mkdirs();
Map<String, String> env = new HashMap<String, String>();
StringBuffer ldLibraryPath = new StringBuffer();
ldLibraryPath.append(workDir.toString());
String oldLdLibraryPath = null;
oldLdLibraryPath = System.getenv("LD_LIBRARY_PATH");
if (oldLdLibraryPath != null) {
ldLibraryPath.append(sep);
ldLibraryPath.append(oldLdLibraryPath);
}
env.put("LD_LIBRARY_PATH", ldLibraryPath.toString());
LOG.info(
"Launching CJT "
+ taskid
+ " in working directory "
+ workDir
+ " Command line "
+ vargs);
jvmManager.launchJvm(
this,
jvmManager.constructJvmEnv(setup, vargs, stdout, stderr, logSize, workDir, env, conf));
synchronized (lock) {
while (!done) {
lock.wait();
}
}
} catch (IOException e) {
LOG.error("Error while launching CJT ", e);
} catch (InterruptedException e) {
LOG.warn("Error while launching CJT ", e);
}
}
/**
* @param taskid
* @param workDir
* @param classPaths
* @param logSize
* @return
* @throws IOException
*/
private Vector<String> getVMArgs(
TaskAttemptID taskid, File workDir, List<String> classPaths, long logSize)
throws IOException {
Vector<String> vargs = new Vector<String>(8);
File jvm = // use same jvm as parent
new File(new File(System.getProperty("java.home"), "bin"), "java");
vargs.add(jvm.toString());
// Add child (task) java-vm options.
//
// The following symbols if present in mapred.{map|reduce}.child.java.opts
// value are replaced:
// + @taskid@ is interpolated with value of TaskID.
// Other occurrences of @ will not be altered.
//
// Example with multiple arguments and substitutions, showing
// jvm GC logging, and start of a passwordless JVM JMX agent so can
// connect with jconsole and the likes to watch child memory, threads
// and get thread dumps.
//
// <property>
// <name>mapred.map.child.java.opts</name>
// <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@[email protected] \
// -Dcom.sun.management.jmxremote.authenticate=false \
// -Dcom.sun.management.jmxremote.ssl=false \
// </value>
// </property>
//
// <property>
// <name>mapred.reduce.child.java.opts</name>
// <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@[email protected] \
// -Dcom.sun.management.jmxremote.authenticate=false \
// -Dcom.sun.management.jmxremote.ssl=false \
// </value>
// </property>
//
String[] javaOptsSplit =
parseChildJavaOpts(getChildJavaOpts(conf, JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS), taskid);
// Add java.library.path; necessary for loading native libraries.
//
// 1. To support native-hadoop library i.e. libhadoop.so, we add the
// parent processes' java.library.path to the child.
// 2. We also add the 'cwd' of the task to it's java.library.path to help
// users distribute native libraries via the DistributedCache.
// 3. The user can also specify extra paths to be added to the
// java.library.path via mapred.{map|reduce}.child.java.opts.
//
String libraryPath = System.getProperty("java.library.path");
if (libraryPath == null) {
libraryPath = workDir.getAbsolutePath();
} else {
libraryPath += SYSTEM_PATH_SEPARATOR + workDir;
}
boolean hasUserLDPath = false;
for (int i = 0; i < javaOptsSplit.length; i++) {
if (javaOptsSplit[i].startsWith("-Djava.library.path=")) {
javaOptsSplit[i] += SYSTEM_PATH_SEPARATOR + libraryPath;
hasUserLDPath = true;
break;
}
}
if (!hasUserLDPath) {
vargs.add("-Djava.library.path=" + libraryPath);
}
for (int i = 0; i < javaOptsSplit.length; i++) {
vargs.add(javaOptsSplit[i]);
}
Path childTmpDir = createChildTmpDir(workDir, conf, false);
vargs.add("-Djava.io.tmpdir=" + childTmpDir);
// Add classpath.
vargs.add("-classpath");
String classPath = StringUtils.join(SYSTEM_PATH_SEPARATOR, classPaths);
vargs.add(classPath);
// Setup the log4j prop
setupLog4jProperties(vargs, taskid, logSize);
if (conf.getProfileEnabled()) {
if (conf.getProfileTaskRange(t.isMapTask()).isIncluded(t.getPartition())) {
File prof = TaskLog.getTaskLogFile(taskid, t.isTaskCleanupTask(), TaskLog.LogName.PROFILE);
vargs.add(String.format(conf.getProfileParams(), prof.toString()));
}
}
// Add main class and its arguments
vargs.add(Child.class.getName()); // main of Child
// pass umbilical address
InetSocketAddress address = tracker.getTaskTrackerReportAddress();
vargs.add(address.getAddress().getHostAddress());
vargs.add(Integer.toString(address.getPort()));
vargs.add(taskid.toString()); // pass task identifier
// pass task log location
vargs.add(TaskLog.getAttemptDir(taskid, t.isTaskCleanupTask()).toString());
return vargs;
}
@After
public void tearDown() throws IOException {
FileUtil.fullyDelete(TaskLog.getUserLogDir());
FileUtil.fullyDelete(new File(TEST_ROOT_DIR));
}
public class TestUserLogCleanup {
private static String jtid = "test";
private static long ONE_HOUR = 1000 * 60 * 60;
private Localizer localizer;
private UserLogManager userLogManager;
private UserLogCleaner userLogCleaner;
private TaskTracker tt;
private FakeClock myClock;
private JobID jobid1 = new JobID(jtid, 1);
private JobID jobid2 = new JobID(jtid, 2);
private JobID jobid3 = new JobID(jtid, 3);
private JobID jobid4 = new JobID(jtid, 4);
private File foo = new File(TaskLog.getUserLogDir(), "foo");
private File bar = new File(TaskLog.getUserLogDir(), "bar");
private static String TEST_ROOT_DIR = System.getProperty("test.build.data", "/tmp");
public TestUserLogCleanup() throws IOException, InterruptedException {
JobConf conf = new JobConf();
startTT(conf);
}
@After
public void tearDown() throws IOException {
FileUtil.fullyDelete(TaskLog.getUserLogDir());
FileUtil.fullyDelete(new File(TEST_ROOT_DIR));
}
private File localizeJob(JobID jobid) throws IOException {
String user = UserGroupInformation.getCurrentUser().getShortUserName();
new JobLocalizer(tt.getJobConf(), user, jobid.toString()).initializeJobLogDir();
File jobUserlog = TaskLog.getJobDir(jobid);
JobConf conf = new JobConf();
// localize job log directory
tt.saveLogDir(jobid, conf);
assertTrue(jobUserlog + " directory is not created.", jobUserlog.exists());
return jobUserlog;
}
private void jobFinished(JobID jobid, int logRetainHours) {
JobCompletedEvent jce = new JobCompletedEvent(jobid, myClock.getTime(), logRetainHours);
userLogManager.addLogEvent(jce);
}
private void startTT(JobConf conf) throws IOException, InterruptedException {
myClock = new FakeClock(); // clock is reset.
String localdirs = TEST_ROOT_DIR + "/userlogs/local/0," + TEST_ROOT_DIR + "/userlogs/local/1";
conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, localdirs);
tt = new TaskTracker();
tt.setConf(new JobConf(conf));
LocalDirAllocator localDirAllocator = new LocalDirAllocator("mapred.local.dir");
tt.setLocalDirAllocator(localDirAllocator);
LocalStorage localStorage = new LocalStorage(conf.getLocalDirs());
localStorage.checkLocalDirs();
tt.setLocalStorage(localStorage);
localizer =
new Localizer(FileSystem.get(conf), conf.getStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY));
tt.setLocalizer(localizer);
userLogManager = new UtilsForTests.InLineUserLogManager(conf);
TaskController taskController = userLogManager.getTaskController();
taskController.setup(localDirAllocator, localStorage);
tt.setTaskController(taskController);
userLogCleaner = userLogManager.getUserLogCleaner();
userLogCleaner.setClock(myClock);
tt.setUserLogManager(userLogManager);
userLogManager.clearOldUserLogs(conf);
}
private void ttReinited() throws IOException {
JobConf conf = new JobConf();
conf.setInt(JobContext.USER_LOG_RETAIN_HOURS, 3);
userLogManager.clearOldUserLogs(conf);
}
private void ttRestarted() throws IOException, InterruptedException {
JobConf conf = new JobConf();
conf.setInt(JobContext.USER_LOG_RETAIN_HOURS, 3);
startTT(conf);
}
/**
* Tests job user-log directory deletion.
*
* <p>Adds two jobs for log deletion. One with one hour retain hours, other with two retain hours.
* After an hour, TaskLogCleanupThread.processCompletedJobs() call, makes sure job with 1hr retain
* hours is removed and other is retained. After one more hour, job with 2hr retain hours is also
* removed.
*
* @throws IOException
*/
@Test
public void testJobLogCleanup() throws IOException {
File jobUserlog1 = localizeJob(jobid1);
File jobUserlog2 = localizeJob(jobid2);
// add job user log directory for deletion, with 2 hours for deletion
jobFinished(jobid1, 2);
// add the job for deletion with one hour as retain hours
jobFinished(jobid2, 1);
// remove old logs and see jobid1 is not removed and jobid2 is removed
myClock.advance(ONE_HOUR);
userLogCleaner.processCompletedJobs();
assertTrue(jobUserlog1 + " got deleted", jobUserlog1.exists());
assertFalse(jobUserlog2 + " still exists.", jobUserlog2.exists());
myClock.advance(ONE_HOUR);
// remove old logs and see jobid1 is removed now
userLogCleaner.processCompletedJobs();
assertFalse(jobUserlog1 + " still exists.", jobUserlog1.exists());
}
/**
* Tests user-log directory cleanup on a TT re-init with 3 hours as log retain hours for tracker.
*
* <p>Adds job1 deletion before the re-init with 2 hour retain hours. Adds job2 for which there
* are no tasks/killJobAction after the re-init. Adds job3 for which there is localizeJob followed
* by killJobAction with 3 hours as retain hours. Adds job4 for which there are some tasks after
* the re-init.
*
* @throws IOException
*/
@Test
public void testUserLogCleanup() throws IOException {
File jobUserlog1 = localizeJob(jobid1);
File jobUserlog2 = localizeJob(jobid2);
File jobUserlog3 = localizeJob(jobid3);
File jobUserlog4 = localizeJob(jobid4);
// create a some files/dirs in userlog
foo.mkdirs();
bar.createNewFile();
// add the jobid1 for deletion with retainhours = 2
jobFinished(jobid1, 2);
// time is now 1.
myClock.advance(ONE_HOUR);
// mimic TaskTracker reinit
// re-init the tt with 3 hours as user log retain hours.
// This re-init clears the user log directory
// job directories will be added with 3 hours as retain hours.
// i.e. They will be deleted at time 4.
ttReinited();
assertFalse(foo.exists());
assertFalse(bar.exists());
assertTrue(jobUserlog1.exists());
assertTrue(jobUserlog2.exists());
assertTrue(jobUserlog3.exists());
assertTrue(jobUserlog4.exists());
myClock.advance(ONE_HOUR);
// time is now 2.
userLogCleaner.processCompletedJobs();
assertFalse(jobUserlog1.exists());
assertTrue(jobUserlog2.exists());
assertTrue(jobUserlog3.exists());
assertTrue(jobUserlog4.exists());
// mimic localizeJob followed KillJobAction for jobid3
// add the job for deletion with retainhours = 3.
// jobid3 should be deleted at time 5.
jobUserlog3 = localizeJob(jobid3);
jobFinished(jobid3, 3);
// mimic localizeJob for jobid4
jobUserlog4 = localizeJob(jobid4);
// do cleanup
myClock.advance(2 * ONE_HOUR);
// time is now 4.
userLogCleaner.processCompletedJobs();
// jobid2 will be deleted
assertFalse(jobUserlog1.exists());
assertFalse(jobUserlog2.exists());
assertTrue(jobUserlog3.exists());
assertTrue(jobUserlog4.exists());
myClock.advance(ONE_HOUR);
// time is now 5.
// do cleanup again
userLogCleaner.processCompletedJobs();
// jobid3 will be deleted
assertFalse(jobUserlog1.exists());
assertFalse(jobUserlog2.exists());
assertFalse(jobUserlog3.exists());
assertTrue(jobUserlog4.exists());
}
/**
* Tests user-log directory cleanup on a TT restart.
*
* <p>Adds job1 deletion before the restart with 2 hour retain hours. Adds job2 for which there
* are no tasks/killJobAction after the restart. Adds job3 for which there is localizeJob followed
* by killJobAction after the restart with 3 hours retain hours. Adds job4 for which there are
* some tasks after the restart.
*
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testUserLogCleanupAfterRestart() throws IOException, InterruptedException {
File jobUserlog1 = localizeJob(jobid1);
File jobUserlog2 = localizeJob(jobid2);
File jobUserlog3 = localizeJob(jobid3);
File jobUserlog4 = localizeJob(jobid4);
// create a some files/dirs in userlog
foo.mkdirs();
bar.createNewFile();
// add the jobid1 for deletion with retain hours = 2
jobFinished(jobid1, 2);
// time is now 1.
myClock.advance(ONE_HOUR);
// Mimic the TaskTracker restart
// Restart the tt with 3 hours as user log retain hours.
// This restart clears the user log directory
// job directories will be added with 3 hours as retain hours.
// i.e. They will be deleted at time 3 as clock will reset after the restart
ttRestarted();
assertFalse(foo.exists());
assertFalse(bar.exists());
assertTrue(jobUserlog1.exists());
assertTrue(jobUserlog2.exists());
assertTrue(jobUserlog3.exists());
assertTrue(jobUserlog4.exists());
myClock.advance(ONE_HOUR);
// time is now 1.
userLogCleaner.processCompletedJobs();
assertTrue(jobUserlog1.exists());
assertTrue(jobUserlog2.exists());
assertTrue(jobUserlog3.exists());
assertTrue(jobUserlog4.exists());
// mimic localizeJob followed KillJobAction for jobid3
// add the job for deletion with retainhours = 3.
// jobid3 should be deleted at time 4.
jobUserlog3 = localizeJob(jobid3);
jobFinished(jobid3, 3);
// mimic localizeJob for jobid4
jobUserlog4 = localizeJob(jobid4);
// do cleanup
myClock.advance(2 * ONE_HOUR);
// time is now 3.
userLogCleaner.processCompletedJobs();
// jobid1 and jobid2 will be deleted
assertFalse(jobUserlog1.exists());
assertFalse(jobUserlog2.exists());
assertTrue(jobUserlog3.exists());
assertTrue(jobUserlog4.exists());
myClock.advance(ONE_HOUR);
// time is now 4.
// do cleanup again
userLogCleaner.processCompletedJobs();
// jobid3 will be deleted
assertFalse(jobUserlog1.exists());
assertFalse(jobUserlog2.exists());
assertFalse(jobUserlog3.exists());
assertTrue(jobUserlog4.exists());
}
}
@Override
public final void run() {
try {
// before preparing the job localize
// all the archives
TaskAttemptID taskid = t.getTaskID();
LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir");
File jobCacheDir = null;
if (conf.getJar() != null) {
jobCacheDir = new File(new Path(conf.getJar()).getParent().toString());
}
File workDir =
new File(
lDirAlloc
.getLocalPathToRead(
TaskTracker.getJobCacheSubdir()
+ Path.SEPARATOR
+ t.getJobID()
+ Path.SEPARATOR
+ t.getTaskID()
+ Path.SEPARATOR
+ MRConstants.WORKDIR,
conf)
.toString());
URI[] archives = DistributedCache.getCacheArchives(conf);
URI[] files = DistributedCache.getCacheFiles(conf);
FileStatus fileStatus;
FileSystem fileSystem;
Path localPath;
String baseDir;
if ((archives != null) || (files != null)) {
if (archives != null) {
String[] archivesTimestamps = DistributedCache.getArchiveTimestamps(conf);
Path[] p = new Path[archives.length];
for (int i = 0; i < archives.length; i++) {
fileSystem = FileSystem.get(archives[i], conf);
fileStatus = fileSystem.getFileStatus(new Path(archives[i].getPath()));
String cacheId = DistributedCache.makeRelative(archives[i], conf);
String cachePath = TaskTracker.getCacheSubdir() + Path.SEPARATOR + cacheId;
if (lDirAlloc.ifExists(cachePath, conf)) {
localPath = lDirAlloc.getLocalPathToRead(cachePath, conf);
} else {
localPath = lDirAlloc.getLocalPathForWrite(cachePath, fileStatus.getLen(), conf);
}
baseDir = localPath.toString().replace(cacheId, "");
p[i] =
DistributedCache.getLocalCache(
archives[i],
conf,
new Path(baseDir),
fileStatus,
true,
Long.parseLong(archivesTimestamps[i]),
new Path(workDir.getAbsolutePath()),
false);
}
DistributedCache.setLocalArchives(conf, stringifyPathArray(p));
}
if ((files != null)) {
String[] fileTimestamps = DistributedCache.getFileTimestamps(conf);
Path[] p = new Path[files.length];
for (int i = 0; i < files.length; i++) {
fileSystem = FileSystem.get(files[i], conf);
fileStatus = fileSystem.getFileStatus(new Path(files[i].getPath()));
String cacheId = DistributedCache.makeRelative(files[i], conf);
String cachePath = TaskTracker.getCacheSubdir() + Path.SEPARATOR + cacheId;
if (lDirAlloc.ifExists(cachePath, conf)) {
localPath = lDirAlloc.getLocalPathToRead(cachePath, conf);
} else {
localPath = lDirAlloc.getLocalPathForWrite(cachePath, fileStatus.getLen(), conf);
}
baseDir = localPath.toString().replace(cacheId, "");
p[i] =
DistributedCache.getLocalCache(
files[i],
conf,
new Path(baseDir),
fileStatus,
false,
Long.parseLong(fileTimestamps[i]),
new Path(workDir.getAbsolutePath()),
false);
}
DistributedCache.setLocalFiles(conf, stringifyPathArray(p));
}
Path localTaskFile = new Path(t.getJobFile());
FileSystem localFs = FileSystem.getLocal(conf);
localFs.delete(localTaskFile, true);
OutputStream out = localFs.create(localTaskFile);
try {
conf.writeXml(out);
} finally {
out.close();
}
}
if (!prepare()) {
return;
}
String sep = System.getProperty("path.separator");
StringBuffer classPath = new StringBuffer();
// start with same classpath as parent process
classPath.append(System.getProperty("java.class.path"));
classPath.append(sep);
if (!workDir.mkdirs()) {
if (!workDir.isDirectory()) {
LOG.fatal("Mkdirs failed to create " + workDir.toString());
}
}
String jar = conf.getJar();
if (jar != null) {
// if jar exists, it into workDir
File[] libs = new File(jobCacheDir, "lib").listFiles();
if (libs != null) {
for (int i = 0; i < libs.length; i++) {
classPath.append(sep); // add libs from jar to classpath
classPath.append(libs[i]);
}
}
classPath.append(sep);
classPath.append(new File(jobCacheDir, "classes"));
classPath.append(sep);
classPath.append(jobCacheDir);
}
// include the user specified classpath
// archive paths
Path[] archiveClasspaths = DistributedCache.getArchiveClassPaths(conf);
if (archiveClasspaths != null && archives != null) {
Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
if (localArchives != null) {
for (int i = 0; i < archives.length; i++) {
for (int j = 0; j < archiveClasspaths.length; j++) {
if (archives[i].getPath().equals(archiveClasspaths[j].toString())) {
classPath.append(sep);
classPath.append(localArchives[i].toString());
}
}
}
}
}
// file paths
Path[] fileClasspaths = DistributedCache.getFileClassPaths(conf);
if (fileClasspaths != null && files != null) {
Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
if (localFiles != null) {
for (int i = 0; i < files.length; i++) {
for (int j = 0; j < fileClasspaths.length; j++) {
if (files[i].getPath().equals(fileClasspaths[j].toString())) {
classPath.append(sep);
classPath.append(localFiles[i].toString());
}
}
}
}
}
classPath.append(sep);
classPath.append(workDir);
// Build exec child jmv args.
Vector<String> vargs = new Vector<String>(8);
File jvm = // use same jvm as parent
new File(new File(System.getProperty("java.home"), "bin"), "java");
vargs.add(jvm.toString());
// Add child (task) java-vm options.
//
// The following symbols if present in mapred.child.java.opts value are
// replaced:
// + @taskid@ is interpolated with value of TaskID.
// Other occurrences of @ will not be altered.
//
// Example with multiple arguments and substitutions, showing
// jvm GC logging, and start of a passwordless JVM JMX agent so can
// connect with jconsole and the likes to watch child memory, threads
// and get thread dumps.
//
// <property>
// <name>mapred.child.java.opts</name>
// <value>-verbose:gc -Xloggc:/tmp/@[email protected] \
// -Dcom.sun.management.jmxremote.authenticate=false \
// -Dcom.sun.management.jmxremote.ssl=false \
// </value>
// </property>
//
String javaOpts = conf.get("mapred.child.java.opts", "-Xmx200m");
javaOpts = javaOpts.replace("@taskid@", taskid.toString());
String[] javaOptsSplit = javaOpts.split(" ");
// Add java.library.path; necessary for loading native libraries.
//
// 1. To support native-hadoop library i.e. libhadoop.so, we add the
// parent processes' java.library.path to the child.
// 2. We also add the 'cwd' of the task to it's java.library.path to help
// users distribute native libraries via the DistributedCache.
// 3. The user can also specify extra paths to be added to the
// java.library.path via mapred.child.java.opts.
//
String libraryPath = System.getProperty("java.library.path");
if (libraryPath == null) {
libraryPath = workDir.getAbsolutePath();
} else {
libraryPath += sep + workDir;
}
boolean hasUserLDPath = false;
for (int i = 0; i < javaOptsSplit.length; i++) {
if (javaOptsSplit[i].startsWith("-Djava.library.path=")) {
javaOptsSplit[i] += sep + libraryPath;
hasUserLDPath = true;
break;
}
}
if (!hasUserLDPath) {
vargs.add("-Djava.library.path=" + libraryPath);
}
for (int i = 0; i < javaOptsSplit.length; i++) {
vargs.add(javaOptsSplit[i]);
}
// add java.io.tmpdir given by mapred.child.tmp
String tmp = conf.get("mapred.child.tmp", "./tmp");
Path tmpDir = new Path(tmp);
// if temp directory path is not absolute
// prepend it with workDir.
if (!tmpDir.isAbsolute()) {
tmpDir = new Path(workDir.toString(), tmp);
}
FileSystem localFs = FileSystem.getLocal(conf);
if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) {
throw new IOException("Mkdirs failed to create " + tmpDir.toString());
}
vargs.add("-Djava.io.tmpdir=" + tmpDir.toString());
// Add classpath.
vargs.add("-classpath");
vargs.add(classPath.toString());
// Setup the log4j prop
long logSize = TaskLog.getTaskLogLength(conf);
vargs.add(
"-Dhadoop.log.dir=" + new File(System.getProperty("hadoop.log.dir")).getAbsolutePath());
vargs.add("-Dhadoop.root.logger=INFO,TLA");
vargs.add("-Dhadoop.tasklog.taskid=" + taskid);
vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize);
if (conf.getProfileEnabled()) {
if (conf.getProfileTaskRange(t.isMapTask()).isIncluded(t.getPartition())) {
File prof = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.PROFILE);
vargs.add(String.format(conf.getProfileParams(), prof.toString()));
}
}
// Add main class and its arguments
vargs.add(Child.class.getName()); // main of Child
// pass umbilical address
InetSocketAddress address = tracker.getTaskTrackerReportAddress();
vargs.add(address.getAddress().getHostAddress());
vargs.add(Integer.toString(address.getPort()));
vargs.add(taskid.toString()); // pass task identifier
String pidFile = null;
if (tracker.isTaskMemoryManagerEnabled()) {
pidFile =
lDirAlloc
.getLocalPathForWrite(
(TaskTracker.getPidFilesSubdir() + Path.SEPARATOR + taskid), this.conf)
.toString();
}
// set memory limit using ulimit if feasible and necessary ...
String[] ulimitCmd = Shell.getUlimitMemoryCommand(conf);
List<String> setup = null;
if (ulimitCmd != null) {
setup = new ArrayList<String>();
for (String arg : ulimitCmd) {
setup.add(arg);
}
}
// Set up the redirection of the task's stdout and stderr streams
File stdout = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDOUT);
File stderr = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDERR);
stdout.getParentFile().mkdirs();
tracker.getTaskTrackerInstrumentation().reportTaskLaunch(taskid, stdout, stderr);
Map<String, String> env = new HashMap<String, String>();
StringBuffer ldLibraryPath = new StringBuffer();
ldLibraryPath.append(workDir.toString());
String oldLdLibraryPath = null;
oldLdLibraryPath = System.getenv("LD_LIBRARY_PATH");
if (oldLdLibraryPath != null) {
ldLibraryPath.append(sep);
ldLibraryPath.append(oldLdLibraryPath);
}
env.put("LD_LIBRARY_PATH", ldLibraryPath.toString());
jvmManager.launchJvm(
this,
jvmManager.constructJvmEnv(
setup, vargs, stdout, stderr, logSize, workDir, env, pidFile, conf));
synchronized (lock) {
while (!done) {
lock.wait();
}
}
tracker.getTaskTrackerInstrumentation().reportTaskEnd(t.getTaskID());
if (exitCodeSet) {
if (!killed && exitCode != 0) {
if (exitCode == 65) {
tracker.getTaskTrackerInstrumentation().taskFailedPing(t.getTaskID());
}
throw new IOException("Task process exit with nonzero status of " + exitCode + ".");
}
}
} catch (FSError e) {
LOG.fatal("FSError", e);
try {
tracker.fsError(t.getTaskID(), e.getMessage());
} catch (IOException ie) {
LOG.fatal(t.getTaskID() + " reporting FSError", ie);
}
} catch (Throwable throwable) {
LOG.warn(t.getTaskID() + " Child Error", throwable);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
throwable.printStackTrace(new PrintStream(baos));
try {
tracker.reportDiagnosticInfo(t.getTaskID(), baos.toString());
} catch (IOException e) {
LOG.warn(t.getTaskID() + " Reporting Diagnostics", e);
}
} finally {
try {
URI[] archives = DistributedCache.getCacheArchives(conf);
URI[] files = DistributedCache.getCacheFiles(conf);
if (archives != null) {
for (int i = 0; i < archives.length; i++) {
DistributedCache.releaseCache(archives[i], conf);
}
}
if (files != null) {
for (int i = 0; i < files.length; i++) {
DistributedCache.releaseCache(files[i], conf);
}
}
} catch (IOException ie) {
LOG.warn("Error releasing caches : Cache files might not have been cleaned up");
}
tracker.reportTaskFinished(t.getTaskID(), false);
if (t.isMapTask()) {
tracker.addFreeMapSlot();
} else {
tracker.addFreeReduceSlot();
}
}
}
@Override
public int launchTask(
String user,
String jobId,
String attemptId,
List<String> setup,
List<String> jvmArguments,
File currentWorkDirectory,
String stdout,
String stderr)
throws IOException {
ShellCommandExecutor shExec = null;
try {
FileSystem rawFs = FileSystem.getLocal(getConf()).getRaw();
long logSize = 0; // TODO, Ref BUG:2854624
// get the JVM command line.
String cmdLine =
TaskLog.buildCommandLine(
setup, jvmArguments, new File(stdout), new File(stderr), logSize, true);
// write the command to a file in the
// task specific cache directory
Path p =
new Path(
allocator.getLocalPathForWrite(
TaskTracker.getPrivateDirTaskScriptLocation(user, jobId, attemptId), getConf()),
COMMAND_FILE);
String commandFile = writeCommand(cmdLine, rawFs, p);
String[] command =
new String[] {
taskControllerExe,
user,
localStorage.getDirsString(),
Integer.toString(Commands.LAUNCH_TASK_JVM.getValue()),
jobId,
attemptId,
currentWorkDirectory.toString(),
commandFile
};
shExec = new ShellCommandExecutor(command);
if (LOG.isDebugEnabled()) {
LOG.debug("launchTask: " + Arrays.toString(command));
}
shExec.execute();
} catch (Exception e) {
if (shExec == null) {
return -1;
}
int exitCode = shExec.getExitCode();
LOG.warn("Exit code from task is : " + exitCode);
// 143 (SIGTERM) and 137 (SIGKILL) exit codes means the task was
// terminated/killed forcefully. In all other cases, log the
// task-controller output
if (exitCode != 143 && exitCode != 137) {
LOG.warn(
"Exception thrown while launching task JVM : " + StringUtils.stringifyException(e));
LOG.info("Output from LinuxTaskController's launchTaskJVM follows:");
logOutput(shExec.getOutput());
}
return exitCode;
}
if (LOG.isDebugEnabled()) {
LOG.debug("Output from LinuxTaskController's launchTask follows:");
logOutput(shExec.getOutput());
}
return 0;
}