@SuppressWarnings("deprecation") public CoronaJobTrackerRunner( TaskTracker.TaskInProgress tip, Task task, TaskTracker tracker, JobConf ttConf, CoronaSessionInfo info, String originalPath, String releasePath) throws IOException { super(tip, task, tracker, ttConf); this.coronaSessionInfo = info; this.originalPath = originalPath; this.releasePath = releasePath; LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir"); workDir = new File( lDirAlloc .getLocalPathForWrite( TaskTracker.getLocalTaskDir( task.getJobID().toString(), task.getTaskID().toString(), task.isTaskCleanupTask()) + Path.SEPARATOR + MRConstants.WORKDIR, conf) .toString()); if (!workDir.mkdirs()) { if (!workDir.isDirectory()) { throw new IOException("Mkdirs failed to create " + workDir.toString()); } } localizeTaskConfiguration(tracker, ttConf, workDir.toString(), task, task.getJobID()); }
private void setupLog4jProperties(Vector<String> vargs, TaskAttemptID taskid, long logSize) { vargs.add( "-Dhadoop.log.dir=" + new File(System.getProperty("hadoop.log.dir")).getAbsolutePath()); vargs.add("-Dhadoop.root.logger=INFO,TLA"); vargs.add("-Dhadoop.tasklog.taskid=" + taskid); vargs.add("-Dhadoop.tasklog.iscleanup=" + t.isTaskCleanupTask()); vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize); }
private void setupLog4jProperties(Vector<String> vargs, TaskAttemptID taskid, long logSize) { vargs.add("-Dlog4j.configuration=task-log4j.properties"); vargs.add( "-Dhadoop.log.dir=" + new File(System.getProperty("hadoop.log.dir")).getAbsolutePath()); vargs.add("-Dhadoop.root.logger=" + getLogLevel(conf).toString() + ",TLA"); vargs.add("-D" + TaskLogAppender.TASKID_PROPERTY + "=" + taskid); vargs.add("-D" + TaskLogAppender.ISCLEANUP_PROPERTY + "=" + t.isTaskCleanupTask()); vargs.add("-D" + TaskLogAppender.LOGSIZE_PROPERTY + "=" + logSize); }
/** * Write the child's configuration to the disk and set it in configuration so that the child can * pick it up from there. * * @param lDirAlloc * @throws IOException */ void setupChildTaskConfiguration(LocalDirAllocator lDirAlloc) throws IOException { Path localTaskFile = lDirAlloc.getLocalPathForWrite( TaskTracker.getTaskConfFile( t.getUser(), t.getJobID().toString(), t.getTaskID().toString(), t.isTaskCleanupTask()), conf); // write the child's task configuration file to the local disk JobLocalizer.writeLocalJobFile(localTaskFile, conf); // Set the final job file in the task. The child needs to know the correct // path to job.xml. So set this path accordingly. t.setJobFile(localTaskFile.toString()); }
/** * Prepare the mapred.local.dir for the child. The child is sand-boxed now. Whenever it uses * LocalDirAllocator from now on inside the child, it will only see files inside the * attempt-directory. This is done in the Child's process space. */ static void setupChildMapredLocalDirs(Task t, JobConf conf) { String[] localDirs = conf.getTrimmedStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY); String jobId = t.getJobID().toString(); String taskId = t.getTaskID().toString(); boolean isCleanup = t.isTaskCleanupTask(); String user = t.getUser(); StringBuffer childMapredLocalDir = new StringBuffer( localDirs[0] + Path.SEPARATOR + TaskTracker.getLocalTaskDir(user, jobId, taskId, isCleanup)); for (int i = 1; i < localDirs.length; i++) { childMapredLocalDir.append( "," + localDirs[i] + Path.SEPARATOR + TaskTracker.getLocalTaskDir(user, jobId, taskId, isCleanup)); } LOG.debug("mapred.local.dir for child : " + childMapredLocalDir); conf.set("mapred.local.dir", childMapredLocalDir.toString()); }
private String getVMEnvironment( String errorInfo, File workDir, JobConf conf, Map<String, String> env, TaskAttemptID taskid, long logSize) throws Throwable { StringBuffer ldLibraryPath = new StringBuffer(); ldLibraryPath.append(workDir.toString()); String oldLdLibraryPath = null; oldLdLibraryPath = System.getenv("LD_LIBRARY_PATH"); if (oldLdLibraryPath != null) { ldLibraryPath.append(SYSTEM_PATH_SEPARATOR); ldLibraryPath.append(oldLdLibraryPath); } env.put("LD_LIBRARY_PATH", ldLibraryPath.toString()); env.put(HADOOP_WORK_DIR, workDir.toString()); // put jobTokenFile name into env String jobTokenFile = conf.get(TokenCache.JOB_TOKENS_FILENAME); LOG.debug("putting jobToken file name into environment " + jobTokenFile); env.put(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, jobTokenFile); // for the child of task jvm, set hadoop.root.logger env.put("HADOOP_ROOT_LOGGER", "INFO,TLA"); String hadoopClientOpts = System.getenv("HADOOP_CLIENT_OPTS"); if (hadoopClientOpts == null) { hadoopClientOpts = ""; } else { hadoopClientOpts = hadoopClientOpts + " "; } hadoopClientOpts = hadoopClientOpts + "-Dhadoop.tasklog.taskid=" + taskid + " -Dhadoop.tasklog.iscleanup=" + t.isTaskCleanupTask() + " -Dhadoop.tasklog.totalLogFileSize=" + logSize; // following line is a backport from jira MAPREDUCE-1286 env.put("HADOOP_CLIENT_OPTS", hadoopClientOpts); // add the env variables passed by the user String mapredChildEnv = getChildEnv(conf); if (mapredChildEnv != null && mapredChildEnv.length() > 0) { String childEnvs[] = mapredChildEnv.split(","); for (String cEnv : childEnvs) { try { String[] parts = cEnv.split("="); // split on '=' String value = env.get(parts[0]); if (value != null) { // replace $env with the child's env constructed by tt's // example LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp value = parts[1].replace("$" + parts[0], value); } else { // this key is not configured by the tt for the child .. get it // from the tt's env // example PATH=$PATH:/tmp value = System.getenv(parts[0]); if (value != null) { // the env key is present in the tt's env value = parts[1].replace("$" + parts[0], value); } else { // the env key is note present anywhere .. simply set it // example X=$X:/tmp or X=/tmp value = parts[1].replace("$" + parts[0], ""); } } env.put(parts[0], value); } catch (Throwable t) { // set the error msg errorInfo = "Invalid User environment settings : " + mapredChildEnv + ". Failed to parse user-passed environment param." + " Expecting : env1=value1,env2=value2..."; LOG.warn(errorInfo); throw t; } } } return errorInfo; }
/** * @param taskid * @param workDir * @param classPaths * @param logSize * @return * @throws IOException */ private Vector<String> getVMArgs( TaskAttemptID taskid, File workDir, List<String> classPaths, long logSize) throws IOException { Vector<String> vargs = new Vector<String>(8); File jvm = // use same jvm as parent new File(new File(System.getProperty("java.home"), "bin"), "java"); vargs.add(jvm.toString()); // Add child (task) java-vm options. // // The following symbols if present in mapred.{map|reduce}.child.java.opts // value are replaced: // + @taskid@ is interpolated with value of TaskID. // Other occurrences of @ will not be altered. // // Example with multiple arguments and substitutions, showing // jvm GC logging, and start of a passwordless JVM JMX agent so can // connect with jconsole and the likes to watch child memory, threads // and get thread dumps. // // <property> // <name>mapred.map.child.java.opts</name> // <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@[email protected] \ // -Dcom.sun.management.jmxremote.authenticate=false \ // -Dcom.sun.management.jmxremote.ssl=false \ // </value> // </property> // // <property> // <name>mapred.reduce.child.java.opts</name> // <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@[email protected] \ // -Dcom.sun.management.jmxremote.authenticate=false \ // -Dcom.sun.management.jmxremote.ssl=false \ // </value> // </property> // String[] javaOptsSplit = parseChildJavaOpts(getChildJavaOpts(conf, JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS), taskid); // Add java.library.path; necessary for loading native libraries. // // 1. To support native-hadoop library i.e. libhadoop.so, we add the // parent processes' java.library.path to the child. // 2. We also add the 'cwd' of the task to it's java.library.path to help // users distribute native libraries via the DistributedCache. // 3. The user can also specify extra paths to be added to the // java.library.path via mapred.{map|reduce}.child.java.opts. // String libraryPath = System.getProperty("java.library.path"); if (libraryPath == null) { libraryPath = workDir.getAbsolutePath(); } else { libraryPath += SYSTEM_PATH_SEPARATOR + workDir; } boolean hasUserLDPath = false; for (int i = 0; i < javaOptsSplit.length; i++) { if (javaOptsSplit[i].startsWith("-Djava.library.path=")) { javaOptsSplit[i] += SYSTEM_PATH_SEPARATOR + libraryPath; hasUserLDPath = true; break; } } if (!hasUserLDPath) { vargs.add("-Djava.library.path=" + libraryPath); } for (int i = 0; i < javaOptsSplit.length; i++) { vargs.add(javaOptsSplit[i]); } Path childTmpDir = createChildTmpDir(workDir, conf, false); vargs.add("-Djava.io.tmpdir=" + childTmpDir); // Add classpath. vargs.add("-classpath"); String classPath = StringUtils.join(SYSTEM_PATH_SEPARATOR, classPaths); vargs.add(classPath); // Setup the log4j prop setupLog4jProperties(vargs, taskid, logSize); if (conf.getProfileEnabled()) { if (conf.getProfileTaskRange(t.isMapTask()).isIncluded(t.getPartition())) { File prof = TaskLog.getTaskLogFile(taskid, t.isTaskCleanupTask(), TaskLog.LogName.PROFILE); vargs.add(String.format(conf.getProfileParams(), prof.toString())); } } // Add main class and its arguments vargs.add(Child.class.getName()); // main of Child // pass umbilical address InetSocketAddress address = tracker.getTaskTrackerReportAddress(); vargs.add(address.getAddress().getHostAddress()); vargs.add(Integer.toString(address.getPort())); vargs.add(taskid.toString()); // pass task identifier // pass task log location vargs.add(TaskLog.getAttemptDir(taskid, t.isTaskCleanupTask()).toString()); return vargs; }
@Override public final void run() { String errorInfo = "Child Error"; try { // before preparing the job localize // all the archives TaskAttemptID taskid = t.getTaskID(); final LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir"); // simply get the location of the workDir and pass it to the child. The // child will do the actual dir creation final File workDir = new File( new Path( localdirs[rand.nextInt(localdirs.length)], TaskTracker.getTaskWorkDir( t.getUser(), taskid.getJobID().toString(), taskid.toString(), t.isTaskCleanupTask())) .toString()); // Set up the child task's configuration. After this call, no localization // of files should happen in the TaskTracker's process space. Any changes to // the conf object after this will NOT be reflected to the child. // setupChildTaskConfiguration(lDirAlloc); if (!prepare()) { return; } // Accumulates class paths for child. List<String> classPaths = getClassPaths(conf, workDir, taskDistributedCacheManager); long logSize = TaskLog.getTaskLogLength(conf); // Build exec child JVM args. Vector<String> vargs = getVMArgs(taskid, workDir, classPaths, logSize); tracker.addToMemoryManager(t.getTaskID(), t.isMapTask(), conf); // set memory limit using ulimit if feasible and necessary ... String setup = getVMSetupCmd(); // Set up the redirection of the task's stdout and stderr streams File[] logFiles = prepareLogFiles(taskid, t.isTaskCleanupTask()); File stdout = logFiles[0]; File stderr = logFiles[1]; tracker.getTaskTrackerInstrumentation().reportTaskLaunch(taskid, stdout, stderr); Map<String, String> env = new HashMap<String, String>(); errorInfo = getVMEnvironment(errorInfo, workDir, conf, env, taskid, logSize); // flatten the env as a set of export commands List<String> setupCmds = new ArrayList<String>(); appendEnvExports(setupCmds, env); setupCmds.add(setup); launchJvmAndWait(setupCmds, vargs, stdout, stderr, logSize, workDir); tracker.getTaskTrackerInstrumentation().reportTaskEnd(t.getTaskID()); if (exitCodeSet) { if (!killed && exitCode != 0) { if (exitCode == 65) { tracker.getTaskTrackerInstrumentation().taskFailedPing(t.getTaskID()); } throw new IOException("Task process exit with nonzero status of " + exitCode + "."); } } } catch (FSError e) { LOG.fatal("FSError", e); try { tracker.internalFsError(t.getTaskID(), e.getMessage()); } catch (IOException ie) { LOG.fatal(t.getTaskID() + " reporting FSError", ie); } } catch (Throwable throwable) { LOG.warn(t.getTaskID() + " : " + errorInfo, throwable); Throwable causeThrowable = new Throwable(errorInfo, throwable); ByteArrayOutputStream baos = new ByteArrayOutputStream(); causeThrowable.printStackTrace(new PrintStream(baos)); try { tracker.internalReportDiagnosticInfo(t.getTaskID(), baos.toString()); } catch (IOException e) { LOG.warn(t.getTaskID() + " Reporting Diagnostics", e); } } finally { // It is safe to call TaskTracker.TaskInProgress.reportTaskFinished with // *false* since the task has either // a) SUCCEEDED - which means commit has been done // b) FAILED - which means we do not need to commit tip.reportTaskFinished(false); } }
@Override public final void run() { String errorInfo = "Child Error"; try { // before preparing the job localize // all the archives TaskAttemptID taskid = t.getTaskID(); final LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir"); final File workDir = formWorkDir(lDirAlloc, taskid, t.isTaskCleanupTask(), conf); // We don't create any symlinks yet, so presence/absence of workDir // actually on the file system doesn't matter. tip.getUGI() .doAs( new PrivilegedExceptionAction<Void>() { public Void run() throws IOException { taskDistributedCacheManager = tracker .getTrackerDistributedCacheManager() .newTaskDistributedCacheManager(conf); taskDistributedCacheManager.setup( lDirAlloc, workDir, TaskTracker.getPrivateDistributedCacheDir(conf.getUser()), TaskTracker.getPublicDistributedCacheDir()); return null; } }); // Set up the child task's configuration. After this call, no localization // of files should happen in the TaskTracker's process space. Any changes to // the conf object after this will NOT be reflected to the child. setupChildTaskConfiguration(lDirAlloc); if (!prepare()) { return; } // Accumulates class paths for child. List<String> classPaths = getClassPaths(conf, workDir, taskDistributedCacheManager); long logSize = TaskLog.getTaskLogLength(conf); // Build exec child JVM args. Vector<String> vargs = getVMArgs(taskid, workDir, classPaths, logSize); tracker.addToMemoryManager(t.getTaskID(), t.isMapTask(), conf); // set memory limit using ulimit if feasible and necessary ... List<String> setup = getVMSetupCmd(); // Set up the redirection of the task's stdout and stderr streams File[] logFiles = prepareLogFiles(taskid, t.isTaskCleanupTask()); File stdout = logFiles[0]; File stderr = logFiles[1]; List<TaskTrackerInstrumentation> ttInstrumentations = tracker.getTaskTrackerInstrumentations(); for (TaskTrackerInstrumentation inst : ttInstrumentations) { inst.reportTaskLaunch(taskid, stdout, stderr); } Map<String, String> env = new HashMap<String, String>(); errorInfo = getVMEnvironment(errorInfo, workDir, conf, env, taskid, logSize); launchJvmAndWait(setup, vargs, stdout, stderr, logSize, workDir, env); for (TaskTrackerInstrumentation inst : ttInstrumentations) { inst.reportTaskEnd(t.getTaskID()); } if (exitCodeSet) { if (!killed && exitCode != 0) { if (exitCode == 65) { for (TaskTrackerInstrumentation inst : ttInstrumentations) { inst.taskFailedPing(t.getTaskID()); } } throw new IOException("Task process exit with nonzero status of " + exitCode + "."); } } } catch (FSError e) { LOG.fatal("FSError", e); try { tracker.internalFsError(t.getTaskID(), e.getMessage()); } catch (IOException ie) { LOG.fatal(t.getTaskID() + " reporting FSError", ie); } } catch (Throwable throwable) { LOG.warn(t.getTaskID() + " : " + errorInfo, throwable); Throwable causeThrowable = new Throwable(errorInfo, throwable); ByteArrayOutputStream baos = new ByteArrayOutputStream(); causeThrowable.printStackTrace(new PrintStream(baos)); try { tracker.internalReportDiagnosticInfo(t.getTaskID(), baos.toString()); } catch (IOException e) { LOG.warn(t.getTaskID() + " Reporting Diagnostics", e); } } finally { try { if (taskDistributedCacheManager != null) { taskDistributedCacheManager.release(); } } catch (IOException ie) { LOG.warn("Error releasing caches : Cache files might not have been cleaned up"); } // It is safe to call TaskTracker.TaskInProgress.reportTaskFinished with // *false* since the task has either // a) SUCCEEDED - which means commit has been done // b) FAILED - which means we do not need to commit tip.reportTaskFinished(false); } }