@Override public void setup(LocalDirAllocator allocator, LocalStorage localStorage) throws IOException { // Check the permissions of the task-controller binary by running // it plainly. If permissions are correct, it returns an error // code 1, else it returns 24 or something else if some other bugs // are also present. String[] taskControllerCmd = new String[] {taskControllerExe}; ShellCommandExecutor shExec = new ShellCommandExecutor(taskControllerCmd); try { shExec.execute(); } catch (ExitCodeException e) { int exitCode = shExec.getExitCode(); if (exitCode != 1) { LOG.warn("Exit code from checking binary permissions is : " + exitCode); logOutput(shExec.getOutput()); throw new IOException( "Task controller setup failed because of invalid" + "permissions/ownership with exit code " + exitCode, e); } } this.allocator = allocator; this.localStorage = localStorage; }
/** * Given a Tar File as input it will untar the file in a the untar directory passed as the second * parameter * * <p>This utility will untar ".tar" files and ".tar.gz","tgz" files. * * @param inFile The tar file as input. * @param untarDir The untar directory where to untar the tar file. * @throws IOException */ public static void unTar(File inFile, File untarDir) throws IOException { if (!untarDir.mkdirs()) { if (!untarDir.isDirectory()) { throw new IOException("Mkdirs failed to create " + untarDir); } } StringBuffer untarCommand = new StringBuffer(); boolean gzipped = inFile.toString().endsWith("gz"); if (gzipped) { untarCommand.append(" gzip -dc '"); untarCommand.append(FileUtil.makeShellPath(inFile)); untarCommand.append("' | ("); } untarCommand.append("cd '"); untarCommand.append(FileUtil.makeShellPath(untarDir)); untarCommand.append("' ; "); untarCommand.append("tar -xf "); if (gzipped) { untarCommand.append(" -)"); } else { untarCommand.append(FileUtil.makeShellPath(inFile)); } String[] shellCmd = {"bash", "-c", untarCommand.toString()}; ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd); shexec.execute(); int exitcode = shexec.getExitCode(); if (exitcode != 0) { throw new IOException( "Error untarring file " + inFile + ". Tar process exited with exit code " + exitcode); } }
@Override public void signalTask(String user, int taskPid, Signal signal) throws IOException { String[] command = new String[] { taskControllerExe, user, localStorage.getDirsString(), Integer.toString(Commands.SIGNAL_TASK.getValue()), Integer.toString(taskPid), Integer.toString(signal.getValue()) }; ShellCommandExecutor shExec = new ShellCommandExecutor(command); if (LOG.isDebugEnabled()) { LOG.debug("signalTask: " + Arrays.toString(command)); } try { shExec.execute(); } catch (ExitCodeException e) { int ret_code = shExec.getExitCode(); if (ret_code != ResultCode.INVALID_TASK_PID.getValue()) { logOutput(shExec.getOutput()); throw new IOException( "Problem signalling task " + taskPid + " with " + signal + "; exit = " + ret_code); } } }
// helper method to override while testing String getAllProcessInfoFromShell() { ShellCommandExecutor shellExecutor = new ShellCommandExecutor( new String[] {Shell.WINUTILS, "task", "processList", taskProcessId}); try { shellExecutor.execute(); return shellExecutor.getOutput(); } catch (IOException e) { LOG.error(StringUtils.stringifyException(e)); } return null; }
static { ShellCommandExecutor shellExecutor = new ShellCommandExecutor(new String[] {"getconf", "CLK_TCK"}); long jiffiesPerSecond = -1; try { shellExecutor.execute(); jiffiesPerSecond = Long.parseLong(shellExecutor.getOutput().replace("\n", "")); } catch (IOException e) { LOG.error(StringUtils.stringifyException(e)); } finally { JIFFY_LENGTH_IN_MILLIS = jiffiesPerSecond != -1 ? Math.round(1000D / jiffiesPerSecond) : -1; } }
static { ShellCommandExecutor shellExecutor = new ShellCommandExecutor(new String[] {"getconf", "PAGESIZE"}); long pageSize = -1; try { shellExecutor.execute(); pageSize = Long.parseLong(shellExecutor.getOutput().replace("\n", "")); } catch (IOException e) { LOG.error(StringUtils.stringifyException(e)); } finally { PAGE_SIZE = pageSize; } }
protected static boolean isSetsidAvailable() { ShellCommandExecutor shexec = null; boolean setsidSupported = true; try { String[] args = {"setsid", "bash", "-c", "echo $$"}; shexec = new ShellCommandExecutor(args); shexec.execute(); } catch (IOException ioe) { LOG.warn("setsid is not available on this machine. So not using it."); setsidSupported = false; } finally { // handle the exit code LOG.info("setsid exited with exit code " + shexec.getExitCode()); } return setsidSupported; }
@Override public void deleteLogAsUser(String user, String subDir) throws IOException { String[] command = new String[] { taskControllerExe, user, localStorage.getDirsString(), Integer.toString(Commands.DELETE_LOG_AS_USER.getValue()), subDir }; ShellCommandExecutor shExec = new ShellCommandExecutor(command); if (LOG.isDebugEnabled()) { LOG.debug("deleteLogAsUser: " + Arrays.toString(command)); } shExec.execute(); }
public static boolean isAvailable() { if (Shell.WINDOWS) { ShellCommandExecutor shellExecutor = new ShellCommandExecutor(new String[] {Shell.WINUTILS, "help"}); try { shellExecutor.execute(); } catch (IOException e) { LOG.error(StringUtils.stringifyException(e)); } finally { String output = shellExecutor.getOutput(); if (output != null && output.contains("Prints to stdout a list of processes in the task")) { return true; } } } return false; }
private String runResolveCommand(List<String> args) { int loopCount = 0; if (args.size() == 0) { return null; } StringBuffer allOutput = new StringBuffer(); int numProcessed = 0; if (maxArgs < MIN_ALLOWABLE_ARGS) { LOG.warn( "Invalid value " + Integer.toString(maxArgs) + " for " + SCRIPT_ARG_COUNT_KEY + "; must be >= " + Integer.toString(MIN_ALLOWABLE_ARGS)); return null; } while (numProcessed != args.size()) { int start = maxArgs * loopCount; List<String> cmdList = new ArrayList<String>(); cmdList.add(scriptName); for (numProcessed = start; numProcessed < (start + maxArgs) && numProcessed < args.size(); numProcessed++) { cmdList.add(args.get(numProcessed)); } File dir = null; String userDir; if ((userDir = System.getProperty("user.dir")) != null) { dir = new File(userDir); } ShellCommandExecutor s = new ShellCommandExecutor(cmdList.toArray(new String[0]), dir); try { s.execute(); allOutput.append(s.getOutput() + " "); } catch (Exception e) { LOG.warn(StringUtils.stringifyException(e)); return null; } loopCount++; } return allOutput.toString(); }
/** * Change the permissions on a file / directory, recursively, if needed. * * @param filename name of the file whose permissions are to change * @param perm permission string * @param recursive true, if permissions should be changed recursively * @return the exit code from the command. * @throws IOException * @throws InterruptedException */ public static int chmod(String filename, String perm, boolean recursive) throws IOException { StringBuffer cmdBuf = new StringBuffer(); cmdBuf.append("chmod "); if (recursive) { cmdBuf.append("-R "); } cmdBuf.append(perm).append(" "); cmdBuf.append(filename); String[] shellCmd = {"bash", "-c", cmdBuf.toString()}; ShellCommandExecutor shExec = new ShellCommandExecutor(shellCmd); try { shExec.execute(); } catch (IOException e) { if (LOG.isDebugEnabled()) { LOG.debug( "Error while changing permission : " + filename + " Exception: " + StringUtils.stringifyException(e)); } } return shExec.getExitCode(); }
@Override public void initializeJob( String user, String jobid, Path credentials, Path jobConf, TaskUmbilicalProtocol taskTracker, InetSocketAddress ttAddr) throws IOException { List<String> command = new ArrayList<String>( Arrays.asList( taskControllerExe, user, localStorage.getDirsString(), Integer.toString(Commands.INITIALIZE_JOB.getValue()), jobid, credentials.toUri().getPath().toString(), jobConf.toUri().getPath().toString())); File jvm = // use same jvm as parent new File(new File(System.getProperty("java.home"), "bin"), "java"); command.add(jvm.toString()); command.add("-classpath"); command.add(System.getProperty("java.class.path")); command.add("-Dhadoop.log.dir=" + TaskLog.getBaseLogDir()); command.add("-Dhadoop.root.logger=INFO,console"); command.add(JobLocalizer.class.getName()); // main of JobLocalizer command.add(user); command.add(jobid); // add the task tracker's reporting address command.add(ttAddr.getHostName()); command.add(Integer.toString(ttAddr.getPort())); String[] commandArray = command.toArray(new String[0]); ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray); if (LOG.isDebugEnabled()) { LOG.debug("initializeJob: " + Arrays.toString(commandArray)); } try { shExec.execute(); if (LOG.isDebugEnabled()) { logOutput(shExec.getOutput()); } } catch (ExitCodeException e) { int exitCode = shExec.getExitCode(); logOutput(shExec.getOutput()); throw new IOException( "Job initialization failed (" + exitCode + ") with output: " + shExec.getOutput(), e); } }
private static void sendSignal(String pid, int signal) throws IOException { ShellCommandExecutor shexec = null; String[] arg = {"kill", "-" + signal, pid}; shexec = new ShellCommandExecutor(arg); shexec.execute(); }
@Override public void execute() throws IOException { super.execute(); }
@Override public void truncateLogsAsUser(String user, List<Task> allAttempts) throws IOException { Task firstTask = allAttempts.get(0); String taskid = firstTask.getTaskID().toString(); LocalDirAllocator ldirAlloc = new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY); String taskRanFile = TaskTracker.TT_LOG_TMP_DIR + Path.SEPARATOR + taskid; Configuration conf = getConf(); // write the serialized task information to a file to pass to the truncater Path taskRanFilePath = ldirAlloc.getLocalPathForWrite(taskRanFile, conf); LocalFileSystem lfs = FileSystem.getLocal(conf); FSDataOutputStream out = lfs.create(taskRanFilePath); out.writeInt(allAttempts.size()); for (Task t : allAttempts) { out.writeBoolean(t.isMapTask()); t.write(out); } out.close(); lfs.setPermission(taskRanFilePath, FsPermission.createImmutable((short) 0755)); List<String> command = new ArrayList<String>(); File jvm = // use same jvm as parent new File(new File(System.getProperty("java.home"), "bin"), "java"); command.add(jvm.toString()); command.add("-Djava.library.path=" + System.getProperty("java.library.path")); command.add("-Dhadoop.log.dir=" + TaskLog.getBaseLogDir()); command.add("-Dhadoop.root.logger=INFO,console"); command.add("-classpath"); command.add(System.getProperty("java.class.path")); // main of TaskLogsTruncater command.add(TaskLogsTruncater.class.getName()); command.add(taskRanFilePath.toString()); String[] taskControllerCmd = new String[4 + command.size()]; taskControllerCmd[0] = taskControllerExe; taskControllerCmd[1] = user; taskControllerCmd[2] = localStorage.getDirsString(); taskControllerCmd[3] = Integer.toString(Commands.RUN_COMMAND_AS_USER.getValue()); int i = 4; for (String cmdArg : command) { taskControllerCmd[i++] = cmdArg; } if (LOG.isDebugEnabled()) { for (String cmd : taskControllerCmd) { LOG.debug("taskctrl command = " + cmd); } } ShellCommandExecutor shExec = new ShellCommandExecutor(taskControllerCmd); try { shExec.execute(); } catch (Exception e) { LOG.warn( "Exit code from " + taskControllerExe.toString() + " is : " + shExec.getExitCode() + " for truncateLogs"); LOG.warn( "Exception thrown by " + taskControllerExe.toString() + " : " + StringUtils.stringifyException(e)); LOG.info("Output from LinuxTaskController's " + taskControllerExe.toString() + " follows:"); logOutput(shExec.getOutput()); lfs.delete(taskRanFilePath, false); throw new IOException(e); } lfs.delete(taskRanFilePath, false); if (LOG.isDebugEnabled()) { LOG.info("Output from LinuxTaskController's " + taskControllerExe.toString() + " follows:"); logOutput(shExec.getOutput()); } }
@Override public int launchTask( String user, String jobId, String attemptId, List<String> setup, List<String> jvmArguments, File currentWorkDirectory, String stdout, String stderr) throws IOException { ShellCommandExecutor shExec = null; try { FileSystem rawFs = FileSystem.getLocal(getConf()).getRaw(); long logSize = 0; // TODO, Ref BUG:2854624 // get the JVM command line. String cmdLine = TaskLog.buildCommandLine( setup, jvmArguments, new File(stdout), new File(stderr), logSize, true); // write the command to a file in the // task specific cache directory Path p = new Path( allocator.getLocalPathForWrite( TaskTracker.getPrivateDirTaskScriptLocation(user, jobId, attemptId), getConf()), COMMAND_FILE); String commandFile = writeCommand(cmdLine, rawFs, p); String[] command = new String[] { taskControllerExe, user, localStorage.getDirsString(), Integer.toString(Commands.LAUNCH_TASK_JVM.getValue()), jobId, attemptId, currentWorkDirectory.toString(), commandFile }; shExec = new ShellCommandExecutor(command); if (LOG.isDebugEnabled()) { LOG.debug("launchTask: " + Arrays.toString(command)); } shExec.execute(); } catch (Exception e) { if (shExec == null) { return -1; } int exitCode = shExec.getExitCode(); LOG.warn("Exit code from task is : " + exitCode); // 143 (SIGTERM) and 137 (SIGKILL) exit codes means the task was // terminated/killed forcefully. In all other cases, log the // task-controller output if (exitCode != 143 && exitCode != 137) { LOG.warn( "Exception thrown while launching task JVM : " + StringUtils.stringifyException(e)); LOG.info("Output from LinuxTaskController's launchTaskJVM follows:"); logOutput(shExec.getOutput()); } return exitCode; } if (LOG.isDebugEnabled()) { LOG.debug("Output from LinuxTaskController's launchTask follows:"); logOutput(shExec.getOutput()); } return 0; }