Ejemplo n.º 1
0
 @Override
 public void initializeJob(
     String user,
     String jobid,
     Path credentials,
     Path jobConf,
     TaskUmbilicalProtocol taskTracker,
     InetSocketAddress ttAddr)
     throws IOException {
   List<String> command =
       new ArrayList<String>(
           Arrays.asList(
               taskControllerExe,
               user,
               localStorage.getDirsString(),
               Integer.toString(Commands.INITIALIZE_JOB.getValue()),
               jobid,
               credentials.toUri().getPath().toString(),
               jobConf.toUri().getPath().toString()));
   File jvm = // use same jvm as parent
       new File(new File(System.getProperty("java.home"), "bin"), "java");
   command.add(jvm.toString());
   command.add("-classpath");
   command.add(System.getProperty("java.class.path"));
   command.add("-Dhadoop.log.dir=" + TaskLog.getBaseLogDir());
   command.add("-Dhadoop.root.logger=INFO,console");
   command.add(JobLocalizer.class.getName()); // main of JobLocalizer
   command.add(user);
   command.add(jobid);
   // add the task tracker's reporting address
   command.add(ttAddr.getHostName());
   command.add(Integer.toString(ttAddr.getPort()));
   String[] commandArray = command.toArray(new String[0]);
   ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
   if (LOG.isDebugEnabled()) {
     LOG.debug("initializeJob: " + Arrays.toString(commandArray));
   }
   try {
     shExec.execute();
     if (LOG.isDebugEnabled()) {
       logOutput(shExec.getOutput());
     }
   } catch (ExitCodeException e) {
     int exitCode = shExec.getExitCode();
     logOutput(shExec.getOutput());
     throw new IOException(
         "Job initialization failed (" + exitCode + ") with output: " + shExec.getOutput(), e);
   }
 }
Ejemplo n.º 2
0
  @Override
  public void truncateLogsAsUser(String user, List<Task> allAttempts) throws IOException {

    Task firstTask = allAttempts.get(0);
    String taskid = firstTask.getTaskID().toString();

    LocalDirAllocator ldirAlloc = new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY);
    String taskRanFile = TaskTracker.TT_LOG_TMP_DIR + Path.SEPARATOR + taskid;
    Configuration conf = getConf();

    // write the serialized task information to a file to pass to the truncater
    Path taskRanFilePath = ldirAlloc.getLocalPathForWrite(taskRanFile, conf);
    LocalFileSystem lfs = FileSystem.getLocal(conf);
    FSDataOutputStream out = lfs.create(taskRanFilePath);
    out.writeInt(allAttempts.size());
    for (Task t : allAttempts) {
      out.writeBoolean(t.isMapTask());
      t.write(out);
    }
    out.close();
    lfs.setPermission(taskRanFilePath, FsPermission.createImmutable((short) 0755));

    List<String> command = new ArrayList<String>();
    File jvm = // use same jvm as parent
        new File(new File(System.getProperty("java.home"), "bin"), "java");
    command.add(jvm.toString());
    command.add("-Djava.library.path=" + System.getProperty("java.library.path"));
    command.add("-Dhadoop.log.dir=" + TaskLog.getBaseLogDir());
    command.add("-Dhadoop.root.logger=INFO,console");
    command.add("-classpath");
    command.add(System.getProperty("java.class.path"));
    // main of TaskLogsTruncater
    command.add(TaskLogsTruncater.class.getName());
    command.add(taskRanFilePath.toString());
    String[] taskControllerCmd = new String[4 + command.size()];
    taskControllerCmd[0] = taskControllerExe;
    taskControllerCmd[1] = user;
    taskControllerCmd[2] = localStorage.getDirsString();
    taskControllerCmd[3] = Integer.toString(Commands.RUN_COMMAND_AS_USER.getValue());

    int i = 4;
    for (String cmdArg : command) {
      taskControllerCmd[i++] = cmdArg;
    }
    if (LOG.isDebugEnabled()) {
      for (String cmd : taskControllerCmd) {
        LOG.debug("taskctrl command = " + cmd);
      }
    }
    ShellCommandExecutor shExec = new ShellCommandExecutor(taskControllerCmd);
    try {
      shExec.execute();
    } catch (Exception e) {
      LOG.warn(
          "Exit code from "
              + taskControllerExe.toString()
              + " is : "
              + shExec.getExitCode()
              + " for truncateLogs");
      LOG.warn(
          "Exception thrown by "
              + taskControllerExe.toString()
              + " : "
              + StringUtils.stringifyException(e));
      LOG.info("Output from LinuxTaskController's " + taskControllerExe.toString() + " follows:");
      logOutput(shExec.getOutput());
      lfs.delete(taskRanFilePath, false);
      throw new IOException(e);
    }
    lfs.delete(taskRanFilePath, false);
    if (LOG.isDebugEnabled()) {
      LOG.info("Output from LinuxTaskController's " + taskControllerExe.toString() + " follows:");
      logOutput(shExec.getOutput());
    }
  }