Exemplo n.º 1
0
 public static void raiseException(Throwable t) {
   error(t + ":\n" + StrUtils.join(t.getStackTrace(), "\n"));
   t = t.getCause();
   if (t != null) error("Caused by " + t + ":\n" + StrUtils.join(t.getStackTrace(), "\n"));
   Execution.putOutput("exec.status", "exception");
   exitCode = 1;
 }
Exemplo n.º 2
0
 // This should be all we need to put in a main function.
 // args are the commandline arguments
 // First object is the Runnable object to call run on.
 // All of them are objects whose options args is to supposed to populate.
 public static void run(String[] args, Object... objects) {
   init(args, objects);
   if (makeThunk) {
     setExecStatus("thunk", true);
     printOutputMap(Execution.getFile("output.map"));
     List<String> cmd = new ArrayList();
     cmd.add("java");
     if (thunkJavaOpts != null) cmd.add(thunkJavaOpts);
     cmd.add("-cp " + StrUtils.join(jarFiles, ":") + ":" + System.getProperty("java.class.path"));
     cmd.addAll(
         ListUtils.newList(
             objects[0].getClass().getName(),
             "++" + virtualExecDir, // Load these options
             // Next time when we run, just run in the same path that we used to create the thunk
             "-execDir",
             virtualExecDir,
             "-overwriteExecDir"));
     IOUtils.printLinesHard(
         Execution.getFile("job.map"),
         ListUtils.newList(
             "workingDir\t" + SysInfoUtils.getcwd(), // Run from current directory
             "command\t" + StrUtils.join(cmd, "\t"),
             "priority\t" + thunkPriority));
   } else {
     if (dontCatchExceptions) {
       ((Runnable) objects[0]).run();
     } else {
       try {
         ((Runnable) objects[0]).run();
       } catch (Throwable t) {
         raiseException(t);
       }
     }
   }
   finish();
 }
Exemplo n.º 3
0
  /** Return an unused directory in the execution pool directory. Set virtualExecDir */
  public static String createVirtualExecDir() {
    if (useStandardExecPoolDirStrategy) {
      // Assume we are in the run directory, so set the standard paths
      execPoolDir = new File(SysInfoUtils.getcwd(), "state/execs").toString();
      actualExecPoolDir =
          new File(SysInfoUtils.getcwd(), "state/hosts/" + SysInfoUtils.getShortHostName())
              .toString();
      if (!new File(actualExecPoolDir).isDirectory()) actualExecPoolDir = null;
    }
    if (!StrUtils.isEmpty(execPoolDir) && !new File(execPoolDir).isDirectory())
      throw Exceptions.bad("Execution pool directory '" + execPoolDir + "' doesn't exist");
    if (!StrUtils.isEmpty(actualExecPoolDir) && !new File(actualExecPoolDir).isDirectory())
      throw Exceptions.bad(
          "Actual execution pool directory '" + actualExecPoolDir + "' doesn't exist");

    if (!StrUtils.isEmpty(execDir)) { // Use specified execDir
      boolean exists = new File(execDir).isDirectory();
      if (exists && !overwriteExecDir)
        throw Exceptions.bad("Directory %s already exists and overwrite flag is false", execDir);
      if (!exists) mkdirHard(new File(execDir));
      else {
        // This part looks at actualExecPoolDir
        // This case is overwriting an existing execution directory, which
        // happens when we are executing a thunk.  We have to be careful here
        // because the actual symlinked directory that was created when thunking
        // might be using a different actualPoolDir.  If this happens, we need
        // to move the actual thunked symlinked directory into the actual
        // execution pool directory requested.  In fact, we always do this for simplicity.
        String oldActualExecDir = Utils.systemGetStringOutputEasy("readlink " + execDir);
        if (oldActualExecDir == null) { // Not symlink
          if (!StrUtils.isEmpty(actualExecPoolDir))
            throw Exceptions.bad(
                "The old execution directory was not created with actualExecPoolDir but now we want an actualExecPoolDir");
          // Do nothing, just use the directory as is
        } else { // Symlink
          oldActualExecDir = oldActualExecDir.trim();
          if (StrUtils.isEmpty(actualExecPoolDir))
            throw Exceptions.bad(
                "The old execution directory was created with actualExecPoolDir but now we don't want an actualExecPoolDir");
          // Note that now the execution numbers might not correspond between the
          // actual and virtual execution pool directories.
          File newActualExecDir = null;
          for (int i = 0; ; i++) {
            newActualExecDir = new File(actualExecPoolDir, i + "a.exec");
            if (!newActualExecDir.exists()) break;
          }
          // Move the old directory to the new directory
          Utils.systemHard(String.format("mv %s %s", oldActualExecDir, newActualExecDir));
          // Update the symlink (execDir -> newActualExecDir)
          Utils.systemHard(
              String.format("ln -sf %s %s", newActualExecDir.getAbsolutePath(), execDir));
        }
      }
      return virtualExecDir = execDir;
    }

    // execDir hasn't been specified, so we need to pick one from a pool directory
    // execPoolDir must exist; actualExecPoolDir is optional

    // Get a list of files that already exists
    Set<String> files = new HashSet<String>();
    for (String f : new File(execPoolDir).list()) files.add(f);

    // Go through and pick out a file that doesn't exist
    int numFailures = 0;
    for (int i = 0; numFailures < 3; i++) {
      // Either the virtual file (a link) or the actual file
      File f = new File(execPoolDir, i + ".exec");
      // Actual file
      File g =
          StrUtils.isEmpty(actualExecPoolDir) ? null : new File(actualExecPoolDir, i + ".exec");

      if (!files.contains(i + ".exec") && (g == null || !g.exists())) {
        if (g == null || g.equals(f)) {
          mkdirHard(f);
          return virtualExecDir = f.toString();
        }
        // Create symlink before mkdir to try to reserve the name and avoid race conditions
        if (Utils.createSymLink(g.getAbsolutePath(), f.getAbsolutePath())) {
          mkdirHard(g);
          return virtualExecDir = f.toString();
        }

        // Probably because someone else already linked to it
        // in the race condition: so try again
        stderr.println("Cannot create symlink from " + f + " to " + g);
        numFailures++;
      }
    }
    throw Exceptions.bad("Failed many times to create execution directory");
  }
Exemplo n.º 4
0
  public static void init(String[] args, Object... objects) {
    // Parse options
    parser = new OptionsParser();
    parser.doRegister("log", LogInfo.class);
    parser.doRegister("exec", Execution.class);
    parser.doRegisterAll(objects);
    // These options are specific to the execution, so we don't want to overwrite them
    // with a previous execution's.
    parser.setDefaultDirFileName("options.map");
    parser.setIgnoreOptsFromFileName(
        "options.map",
        ListUtils.newList(
            "log.file",
            "exec.execDir",
            "exec.execPoolDir",
            "exec.actualPoolDir",
            "exec.makeThunk"));
    if (!parser.doParse(args)) System.exit(1);

    // Load classes
    if (jarFiles.size() > 0) {
      List<String> names = new ArrayList();
      for (String jarFile : jarFiles) names.add(new File(jarFile).getName());
      stderr.println("Loading JAR files: " + StrUtils.join(names));
      for (String jarFile : jarFiles) // Load classes
      ClassInitializer.initializeJar(jarFile);
    }
    // Set character encoding
    if (charEncoding != null) CharEncUtils.setCharEncoding(charEncoding);

    if (printOptionsAndExit) { // Just print options and exit
      parser.doGetOptionPairs().print(stdout);
      System.exit(0);
    }

    // Create a new directory
    if (create) {
      createVirtualExecDir();
      stderr.println(virtualExecDir);
      if (!makeThunk) LogInfo.file = getFile("log");

      // Copy the Jar files for reference
      if (!makeThunk) {
        for (String jarFile : jarFiles)
          Utils.systemHard(String.format("cp %s %s", jarFile, virtualExecDir));
      }
    } else {
      LogInfo.file = "";
    }

    if (!makeThunk) {
      LogInfo.init();
      if (startMainTrack) track("main()", true);
    }

    // Output options
    if (!makeThunk && virtualExecDir != null) logs("Execution directory: " + virtualExecDir);
    if (!makeThunk) getInfo().printEasy(getFile("info.map"));
    printOptions();
    if (create && addToView.size() > 0)
      IOUtils.printLinesHard(Execution.getFile("addToView"), addToView);

    // Start monitoring
    if (!makeThunk && monitor) {
      monitorThread = new MonitorThread();
      monitorThread.start();
    }

    if (!makeThunk) Record.init(Execution.getFile("record"));
  }
Exemplo n.º 5
0
 public static synchronized void printOutputMap(String path) {
   if (StrUtils.isEmpty(path)) return;
   // First write to a temporary directory and then rename the file
   String tmpPath = path + ".tmp";
   if (outputMap.printEasy(tmpPath)) new File(tmpPath).renameTo(new File(path));
 }
Exemplo n.º 6
0
 public static synchronized void putOutput(String s, Object t) {
   outputMap.put(s, StrUtils.toString(t));
 }
Exemplo n.º 7
0
 public static void linkFileFromExec(String file, String realFileName) {
   if (StrUtils.isEmpty(realFileName) || StrUtils.isEmpty(file)) return;
   File f = new File(realFileName);
   Utils.createSymLink(getFile(file), f.getAbsolutePath());
 }
Exemplo n.º 8
0
 // Get the path of the file (in the execution directory)
 public static String getFile(String file) {
   if (StrUtils.isEmpty(virtualExecDir)) return null;
   if (StrUtils.isEmpty(file)) return null;
   return new File(virtualExecDir, file).toString();
 }