@Override
  public void parse(String cmdResult[]) {
    long swap, swapFree, mem, memFree;
    swap = swapFree = mem = memFree = -1;

    for (int line = 0; line < cmdResult.length; line++) {
      String fields[] = cmdResult[line].replace(':', ' ').split("\\s+");
      if (fields[0].equalsIgnoreCase("MemTotal")) {
        mem = Gpr.parseIntSafe(fields[1]);
        host.getResources().setMem(mem);
      }
      if (fields[0].equalsIgnoreCase("MemFree")) memFree = Gpr.parseLongSafe(fields[1]);
      if (fields[0].equalsIgnoreCase("SwapTotal")) swap = Gpr.parseLongSafe(fields[1]);
      if (fields[0].equalsIgnoreCase("SwapFree")) swapFree = Gpr.parseLongSafe(fields[1]);

      if (debug) {
        Gpr.debug("line[" + line + "]: " + cmdResult[line]);
        for (int i = 0; i < fields.length; i++)
          Gpr.debug("\tfields[" + i + "]: '" + fields[i] + "'");
      }
    }

    // Calculate swap usage
    if (debug) Gpr.debug("swap" + swap + " swapFree:" + swapFree);
    if ((swap > 0) && (swapFree >= 0)) {
      float swapUsage = ((float) (swap - swapFree)) / ((float) swap);
      host.getHealth().setSwapUsage(swapUsage);
    }

    // Calculate memory usage
    if (debug) Gpr.debug("mem:" + mem + " memFree:" + memFree);
    if ((mem > 0) && (memFree >= 0)) {
      float memUsage = ((float) (mem - memFree)) / ((float) mem);
      host.getHealth().setMemUsage(memUsage);
    }
  }
예제 #2
0
  /**
   * Parse command line arguments
   *
   * @param args
   */
  public void parse(String[] args) {
    // Nothing? Show command line options
    if (args.length <= 0) usage(null);

    programArgs = new ArrayList<String>();
    bdsAction = BdsAction.RUN;

    for (int i = 0; i < args.length; i++) {
      String arg = args[i];

      if (programFileName != null) {
        // Everything after 'programFileName' is an command line
        // argument for the BigDataScript program
        programArgs.add(arg);
      } else if (isOpt(arg)) {

        switch (arg.toLowerCase()) {
          case "-checkpidregex":
            checkPidRegex = true;
            break;

          case "-c":
          case "-config":
            // Checkpoint restore
            if ((i + 1) < args.length) configFile = args[++i];
            else usage("Option '-c' without restore file argument");
            break;

          case "-d":
          case "-debug":
            debug = verbose = true; // Debug implies verbose
            break;

          case "-download":
            if ((i + 2) < args.length) {
              config();
              boolean ok = download(args[++i], args[++i]);
              System.exit(ok ? 0 : 1);
            } else usage("Option '-download' requires two parameters (URL and file)");
            break;

          case "-dryrun":
            dryRun = true;
            noRmOnExit = true; // Not running, so don't delete files
            reportHtml = reportYaml = false;
            break;

          case "-extractsource":
            extractSource = true;
            break;

          case "-h":
          case "-help":
          case "--help":
            usage(null);
            break;

          case "-i":
          case "-info":
            // Checkpoint info
            if ((i + 1) < args.length) chekcpointRestoreFile = args[++i];
            else usage("Option '-i' without checkpoint file argument");
            bdsAction = BdsAction.INFO_CHECKPOINT;
            break;

          case "-l":
          case "-log":
            log = true;
            break;

          case "-nochp":
            noCheckpoint = true;
            break;

          case "-noreport":
            reportHtml = reportYaml = false;
            break;

          case "-noreporthtml":
            reportHtml = false;
            break;

          case "-noreportyaml":
            reportYaml = false;
            break;

          case "-normonexit":
            noRmOnExit = true;
            break;

          case "-pid":
            // PID file
            if ((i + 1) < args.length) pidFile = args[++i];
            else usage("Option '-pid' without file argument");
            break;

          case "-q":
          case "-queue":
            // Queue name
            if ((i + 1) < args.length) queue = args[++i];
            else usage("Option '-queue' without file argument");
            break;

          case "-quiet":
            verbose = false;
            debug = false;
            quiet = true;
            break;

          case "-r":
          case "-restore":
            // Checkpoint restore
            if ((i + 1) < args.length) chekcpointRestoreFile = args[++i];
            else usage("Option '-r' without checkpoint file argument");
            bdsAction = BdsAction.RUN_CHECKPOINT;
            break;

          case "-reporthtml":
            reportHtml = true;
            break;

          case "-reportname":
            if ((i + 1) < args.length) reportFileName = args[++i];
            else usage("Option '-reportName' without name argument");
            break;

          case "-reportyaml":
          case "-yaml":
            reportYaml = true;
            break;

          case "-s":
          case "-system":
            // System type
            if ((i + 1) < args.length) system = args[++i];
            else usage("Option '-system' without file argument");
            break;

          case "-t":
          case "-test":
            bdsAction = BdsAction.TEST;
            break;

          case "-upload":
            if ((i + 2) < args.length) {
              config();
              boolean ok = upload(args[++i], args[++i]);
              System.exit(ok ? 0 : 1);
            } else usage("Option '-upload' requires two parameters (file and URL)");
            break;

          case "-v":
          case "-verbose":
            verbose = true;
            break;

          case "-version":
            System.out.println(VERSION);
            System.exit(0);
            break;

          case "-y":
          case "-retry":
            // Number of retries
            if ((i + 1) < args.length) taskFailCount = Gpr.parseIntSafe(args[++i]);
            else usage("Option '-t' without number argument");
            break;

          default:
            usage("Unknown command line option " + arg);
        }
      } else if (programFileName == null) programFileName = arg; // Get program file name
    }

    // Sanity checks
    if (checkPidRegex) {
      // OK: Nothing to chek
    } else if ((programFileName == null) && (chekcpointRestoreFile == null)) {
      // No file name => Error
      usage("Missing program file name.");
    }
  }
예제 #3
0
  /** Add symbols to global scope */
  void initilaizeGlobalScope() {
    if (debug) log("Initialize global scope.");

    // Reset Global scope
    Scope.resetGlobalScope();
    Scope globalScope = Scope.getGlobalScope();

    // --
    // Get default veluas from command line or config file
    // ---

    // Command line parameters override defaults
    String cpusStr =
        config.getString(ExpressionTask.TASK_OPTION_CPUS, "1"); // Default number of cpus: 1
    long cpus = Gpr.parseIntSafe(cpusStr);
    if (cpus <= 0)
      throw new RuntimeException("Number of cpus must be a positive number ('" + cpusStr + "')");

    long mem =
        Gpr.parseMemSafe(
            config.getString(
                ExpressionTask.TASK_OPTION_MEM,
                "-1")); // Default amount of memory: -1 (unrestricted)
    String node = config.getString(ExpressionTask.TASK_OPTION_NODE, "");
    if (queue == null) queue = config.getString(ExpressionTask.TASK_OPTION_QUEUE, "");
    if (system == null)
      system =
          config.getString(
              ExpressionTask.TASK_OPTION_SYSTEM, ExecutionerType.LOCAL.toString().toLowerCase());
    if (taskFailCount < 0)
      taskFailCount = Gpr.parseIntSafe(config.getString(ExpressionTask.TASK_OPTION_RETRY, "0"));

    long oneDay = 1L * 24 * 60 * 60;
    long timeout =
        Gpr.parseLongSafe(config.getString(ExpressionTask.TASK_OPTION_TIMEOUT, "" + oneDay));
    long wallTimeout =
        Gpr.parseLongSafe(config.getString(ExpressionTask.TASK_OPTION_WALL_TIMEOUT, "" + oneDay));

    // ---
    // Add global symbols
    // ---
    globalScope.add(
        new ScopeSymbol(
            Scope.GLOBAL_VAR_PROGRAM_NAME,
            Type.STRING,
            "")); // Now is empty, but they are assigned later
    globalScope.add(new ScopeSymbol(Scope.GLOBAL_VAR_PROGRAM_PATH, Type.STRING, ""));

    // Task related variables: Default values
    globalScope.add(
        new ScopeSymbol(
            ExpressionTask.TASK_OPTION_SYSTEM,
            Type.STRING,
            system)); // System type: "local", "ssh", "cluster", "aws", etc.
    globalScope.add(
        new ScopeSymbol(ExpressionTask.TASK_OPTION_CPUS, Type.INT, cpus)); // Default number of cpus
    globalScope.add(
        new ScopeSymbol(
            ExpressionTask.TASK_OPTION_MEM,
            Type.INT,
            mem)); // Default amount of memory (unrestricted)
    globalScope.add(
        new ScopeSymbol(
            ExpressionTask.TASK_OPTION_QUEUE, Type.STRING, queue)); // Default queue: none
    globalScope.add(
        new ScopeSymbol(ExpressionTask.TASK_OPTION_NODE, Type.STRING, node)); // Default node: none
    globalScope.add(
        new ScopeSymbol(
            ExpressionTask.TASK_OPTION_CAN_FAIL,
            Type.BOOL,
            false)); // Task fail triggers checkpoint & exit (a task cannot fail)
    globalScope.add(
        new ScopeSymbol(
            ExpressionTask.TASK_OPTION_ALLOW_EMPTY,
            Type.BOOL,
            false)); // Tasks are allowed to have empty output file/s
    globalScope.add(
        new ScopeSymbol(
            ExpressionTask.TASK_OPTION_RETRY,
            Type.INT,
            (long) taskFailCount)); // Task fail can be re-tried (re-run) N times before considering
    // failed.
    globalScope.add(
        new ScopeSymbol(
            ExpressionTask.TASK_OPTION_TIMEOUT, Type.INT, timeout)); // Task default timeout
    globalScope.add(
        new ScopeSymbol(
            ExpressionTask.TASK_OPTION_WALL_TIMEOUT,
            Type.INT,
            wallTimeout)); // Task default wall-timeout

    // Number of local CPUs
    // Kilo, Mega, Giga, Tera, Peta.
    LinkedList<ScopeSymbol> constants = new LinkedList<ScopeSymbol>();
    constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_LOCAL_CPUS, Type.INT, (long) Gpr.NUM_CORES));
    constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_K, Type.INT, 1024L));
    constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_M, Type.INT, 1024L * 1024L));
    constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_G, Type.INT, 1024L * 1024L * 1024L));
    constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_T, Type.INT, 1024L * 1024L * 1024L * 1024L));
    constants.add(
        new ScopeSymbol(Scope.GLOBAL_VAR_P, Type.INT, 1024L * 1024L * 1024L * 1024L * 1024L));
    constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_MINUTE, Type.INT, 60L));
    constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_HOUR, Type.INT, (long) (60 * 60)));
    constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_DAY, Type.INT, (long) (24 * 60 * 60)));
    constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_WEEK, Type.INT, (long) (7 * 24 * 60 * 60)));

    // Math constants
    constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_E, Type.REAL, Math.E));
    constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_PI, Type.REAL, Math.PI));

    // Add all constants
    for (ScopeSymbol ss : constants) {
      ss.setConstant(true);
      globalScope.add(ss);
    }

    // Set "physical" path
    String path;
    try {
      path = new File(".").getCanonicalPath();
    } catch (IOException e) {
      throw new RuntimeException("Cannot get cannonical path for current dir");
    }
    globalScope.add(new ScopeSymbol(ExpressionTask.TASK_OPTION_PHYSICAL_PATH, Type.STRING, path));

    // Set all environment variables
    Map<String, String> envMap = System.getenv();
    for (String varName : envMap.keySet()) {
      String varVal = envMap.get(varName);
      globalScope.add(new ScopeSymbol(varName, Type.STRING, varVal));
    }

    // Command line arguments (default: empty list)
    // This is properly set in 'initializeArgs()' method, but
    // we have to set something now, otherwise we'll get a "variable
    // not found" error at compiler time, if the program attempts
    // to use 'args'.
    Scope.getGlobalScope()
        .add(
            new ScopeSymbol(
                Scope.GLOBAL_VAR_ARGS_LIST, TypeList.get(Type.STRING), new ArrayList<String>()));
  }
예제 #4
0
  /**
   * Set command line arguments as global variables
   *
   * <p>How it works: - Program is executes as something like:
   *
   * <p>java -jar BigDataScript.jar [options] programFile.bds [programOptions]
   *
   * <p>- Any command line argument AFTER "programFile.bds" is considered a command line argument
   * for the BigDataScript program. E.g.
   *
   * <p>java -jar BigDataScript.jar -v program.bds -file myFile.txt -verbose -num 7
   *
   * <p>So our program "program.bds" has command line options: -file myFile.txt -verbose -num 7
   * (notice that "-v" is a command line option for loudScript.jar and not for "program.bds")
   *
   * <p>- We look for variables in ProgramUnit that match the name of these command line arguments
   *
   * <p>- Then we add those values to the variable initialization. Thus overriding any
   * initialization values provided in the program. E.g.
   *
   * <p>Our program has the following variable declarations: string file = "default_file.txt" int
   * num = 3 bool verbose = false
   *
   * <p>We execute the program: java -jar BigDataScript.jar -v program.bds -file myFile.txt -verbose
   * -num 7
   *
   * <p>The variable declarations are replaced as follows: string file = "myFile.txt" int num = 7
   * bool verbose = true
   *
   * <p>- Note: Only primitive types are supported (i.e.: string, bool, int & real)
   *
   * <p>- Note: Unmatched variables names will be silently ignored (same for variables that match,
   * but are non-primitive)
   *
   * <p>- Note: If a variable is matched, is primitive, but cannot be converted. An error is thrown.
   * E.g.: Program: int num = 1
   *
   * <p>Command line: java -jar BigDataScript.jar program.bds -num "hello" <- This is an error
   * because 'num' is an int
   *
   * <p>- Note: Unprocessed arguments will be available to the program as an 'args' list
   */
  void initializeArgs() {
    // Set program arguments as global variables
    for (int argNum = 0; argNum < programArgs.size(); argNum++) {
      String arg = programArgs.get(argNum);

      // Parse '-OPT' option
      if (arg.equalsIgnoreCase("-h")
          || arg.equalsIgnoreCase("-help")
          || arg.equalsIgnoreCase("--help")) {
        if (debug) Timer.showStdErr("Activating 'show help' mode");
        showHelp = true;
      } else if (arg.startsWith("-")) {
        // Get variable name and value
        String varName = arg.substring(1);

        // Find all variable declarations that match this command line argument
        for (VarDeclaration varDecl : programUnit.varDeclarations(true)) {
          Type varType = varDecl.getType();

          // Is is a primitive variable or a primitive list?
          if (varType.isPrimitiveType() || varType.isList()) {

            // Find an initialization that matches the command line argument
            for (VariableInit varInit : varDecl.getVarInit())
              if (varInit.getVarName().equals(varName)) { // Name matches?
                int argNumOri = argNum;
                boolean useVal = false;

                if (varType.isList()) {
                  // Create a list of arguments and use them to initialize the variable (list)
                  ArrayList<String> vals = new ArrayList<String>();
                  for (int i = argNum + 1; i < programArgs.size(); i++)
                    if (programArgs.get(i).startsWith("-")) break;
                    else vals.add(programArgs.get(i));

                  useVal =
                      initializeArgs(
                          varType, varInit,
                          vals); // Found variable, try to replace or add LITERAL to this VarInit
                } else if (varType.isBool()) {
                  String valStr = "true";

                  // Booleans may not have a value (just '-varName' sets them to 'true')
                  if (programArgs.size() > (argNum + 1)) {
                    // Is the next argument 'true' or 'false'? => Set argument
                    String boolVal = programArgs.get(argNum + 1);
                    if (valStr.equalsIgnoreCase("true") || valStr.equalsIgnoreCase("false"))
                      valStr = boolVal;
                  }

                  initializeArgs(varType, varInit, valStr);
                } else {
                  String val =
                      (argNum < programArgs.size()
                          ? programArgs.get(++argNum)
                          : ""); // Get one argument and use it to initialize the variable
                  useVal =
                      initializeArgs(
                          varType, varInit,
                          val); // Found variable, try to replace or add LITERAL to this VarInit
                }

                if (!useVal) argNum = argNumOri; // We did not use the arguments
              }
          }
        }
      }
    }

    // Make all unprocessed arguments available for the program (in 'args' list)
    Scope.getGlobalScope()
        .add(new ScopeSymbol(Scope.GLOBAL_VAR_ARGS_LIST, TypeList.get(Type.STRING), programArgs));

    // Initialize program name
    String programPath = programUnit.getFileName();
    String progName = Gpr.baseName(programPath);
    Scope.getGlobalScope()
        .add(new ScopeSymbol(Scope.GLOBAL_VAR_PROGRAM_NAME, Type.STRING, progName));
    Scope.getGlobalScope()
        .add(new ScopeSymbol(Scope.GLOBAL_VAR_PROGRAM_PATH, Type.STRING, programPath));
  }
예제 #5
0
  /** Resolve include statements */
  private static boolean resolveIncludes(
      ParseTree tree, boolean debug, Set<String> alreadyIncluded) {
    boolean changed = false;
    if (tree instanceof IncludeFileContext) {
      // Parent file: The one that is including the other file
      File parentFile =
          new File(((IncludeFileContext) tree).getStart().getInputStream().getSourceName());

      // Included file name
      String includedFilename = StatementInclude.includeFileName(tree.getChild(1).getText());

      // Find file (look into all include paths)
      File includedFile = StatementInclude.includeFile(includedFilename, parentFile);
      if (includedFile == null) {
        CompilerMessages.get()
            .add(
                tree,
                parentFile,
                "\n\tIncluded file not found: '"
                    + includedFilename
                    + "'\n\tSearch path: "
                    + Config.get().getIncludePath(),
                MessageType.ERROR);
        return false;
      }

      // Already included? don't bother
      String canonicalFileName = Gpr.getCanonicalFileName(includedFile);
      if (alreadyIncluded.contains(canonicalFileName)) {
        if (debug)
          Gpr.debug(
              "File already included: '"
                  + includedFilename
                  + "'\tCanonical path: '"
                  + canonicalFileName
                  + "'");
        return false;
      }
      if (!includedFile.canRead()) {
        CompilerMessages.get()
            .add(
                tree,
                parentFile,
                "\n\tCannot read included file: '" + includedFilename + "'",
                MessageType.ERROR);
        return false;
      }

      // Parse
      ParseTree treeinc = createAst(includedFile, debug, alreadyIncluded);
      if (treeinc == null) {
        CompilerMessages.get()
            .add(
                tree,
                parentFile,
                "\n\tFatal error including file '" + includedFilename + "'",
                MessageType.ERROR);
        return false;
      }

      // Is a child always a RuleContext?
      for (int i = 0; i < treeinc.getChildCount(); i++) {
        ((IncludeFileContext) tree).addChild((RuleContext) treeinc.getChild(i));
      }
    } else {
      for (int i = 0; i < tree.getChildCount(); i++)
        changed |= resolveIncludes(tree.getChild(i), debug, alreadyIncluded);
    }

    return changed;
  }
예제 #6
0
  /**
   * Create an AST from a program (using ANTLR lexer & parser) Returns null if error Use
   * 'alreadyIncluded' to keep track of from 'include' statements
   */
  public static ParseTree createAst(File file, boolean debug, Set<String> alreadyIncluded) {
    alreadyIncluded.add(Gpr.getCanonicalFileName(file));
    String fileName = file.toString();
    String filePath = fileName;

    BigDataScriptLexer lexer = null;
    BigDataScriptParser parser = null;

    try {
      filePath = file.getCanonicalPath();

      // Input stream
      if (!Gpr.canRead(filePath)) {
        CompilerMessages.get().addError("Can't read file '" + filePath + "'");
        return null;
      }

      // Create a CharStream that reads from standard input
      ANTLRFileStream input = new ANTLRFileStream(fileName);

      // ---
      // Lexer: Create a lexer that feeds off of input CharStream
      // ---
      lexer =
          new BigDataScriptLexer(input) {
            @Override
            public void recover(LexerNoViableAltException e) {
              throw new RuntimeException(e); // Bail out
            }
          };

      // ---
      // Parser
      // ---
      CommonTokenStream tokens = new CommonTokenStream(lexer);
      parser = new BigDataScriptParser(tokens);

      // Parser error handling
      parser.setErrorHandler(
          new CompileErrorStrategy()); // Bail out with exception if errors in parser
      parser.addErrorListener(new CompilerErrorListener()); // Catch some other error messages that
      // 'CompileErrorStrategy' fails to catch

      // Begin parsing at main rule
      ParseTree tree = parser.programUnit();

      // Error loading file?
      if (tree == null) {
        System.err.println("Can't parse file '" + filePath + "'");
        return null;
      }

      // Show main nodes
      if (debug) {
        Timer.showStdErr("AST:");
        for (int childNum = 0; childNum < tree.getChildCount(); childNum++) {
          Tree child = tree.getChild(childNum);
          System.err.println(
              "\t\tChild " + childNum + ":\t" + child + "\tTree:'" + child.toStringTree() + "'");
        }
      }

      // Included files
      boolean resolveIncludePending = true;
      while (resolveIncludePending)
        resolveIncludePending = resolveIncludes(tree, debug, alreadyIncluded);

      return tree;
    } catch (Exception e) {
      String msg = e.getMessage();
      CompilerMessages.get()
          .addError(
              "Could not compile "
                  + filePath //
                  + (msg != null ? " :" + e.getMessage() : "") //
              );
      return null;
    }
  }