Example #1
0
  /** Compile and run */
  int runCompile() {
    // Compile, abort on errors
    if (verbose) Timer.showStdErr("Parsing");
    if (!compile()) {
      // Show errors and warnings, if any
      if (!CompilerMessages.get().isEmpty())
        System.err.println("Compiler messages:\n" + CompilerMessages.get());
      return 1;
    }

    if (verbose) Timer.showStdErr("Initializing");
    initializeArgs();

    // Run the program
    BdsThread bdsThread = new BdsThread(programUnit, config);
    if (verbose) Timer.showStdErr("Process ID: " + bdsThread.getBdsThreadId());

    // Show script's automatic help message
    if (showHelp) {
      if (verbose) Timer.showStdErr("Showing automaic 'help'");
      HelpCreator hc = new HelpCreator(programUnit);
      System.out.println(hc);
      return 0;
    }

    if (verbose) Timer.showStdErr("Running");
    int exitCode = runThread(bdsThread);

    // Check stack
    if (stackCheck) bdsThread.sanityCheckStack();

    return exitCode;
  }
 /** Should we query task states? (i.e. run a command to see if tasks are still alive) */
 protected boolean shouldCheck() {
   if (time == null) time = new Timer();
   if (time.elapsedSecs() > CHECK_TASK_RUNNING_INTERVAL) {
     time.start(); // Restart timer
     return true;
   }
   return false;
 }
  /** Run the program */
  @Override
  protected RunState runStep(BigDataScriptThread csThread) {
    String msg = "";
    if (expr != null) msg = expr.evalString(csThread); // Evaluate expression to show

    Timer.showStdErr("Warning" + (!msg.isEmpty() ? ": " + msg : ""));

    return RunState.OK;
  }
Example #4
0
  /** Compile and run */
  int runTests() {
    // Compile, abort on errors
    if (verbose) Timer.showStdErr("Parsing");
    if (!compile()) {
      // Show errors and warnings, if any
      if (!CompilerMessages.get().isEmpty())
        System.err.println("Compiler messages:\n" + CompilerMessages.get());
      return 1;
    }

    if (verbose) Timer.showStdErr("Initializing");
    initializeArgs();

    // Run the program
    BdsThread bdsThread = new BdsThread(programUnit, config);
    if (verbose) Timer.showStdErr("Process ID: " + bdsThread.getBdsThreadId());

    if (verbose) Timer.showStdErr("Running tests");
    ProgramUnit pu = bdsThread.getProgramUnit();
    List<FunctionDeclaration> testFuncs = pu.testsFunctions();

    // For each test function, create a thread that executes the function's body
    int exitCode = 0;
    int testOk = 0, testError = 0;
    for (FunctionDeclaration testFunc : testFuncs) {
      System.out.println("");
      BdsThread bdsTestThread =
          new BdsThread(
              testFunc.getStatement(),
              bdsThread); // Note: We execute the function's body (not the function declaration)
      int exitValTest = runThread(bdsTestThread);

      // Show test result
      if (exitValTest == 0) {
        Timer.show("Test '" + testFunc.getFunctionName() + "': OK");
        testOk++;
      } else {
        Timer.show("Test '" + testFunc.getFunctionName() + "': FAIL");
        exitCode = 1;
        testError++;
      }
    }

    // Show results
    System.out.println("");
    Timer.show(
        "Totals" //
            + "\n                  OK    : "
            + testOk //
            + "\n                  ERROR : "
            + testError //
        );
    return exitCode;
  }
Example #5
0
	/**
	 * Run the program
	 */
	@Override
	public void runStep(BdsThread bdsThread) {
		String msg = "";
		if (expr != null) {
			// Evaluate expression to show
			bdsThread.run(expr);
			msg = popString(bdsThread);
		}

		// Do not show error during checkpoint recovery
		if (bdsThread.isCheckpointRecover()) return;

		// Error
		Timer.showStdErr("Error" + (!msg.isEmpty() ? ": " + msg : ""));
		bdsThread.setExitValue(1L); // Set exit value
		bdsThread.setRunState(RunState.EXIT);
	}
Example #6
0
  /** Initialize before running or type-checking */
  void initialize() {
    Type.reset();

    // Reset node factory
    BdsNodeFactory.reset();

    // Startup message
    if (verbose || debug) Timer.showStdErr(VERSION);

    // Load config file
    config();

    // Global scope
    initilaizeGlobalScope();

    // Libraries
    initilaizeLibraries();
  }
  /**
   * Increment counter that keeps track on how many times in a row a task was missing
   *
   * @return true if task should be considered 'missing'
   */
  protected boolean incMissingCount(Task task) {
    String id = task.getId();
    int count = (missingCount.containsKey(id) ? missingCount.get(id) + 1 : 1);
    missingCount.put(id, count);

    if (debug)
      Timer.showStdErr(
          "WARNING: Task PID '"
              + task.getPid()
              + "' not found for task '"
              + id
              + "'. Incrementing 'missing counter': "
              + count
              + " (max. allowed "
              + TASK_NOT_FOUND_DISAPPEARED
              + ")");
    return count > TASK_NOT_FOUND_DISAPPEARED;
  }
Example #8
0
  /** Run script */
  public int run() {
    // Initialize
    Executioners executioners = Executioners.getInstance(config);
    TaskDependecies.reset();

    // Check PID regex
    if (checkPidRegex) {
      checkPidRegex();
      return 0;
    }

    // ---
    // Run
    // ---
    int exitValue = 0;
    switch (bdsAction) {
      case RUN_CHECKPOINT:
        exitValue = runCheckpoint();
        break;

      case INFO_CHECKPOINT:
        exitValue = infoCheckpoint();
        break;

      case TEST:
        exitValue = runTests();
        break;

      default:
        exitValue = runCompile(); // Compile & run
    }
    if (verbose) Timer.showStdErr("Finished. Exit code: " + exitValue);

    // ---
    // Kill all executioners
    // ---
    for (Executioner executioner : executioners.getAll()) executioner.kill();

    config.kill(); // Kill 'tail' and 'monitor' threads

    return exitValue;
  }
Example #9
0
 void log(String msg) {
   Timer.showStdErr(getClass().getSimpleName() + ": " + msg);
 }
Example #10
0
  /**
   * Set command line arguments as global variables
   *
   * <p>How it works: - Program is executes as something like:
   *
   * <p>java -jar BigDataScript.jar [options] programFile.bds [programOptions]
   *
   * <p>- Any command line argument AFTER "programFile.bds" is considered a command line argument
   * for the BigDataScript program. E.g.
   *
   * <p>java -jar BigDataScript.jar -v program.bds -file myFile.txt -verbose -num 7
   *
   * <p>So our program "program.bds" has command line options: -file myFile.txt -verbose -num 7
   * (notice that "-v" is a command line option for loudScript.jar and not for "program.bds")
   *
   * <p>- We look for variables in ProgramUnit that match the name of these command line arguments
   *
   * <p>- Then we add those values to the variable initialization. Thus overriding any
   * initialization values provided in the program. E.g.
   *
   * <p>Our program has the following variable declarations: string file = "default_file.txt" int
   * num = 3 bool verbose = false
   *
   * <p>We execute the program: java -jar BigDataScript.jar -v program.bds -file myFile.txt -verbose
   * -num 7
   *
   * <p>The variable declarations are replaced as follows: string file = "myFile.txt" int num = 7
   * bool verbose = true
   *
   * <p>- Note: Only primitive types are supported (i.e.: string, bool, int & real)
   *
   * <p>- Note: Unmatched variables names will be silently ignored (same for variables that match,
   * but are non-primitive)
   *
   * <p>- Note: If a variable is matched, is primitive, but cannot be converted. An error is thrown.
   * E.g.: Program: int num = 1
   *
   * <p>Command line: java -jar BigDataScript.jar program.bds -num "hello" <- This is an error
   * because 'num' is an int
   *
   * <p>- Note: Unprocessed arguments will be available to the program as an 'args' list
   */
  void initializeArgs() {
    // Set program arguments as global variables
    for (int argNum = 0; argNum < programArgs.size(); argNum++) {
      String arg = programArgs.get(argNum);

      // Parse '-OPT' option
      if (arg.equalsIgnoreCase("-h")
          || arg.equalsIgnoreCase("-help")
          || arg.equalsIgnoreCase("--help")) {
        if (debug) Timer.showStdErr("Activating 'show help' mode");
        showHelp = true;
      } else if (arg.startsWith("-")) {
        // Get variable name and value
        String varName = arg.substring(1);

        // Find all variable declarations that match this command line argument
        for (VarDeclaration varDecl : programUnit.varDeclarations(true)) {
          Type varType = varDecl.getType();

          // Is is a primitive variable or a primitive list?
          if (varType.isPrimitiveType() || varType.isList()) {

            // Find an initialization that matches the command line argument
            for (VariableInit varInit : varDecl.getVarInit())
              if (varInit.getVarName().equals(varName)) { // Name matches?
                int argNumOri = argNum;
                boolean useVal = false;

                if (varType.isList()) {
                  // Create a list of arguments and use them to initialize the variable (list)
                  ArrayList<String> vals = new ArrayList<String>();
                  for (int i = argNum + 1; i < programArgs.size(); i++)
                    if (programArgs.get(i).startsWith("-")) break;
                    else vals.add(programArgs.get(i));

                  useVal =
                      initializeArgs(
                          varType, varInit,
                          vals); // Found variable, try to replace or add LITERAL to this VarInit
                } else if (varType.isBool()) {
                  String valStr = "true";

                  // Booleans may not have a value (just '-varName' sets them to 'true')
                  if (programArgs.size() > (argNum + 1)) {
                    // Is the next argument 'true' or 'false'? => Set argument
                    String boolVal = programArgs.get(argNum + 1);
                    if (valStr.equalsIgnoreCase("true") || valStr.equalsIgnoreCase("false"))
                      valStr = boolVal;
                  }

                  initializeArgs(varType, varInit, valStr);
                } else {
                  String val =
                      (argNum < programArgs.size()
                          ? programArgs.get(++argNum)
                          : ""); // Get one argument and use it to initialize the variable
                  useVal =
                      initializeArgs(
                          varType, varInit,
                          val); // Found variable, try to replace or add LITERAL to this VarInit
                }

                if (!useVal) argNum = argNumOri; // We did not use the arguments
              }
          }
        }
      }
    }

    // Make all unprocessed arguments available for the program (in 'args' list)
    Scope.getGlobalScope()
        .add(new ScopeSymbol(Scope.GLOBAL_VAR_ARGS_LIST, TypeList.get(Type.STRING), programArgs));

    // Initialize program name
    String programPath = programUnit.getFileName();
    String progName = Gpr.baseName(programPath);
    Scope.getGlobalScope()
        .add(new ScopeSymbol(Scope.GLOBAL_VAR_PROGRAM_NAME, Type.STRING, progName));
    Scope.getGlobalScope()
        .add(new ScopeSymbol(Scope.GLOBAL_VAR_PROGRAM_PATH, Type.STRING, programPath));
  }
Example #11
0
  /**
   * Create an AST from a program (using ANTLR lexer & parser) Returns null if error Use
   * 'alreadyIncluded' to keep track of from 'include' statements
   */
  public static ParseTree createAst(File file, boolean debug, Set<String> alreadyIncluded) {
    alreadyIncluded.add(Gpr.getCanonicalFileName(file));
    String fileName = file.toString();
    String filePath = fileName;

    BigDataScriptLexer lexer = null;
    BigDataScriptParser parser = null;

    try {
      filePath = file.getCanonicalPath();

      // Input stream
      if (!Gpr.canRead(filePath)) {
        CompilerMessages.get().addError("Can't read file '" + filePath + "'");
        return null;
      }

      // Create a CharStream that reads from standard input
      ANTLRFileStream input = new ANTLRFileStream(fileName);

      // ---
      // Lexer: Create a lexer that feeds off of input CharStream
      // ---
      lexer =
          new BigDataScriptLexer(input) {
            @Override
            public void recover(LexerNoViableAltException e) {
              throw new RuntimeException(e); // Bail out
            }
          };

      // ---
      // Parser
      // ---
      CommonTokenStream tokens = new CommonTokenStream(lexer);
      parser = new BigDataScriptParser(tokens);

      // Parser error handling
      parser.setErrorHandler(
          new CompileErrorStrategy()); // Bail out with exception if errors in parser
      parser.addErrorListener(new CompilerErrorListener()); // Catch some other error messages that
      // 'CompileErrorStrategy' fails to catch

      // Begin parsing at main rule
      ParseTree tree = parser.programUnit();

      // Error loading file?
      if (tree == null) {
        System.err.println("Can't parse file '" + filePath + "'");
        return null;
      }

      // Show main nodes
      if (debug) {
        Timer.showStdErr("AST:");
        for (int childNum = 0; childNum < tree.getChildCount(); childNum++) {
          Tree child = tree.getChild(childNum);
          System.err.println(
              "\t\tChild " + childNum + ":\t" + child + "\tTree:'" + child.toStringTree() + "'");
        }
      }

      // Included files
      boolean resolveIncludePending = true;
      while (resolveIncludePending)
        resolveIncludePending = resolveIncludes(tree, debug, alreadyIncluded);

      return tree;
    } catch (Exception e) {
      String msg = e.getMessage();
      CompilerMessages.get()
          .addError(
              "Could not compile "
                  + filePath //
                  + (msg != null ? " :" + e.getMessage() : "") //
              );
      return null;
    }
  }
  /**
   * Run a command to find running processes PIDs
   *
   * @return true if OK, false on failure
   */
  protected boolean runCommand() {
    // Prepare command line arguments
    ArrayList<String> args = new ArrayList<String>();
    StringBuilder cmdsb = new StringBuilder();
    for (String arg : defaultCmdArgs) {
      args.add(arg);
      cmdsb.append(" " + arg);
    }

    // Execute command
    cmdExecResult = Exec.exec(args, true);
    if (debug)
      Timer.showStdErr(
          "Check task running:" //
              + "\n\tCommand    : '"
              + cmdsb.toString().trim()
              + "'" //
              + "\n\tExit value : "
              + cmdExecResult.exitValue //
              + "\n\tStdout     : "
              + cmdExecResult.stdOut //
              + "\n\tStderr     : "
              + cmdExecResult.stdErr //
          );

    // ---
    // Sanity checks!
    // ---

    // Failed command?
    if (cmdExecResult.exitValue > 0) {
      Timer.showStdErr(
          "WARNING: There was an error executing cluster stat command: '"
              + cmdsb.toString().trim()
              + "'.\nExit code: "
              + cmdExecResult.exitValue);
      return false;
    }

    // Any problems reported on STDERR?
    if (!cmdExecResult.stdErr.isEmpty()) {
      Timer.showStdErr(
          "WARNING: There was an error executing cluster stat command: '"
              + cmdsb.toString().trim()
              + "'\nSTDERR:\n"
              + cmdExecResult.stdErr);
      return false;
    }

    // Empty STDOUT?
    // Note: This might not be a problem, but a cluster (or any computer) running
    // zero processes is really weird. Furthermore, this commands usually show some
    // kind of header, so STDOUT is never empty
    if (verbose && cmdExecResult.stdOut.isEmpty()) {
      Timer.showStdErr(
          "WARNING: Empty STDOUT when executing cluster stat command: '" + cmdsb.toString().trim());
      return false;
    }

    // OK
    return true;
  }