/** Compile and run */ int runCompile() { // Compile, abort on errors if (verbose) Timer.showStdErr("Parsing"); if (!compile()) { // Show errors and warnings, if any if (!CompilerMessages.get().isEmpty()) System.err.println("Compiler messages:\n" + CompilerMessages.get()); return 1; } if (verbose) Timer.showStdErr("Initializing"); initializeArgs(); // Run the program BdsThread bdsThread = new BdsThread(programUnit, config); if (verbose) Timer.showStdErr("Process ID: " + bdsThread.getBdsThreadId()); // Show script's automatic help message if (showHelp) { if (verbose) Timer.showStdErr("Showing automaic 'help'"); HelpCreator hc = new HelpCreator(programUnit); System.out.println(hc); return 0; } if (verbose) Timer.showStdErr("Running"); int exitCode = runThread(bdsThread); // Check stack if (stackCheck) bdsThread.sanityCheckStack(); return exitCode; }
/** Should we query task states? (i.e. run a command to see if tasks are still alive) */ protected boolean shouldCheck() { if (time == null) time = new Timer(); if (time.elapsedSecs() > CHECK_TASK_RUNNING_INTERVAL) { time.start(); // Restart timer return true; } return false; }
/** Run the program */ @Override protected RunState runStep(BigDataScriptThread csThread) { String msg = ""; if (expr != null) msg = expr.evalString(csThread); // Evaluate expression to show Timer.showStdErr("Warning" + (!msg.isEmpty() ? ": " + msg : "")); return RunState.OK; }
/** Compile and run */ int runTests() { // Compile, abort on errors if (verbose) Timer.showStdErr("Parsing"); if (!compile()) { // Show errors and warnings, if any if (!CompilerMessages.get().isEmpty()) System.err.println("Compiler messages:\n" + CompilerMessages.get()); return 1; } if (verbose) Timer.showStdErr("Initializing"); initializeArgs(); // Run the program BdsThread bdsThread = new BdsThread(programUnit, config); if (verbose) Timer.showStdErr("Process ID: " + bdsThread.getBdsThreadId()); if (verbose) Timer.showStdErr("Running tests"); ProgramUnit pu = bdsThread.getProgramUnit(); List<FunctionDeclaration> testFuncs = pu.testsFunctions(); // For each test function, create a thread that executes the function's body int exitCode = 0; int testOk = 0, testError = 0; for (FunctionDeclaration testFunc : testFuncs) { System.out.println(""); BdsThread bdsTestThread = new BdsThread( testFunc.getStatement(), bdsThread); // Note: We execute the function's body (not the function declaration) int exitValTest = runThread(bdsTestThread); // Show test result if (exitValTest == 0) { Timer.show("Test '" + testFunc.getFunctionName() + "': OK"); testOk++; } else { Timer.show("Test '" + testFunc.getFunctionName() + "': FAIL"); exitCode = 1; testError++; } } // Show results System.out.println(""); Timer.show( "Totals" // + "\n OK : " + testOk // + "\n ERROR : " + testError // ); return exitCode; }
/** * Run the program */ @Override public void runStep(BdsThread bdsThread) { String msg = ""; if (expr != null) { // Evaluate expression to show bdsThread.run(expr); msg = popString(bdsThread); } // Do not show error during checkpoint recovery if (bdsThread.isCheckpointRecover()) return; // Error Timer.showStdErr("Error" + (!msg.isEmpty() ? ": " + msg : "")); bdsThread.setExitValue(1L); // Set exit value bdsThread.setRunState(RunState.EXIT); }
/** Initialize before running or type-checking */ void initialize() { Type.reset(); // Reset node factory BdsNodeFactory.reset(); // Startup message if (verbose || debug) Timer.showStdErr(VERSION); // Load config file config(); // Global scope initilaizeGlobalScope(); // Libraries initilaizeLibraries(); }
/** * Increment counter that keeps track on how many times in a row a task was missing * * @return true if task should be considered 'missing' */ protected boolean incMissingCount(Task task) { String id = task.getId(); int count = (missingCount.containsKey(id) ? missingCount.get(id) + 1 : 1); missingCount.put(id, count); if (debug) Timer.showStdErr( "WARNING: Task PID '" + task.getPid() + "' not found for task '" + id + "'. Incrementing 'missing counter': " + count + " (max. allowed " + TASK_NOT_FOUND_DISAPPEARED + ")"); return count > TASK_NOT_FOUND_DISAPPEARED; }
/** Run script */ public int run() { // Initialize Executioners executioners = Executioners.getInstance(config); TaskDependecies.reset(); // Check PID regex if (checkPidRegex) { checkPidRegex(); return 0; } // --- // Run // --- int exitValue = 0; switch (bdsAction) { case RUN_CHECKPOINT: exitValue = runCheckpoint(); break; case INFO_CHECKPOINT: exitValue = infoCheckpoint(); break; case TEST: exitValue = runTests(); break; default: exitValue = runCompile(); // Compile & run } if (verbose) Timer.showStdErr("Finished. Exit code: " + exitValue); // --- // Kill all executioners // --- for (Executioner executioner : executioners.getAll()) executioner.kill(); config.kill(); // Kill 'tail' and 'monitor' threads return exitValue; }
void log(String msg) { Timer.showStdErr(getClass().getSimpleName() + ": " + msg); }
/** * Set command line arguments as global variables * * <p>How it works: - Program is executes as something like: * * <p>java -jar BigDataScript.jar [options] programFile.bds [programOptions] * * <p>- Any command line argument AFTER "programFile.bds" is considered a command line argument * for the BigDataScript program. E.g. * * <p>java -jar BigDataScript.jar -v program.bds -file myFile.txt -verbose -num 7 * * <p>So our program "program.bds" has command line options: -file myFile.txt -verbose -num 7 * (notice that "-v" is a command line option for loudScript.jar and not for "program.bds") * * <p>- We look for variables in ProgramUnit that match the name of these command line arguments * * <p>- Then we add those values to the variable initialization. Thus overriding any * initialization values provided in the program. E.g. * * <p>Our program has the following variable declarations: string file = "default_file.txt" int * num = 3 bool verbose = false * * <p>We execute the program: java -jar BigDataScript.jar -v program.bds -file myFile.txt -verbose * -num 7 * * <p>The variable declarations are replaced as follows: string file = "myFile.txt" int num = 7 * bool verbose = true * * <p>- Note: Only primitive types are supported (i.e.: string, bool, int & real) * * <p>- Note: Unmatched variables names will be silently ignored (same for variables that match, * but are non-primitive) * * <p>- Note: If a variable is matched, is primitive, but cannot be converted. An error is thrown. * E.g.: Program: int num = 1 * * <p>Command line: java -jar BigDataScript.jar program.bds -num "hello" <- This is an error * because 'num' is an int * * <p>- Note: Unprocessed arguments will be available to the program as an 'args' list */ void initializeArgs() { // Set program arguments as global variables for (int argNum = 0; argNum < programArgs.size(); argNum++) { String arg = programArgs.get(argNum); // Parse '-OPT' option if (arg.equalsIgnoreCase("-h") || arg.equalsIgnoreCase("-help") || arg.equalsIgnoreCase("--help")) { if (debug) Timer.showStdErr("Activating 'show help' mode"); showHelp = true; } else if (arg.startsWith("-")) { // Get variable name and value String varName = arg.substring(1); // Find all variable declarations that match this command line argument for (VarDeclaration varDecl : programUnit.varDeclarations(true)) { Type varType = varDecl.getType(); // Is is a primitive variable or a primitive list? if (varType.isPrimitiveType() || varType.isList()) { // Find an initialization that matches the command line argument for (VariableInit varInit : varDecl.getVarInit()) if (varInit.getVarName().equals(varName)) { // Name matches? int argNumOri = argNum; boolean useVal = false; if (varType.isList()) { // Create a list of arguments and use them to initialize the variable (list) ArrayList<String> vals = new ArrayList<String>(); for (int i = argNum + 1; i < programArgs.size(); i++) if (programArgs.get(i).startsWith("-")) break; else vals.add(programArgs.get(i)); useVal = initializeArgs( varType, varInit, vals); // Found variable, try to replace or add LITERAL to this VarInit } else if (varType.isBool()) { String valStr = "true"; // Booleans may not have a value (just '-varName' sets them to 'true') if (programArgs.size() > (argNum + 1)) { // Is the next argument 'true' or 'false'? => Set argument String boolVal = programArgs.get(argNum + 1); if (valStr.equalsIgnoreCase("true") || valStr.equalsIgnoreCase("false")) valStr = boolVal; } initializeArgs(varType, varInit, valStr); } else { String val = (argNum < programArgs.size() ? programArgs.get(++argNum) : ""); // Get one argument and use it to initialize the variable useVal = initializeArgs( varType, varInit, val); // Found variable, try to replace or add LITERAL to this VarInit } if (!useVal) argNum = argNumOri; // We did not use the arguments } } } } } // Make all unprocessed arguments available for the program (in 'args' list) Scope.getGlobalScope() .add(new ScopeSymbol(Scope.GLOBAL_VAR_ARGS_LIST, TypeList.get(Type.STRING), programArgs)); // Initialize program name String programPath = programUnit.getFileName(); String progName = Gpr.baseName(programPath); Scope.getGlobalScope() .add(new ScopeSymbol(Scope.GLOBAL_VAR_PROGRAM_NAME, Type.STRING, progName)); Scope.getGlobalScope() .add(new ScopeSymbol(Scope.GLOBAL_VAR_PROGRAM_PATH, Type.STRING, programPath)); }
/** * Create an AST from a program (using ANTLR lexer & parser) Returns null if error Use * 'alreadyIncluded' to keep track of from 'include' statements */ public static ParseTree createAst(File file, boolean debug, Set<String> alreadyIncluded) { alreadyIncluded.add(Gpr.getCanonicalFileName(file)); String fileName = file.toString(); String filePath = fileName; BigDataScriptLexer lexer = null; BigDataScriptParser parser = null; try { filePath = file.getCanonicalPath(); // Input stream if (!Gpr.canRead(filePath)) { CompilerMessages.get().addError("Can't read file '" + filePath + "'"); return null; } // Create a CharStream that reads from standard input ANTLRFileStream input = new ANTLRFileStream(fileName); // --- // Lexer: Create a lexer that feeds off of input CharStream // --- lexer = new BigDataScriptLexer(input) { @Override public void recover(LexerNoViableAltException e) { throw new RuntimeException(e); // Bail out } }; // --- // Parser // --- CommonTokenStream tokens = new CommonTokenStream(lexer); parser = new BigDataScriptParser(tokens); // Parser error handling parser.setErrorHandler( new CompileErrorStrategy()); // Bail out with exception if errors in parser parser.addErrorListener(new CompilerErrorListener()); // Catch some other error messages that // 'CompileErrorStrategy' fails to catch // Begin parsing at main rule ParseTree tree = parser.programUnit(); // Error loading file? if (tree == null) { System.err.println("Can't parse file '" + filePath + "'"); return null; } // Show main nodes if (debug) { Timer.showStdErr("AST:"); for (int childNum = 0; childNum < tree.getChildCount(); childNum++) { Tree child = tree.getChild(childNum); System.err.println( "\t\tChild " + childNum + ":\t" + child + "\tTree:'" + child.toStringTree() + "'"); } } // Included files boolean resolveIncludePending = true; while (resolveIncludePending) resolveIncludePending = resolveIncludes(tree, debug, alreadyIncluded); return tree; } catch (Exception e) { String msg = e.getMessage(); CompilerMessages.get() .addError( "Could not compile " + filePath // + (msg != null ? " :" + e.getMessage() : "") // ); return null; } }
/** * Run a command to find running processes PIDs * * @return true if OK, false on failure */ protected boolean runCommand() { // Prepare command line arguments ArrayList<String> args = new ArrayList<String>(); StringBuilder cmdsb = new StringBuilder(); for (String arg : defaultCmdArgs) { args.add(arg); cmdsb.append(" " + arg); } // Execute command cmdExecResult = Exec.exec(args, true); if (debug) Timer.showStdErr( "Check task running:" // + "\n\tCommand : '" + cmdsb.toString().trim() + "'" // + "\n\tExit value : " + cmdExecResult.exitValue // + "\n\tStdout : " + cmdExecResult.stdOut // + "\n\tStderr : " + cmdExecResult.stdErr // ); // --- // Sanity checks! // --- // Failed command? if (cmdExecResult.exitValue > 0) { Timer.showStdErr( "WARNING: There was an error executing cluster stat command: '" + cmdsb.toString().trim() + "'.\nExit code: " + cmdExecResult.exitValue); return false; } // Any problems reported on STDERR? if (!cmdExecResult.stdErr.isEmpty()) { Timer.showStdErr( "WARNING: There was an error executing cluster stat command: '" + cmdsb.toString().trim() + "'\nSTDERR:\n" + cmdExecResult.stdErr); return false; } // Empty STDOUT? // Note: This might not be a problem, but a cluster (or any computer) running // zero processes is really weird. Furthermore, this commands usually show some // kind of header, so STDOUT is never empty if (verbose && cmdExecResult.stdOut.isEmpty()) { Timer.showStdErr( "WARNING: Empty STDOUT when executing cluster stat command: '" + cmdsb.toString().trim()); return false; } // OK return true; }