/** Compile and run */
int runCompile() {
// Compile, abort on errors
if (verbose) Timer.showStdErr("Parsing");
if (!compile()) {
// Show errors and warnings, if any
if (!CompilerMessages.get().isEmpty())
System.err.println("Compiler messages:\n" + CompilerMessages.get());
return 1;
}
if (verbose) Timer.showStdErr("Initializing");
initializeArgs();
// Run the program
BdsThread bdsThread = new BdsThread(programUnit, config);
if (verbose) Timer.showStdErr("Process ID: " + bdsThread.getBdsThreadId());
// Show script's automatic help message
if (showHelp) {
if (verbose) Timer.showStdErr("Showing automaic 'help'");
HelpCreator hc = new HelpCreator(programUnit);
System.out.println(hc);
return 0;
}
if (verbose) Timer.showStdErr("Running");
int exitCode = runThread(bdsThread);
// Check stack
if (stackCheck) bdsThread.sanityCheckStack();
return exitCode;
}
/** Should we query task states? (i.e. run a command to see if tasks are still alive) */
protected boolean shouldCheck() {
if (time == null) time = new Timer();
if (time.elapsedSecs() > CHECK_TASK_RUNNING_INTERVAL) {
time.start(); // Restart timer
return true;
}
return false;
}
/** Run the program */
@Override
protected RunState runStep(BigDataScriptThread csThread) {
String msg = "";
if (expr != null) msg = expr.evalString(csThread); // Evaluate expression to show
Timer.showStdErr("Warning" + (!msg.isEmpty() ? ": " + msg : ""));
return RunState.OK;
}
/** Compile and run */
int runTests() {
// Compile, abort on errors
if (verbose) Timer.showStdErr("Parsing");
if (!compile()) {
// Show errors and warnings, if any
if (!CompilerMessages.get().isEmpty())
System.err.println("Compiler messages:\n" + CompilerMessages.get());
return 1;
}
if (verbose) Timer.showStdErr("Initializing");
initializeArgs();
// Run the program
BdsThread bdsThread = new BdsThread(programUnit, config);
if (verbose) Timer.showStdErr("Process ID: " + bdsThread.getBdsThreadId());
if (verbose) Timer.showStdErr("Running tests");
ProgramUnit pu = bdsThread.getProgramUnit();
List<FunctionDeclaration> testFuncs = pu.testsFunctions();
// For each test function, create a thread that executes the function's body
int exitCode = 0;
int testOk = 0, testError = 0;
for (FunctionDeclaration testFunc : testFuncs) {
System.out.println("");
BdsThread bdsTestThread =
new BdsThread(
testFunc.getStatement(),
bdsThread); // Note: We execute the function's body (not the function declaration)
int exitValTest = runThread(bdsTestThread);
// Show test result
if (exitValTest == 0) {
Timer.show("Test '" + testFunc.getFunctionName() + "': OK");
testOk++;
} else {
Timer.show("Test '" + testFunc.getFunctionName() + "': FAIL");
exitCode = 1;
testError++;
}
}
// Show results
System.out.println("");
Timer.show(
"Totals" //
+ "\n OK : "
+ testOk //
+ "\n ERROR : "
+ testError //
);
return exitCode;
}
/**
* Run the program
*/
@Override
public void runStep(BdsThread bdsThread) {
String msg = "";
if (expr != null) {
// Evaluate expression to show
bdsThread.run(expr);
msg = popString(bdsThread);
}
// Do not show error during checkpoint recovery
if (bdsThread.isCheckpointRecover()) return;
// Error
Timer.showStdErr("Error" + (!msg.isEmpty() ? ": " + msg : ""));
bdsThread.setExitValue(1L); // Set exit value
bdsThread.setRunState(RunState.EXIT);
}
/** Initialize before running or type-checking */
void initialize() {
Type.reset();
// Reset node factory
BdsNodeFactory.reset();
// Startup message
if (verbose || debug) Timer.showStdErr(VERSION);
// Load config file
config();
// Global scope
initilaizeGlobalScope();
// Libraries
initilaizeLibraries();
}
/**
* Increment counter that keeps track on how many times in a row a task was missing
*
* @return true if task should be considered 'missing'
*/
protected boolean incMissingCount(Task task) {
String id = task.getId();
int count = (missingCount.containsKey(id) ? missingCount.get(id) + 1 : 1);
missingCount.put(id, count);
if (debug)
Timer.showStdErr(
"WARNING: Task PID '"
+ task.getPid()
+ "' not found for task '"
+ id
+ "'. Incrementing 'missing counter': "
+ count
+ " (max. allowed "
+ TASK_NOT_FOUND_DISAPPEARED
+ ")");
return count > TASK_NOT_FOUND_DISAPPEARED;
}
/** Run script */
public int run() {
// Initialize
Executioners executioners = Executioners.getInstance(config);
TaskDependecies.reset();
// Check PID regex
if (checkPidRegex) {
checkPidRegex();
return 0;
}
// ---
// Run
// ---
int exitValue = 0;
switch (bdsAction) {
case RUN_CHECKPOINT:
exitValue = runCheckpoint();
break;
case INFO_CHECKPOINT:
exitValue = infoCheckpoint();
break;
case TEST:
exitValue = runTests();
break;
default:
exitValue = runCompile(); // Compile & run
}
if (verbose) Timer.showStdErr("Finished. Exit code: " + exitValue);
// ---
// Kill all executioners
// ---
for (Executioner executioner : executioners.getAll()) executioner.kill();
config.kill(); // Kill 'tail' and 'monitor' threads
return exitValue;
}
void log(String msg) {
Timer.showStdErr(getClass().getSimpleName() + ": " + msg);
}
/**
* Set command line arguments as global variables
*
* <p>How it works: - Program is executes as something like:
*
* <p>java -jar BigDataScript.jar [options] programFile.bds [programOptions]
*
* <p>- Any command line argument AFTER "programFile.bds" is considered a command line argument
* for the BigDataScript program. E.g.
*
* <p>java -jar BigDataScript.jar -v program.bds -file myFile.txt -verbose -num 7
*
* <p>So our program "program.bds" has command line options: -file myFile.txt -verbose -num 7
* (notice that "-v" is a command line option for loudScript.jar and not for "program.bds")
*
* <p>- We look for variables in ProgramUnit that match the name of these command line arguments
*
* <p>- Then we add those values to the variable initialization. Thus overriding any
* initialization values provided in the program. E.g.
*
* <p>Our program has the following variable declarations: string file = "default_file.txt" int
* num = 3 bool verbose = false
*
* <p>We execute the program: java -jar BigDataScript.jar -v program.bds -file myFile.txt -verbose
* -num 7
*
* <p>The variable declarations are replaced as follows: string file = "myFile.txt" int num = 7
* bool verbose = true
*
* <p>- Note: Only primitive types are supported (i.e.: string, bool, int & real)
*
* <p>- Note: Unmatched variables names will be silently ignored (same for variables that match,
* but are non-primitive)
*
* <p>- Note: If a variable is matched, is primitive, but cannot be converted. An error is thrown.
* E.g.: Program: int num = 1
*
* <p>Command line: java -jar BigDataScript.jar program.bds -num "hello" <- This is an error
* because 'num' is an int
*
* <p>- Note: Unprocessed arguments will be available to the program as an 'args' list
*/
void initializeArgs() {
// Set program arguments as global variables
for (int argNum = 0; argNum < programArgs.size(); argNum++) {
String arg = programArgs.get(argNum);
// Parse '-OPT' option
if (arg.equalsIgnoreCase("-h")
|| arg.equalsIgnoreCase("-help")
|| arg.equalsIgnoreCase("--help")) {
if (debug) Timer.showStdErr("Activating 'show help' mode");
showHelp = true;
} else if (arg.startsWith("-")) {
// Get variable name and value
String varName = arg.substring(1);
// Find all variable declarations that match this command line argument
for (VarDeclaration varDecl : programUnit.varDeclarations(true)) {
Type varType = varDecl.getType();
// Is is a primitive variable or a primitive list?
if (varType.isPrimitiveType() || varType.isList()) {
// Find an initialization that matches the command line argument
for (VariableInit varInit : varDecl.getVarInit())
if (varInit.getVarName().equals(varName)) { // Name matches?
int argNumOri = argNum;
boolean useVal = false;
if (varType.isList()) {
// Create a list of arguments and use them to initialize the variable (list)
ArrayList<String> vals = new ArrayList<String>();
for (int i = argNum + 1; i < programArgs.size(); i++)
if (programArgs.get(i).startsWith("-")) break;
else vals.add(programArgs.get(i));
useVal =
initializeArgs(
varType, varInit,
vals); // Found variable, try to replace or add LITERAL to this VarInit
} else if (varType.isBool()) {
String valStr = "true";
// Booleans may not have a value (just '-varName' sets them to 'true')
if (programArgs.size() > (argNum + 1)) {
// Is the next argument 'true' or 'false'? => Set argument
String boolVal = programArgs.get(argNum + 1);
if (valStr.equalsIgnoreCase("true") || valStr.equalsIgnoreCase("false"))
valStr = boolVal;
}
initializeArgs(varType, varInit, valStr);
} else {
String val =
(argNum < programArgs.size()
? programArgs.get(++argNum)
: ""); // Get one argument and use it to initialize the variable
useVal =
initializeArgs(
varType, varInit,
val); // Found variable, try to replace or add LITERAL to this VarInit
}
if (!useVal) argNum = argNumOri; // We did not use the arguments
}
}
}
}
}
// Make all unprocessed arguments available for the program (in 'args' list)
Scope.getGlobalScope()
.add(new ScopeSymbol(Scope.GLOBAL_VAR_ARGS_LIST, TypeList.get(Type.STRING), programArgs));
// Initialize program name
String programPath = programUnit.getFileName();
String progName = Gpr.baseName(programPath);
Scope.getGlobalScope()
.add(new ScopeSymbol(Scope.GLOBAL_VAR_PROGRAM_NAME, Type.STRING, progName));
Scope.getGlobalScope()
.add(new ScopeSymbol(Scope.GLOBAL_VAR_PROGRAM_PATH, Type.STRING, programPath));
}
/**
* Create an AST from a program (using ANTLR lexer & parser) Returns null if error Use
* 'alreadyIncluded' to keep track of from 'include' statements
*/
public static ParseTree createAst(File file, boolean debug, Set<String> alreadyIncluded) {
alreadyIncluded.add(Gpr.getCanonicalFileName(file));
String fileName = file.toString();
String filePath = fileName;
BigDataScriptLexer lexer = null;
BigDataScriptParser parser = null;
try {
filePath = file.getCanonicalPath();
// Input stream
if (!Gpr.canRead(filePath)) {
CompilerMessages.get().addError("Can't read file '" + filePath + "'");
return null;
}
// Create a CharStream that reads from standard input
ANTLRFileStream input = new ANTLRFileStream(fileName);
// ---
// Lexer: Create a lexer that feeds off of input CharStream
// ---
lexer =
new BigDataScriptLexer(input) {
@Override
public void recover(LexerNoViableAltException e) {
throw new RuntimeException(e); // Bail out
}
};
// ---
// Parser
// ---
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new BigDataScriptParser(tokens);
// Parser error handling
parser.setErrorHandler(
new CompileErrorStrategy()); // Bail out with exception if errors in parser
parser.addErrorListener(new CompilerErrorListener()); // Catch some other error messages that
// 'CompileErrorStrategy' fails to catch
// Begin parsing at main rule
ParseTree tree = parser.programUnit();
// Error loading file?
if (tree == null) {
System.err.println("Can't parse file '" + filePath + "'");
return null;
}
// Show main nodes
if (debug) {
Timer.showStdErr("AST:");
for (int childNum = 0; childNum < tree.getChildCount(); childNum++) {
Tree child = tree.getChild(childNum);
System.err.println(
"\t\tChild " + childNum + ":\t" + child + "\tTree:'" + child.toStringTree() + "'");
}
}
// Included files
boolean resolveIncludePending = true;
while (resolveIncludePending)
resolveIncludePending = resolveIncludes(tree, debug, alreadyIncluded);
return tree;
} catch (Exception e) {
String msg = e.getMessage();
CompilerMessages.get()
.addError(
"Could not compile "
+ filePath //
+ (msg != null ? " :" + e.getMessage() : "") //
);
return null;
}
}
/**
* Run a command to find running processes PIDs
*
* @return true if OK, false on failure
*/
protected boolean runCommand() {
// Prepare command line arguments
ArrayList<String> args = new ArrayList<String>();
StringBuilder cmdsb = new StringBuilder();
for (String arg : defaultCmdArgs) {
args.add(arg);
cmdsb.append(" " + arg);
}
// Execute command
cmdExecResult = Exec.exec(args, true);
if (debug)
Timer.showStdErr(
"Check task running:" //
+ "\n\tCommand : '"
+ cmdsb.toString().trim()
+ "'" //
+ "\n\tExit value : "
+ cmdExecResult.exitValue //
+ "\n\tStdout : "
+ cmdExecResult.stdOut //
+ "\n\tStderr : "
+ cmdExecResult.stdErr //
);
// ---
// Sanity checks!
// ---
// Failed command?
if (cmdExecResult.exitValue > 0) {
Timer.showStdErr(
"WARNING: There was an error executing cluster stat command: '"
+ cmdsb.toString().trim()
+ "'.\nExit code: "
+ cmdExecResult.exitValue);
return false;
}
// Any problems reported on STDERR?
if (!cmdExecResult.stdErr.isEmpty()) {
Timer.showStdErr(
"WARNING: There was an error executing cluster stat command: '"
+ cmdsb.toString().trim()
+ "'\nSTDERR:\n"
+ cmdExecResult.stdErr);
return false;
}
// Empty STDOUT?
// Note: This might not be a problem, but a cluster (or any computer) running
// zero processes is really weird. Furthermore, this commands usually show some
// kind of header, so STDOUT is never empty
if (verbose && cmdExecResult.stdOut.isEmpty()) {
Timer.showStdErr(
"WARNING: Empty STDOUT when executing cluster stat command: '" + cmdsb.toString().trim());
return false;
}
// OK
return true;
}