@Override
public void parse(String cmdResult[]) {
long swap, swapFree, mem, memFree;
swap = swapFree = mem = memFree = -1;
for (int line = 0; line < cmdResult.length; line++) {
String fields[] = cmdResult[line].replace(':', ' ').split("\\s+");
if (fields[0].equalsIgnoreCase("MemTotal")) {
mem = Gpr.parseIntSafe(fields[1]);
host.getResources().setMem(mem);
}
if (fields[0].equalsIgnoreCase("MemFree")) memFree = Gpr.parseLongSafe(fields[1]);
if (fields[0].equalsIgnoreCase("SwapTotal")) swap = Gpr.parseLongSafe(fields[1]);
if (fields[0].equalsIgnoreCase("SwapFree")) swapFree = Gpr.parseLongSafe(fields[1]);
if (debug) {
Gpr.debug("line[" + line + "]: " + cmdResult[line]);
for (int i = 0; i < fields.length; i++)
Gpr.debug("\tfields[" + i + "]: '" + fields[i] + "'");
}
}
// Calculate swap usage
if (debug) Gpr.debug("swap" + swap + " swapFree:" + swapFree);
if ((swap > 0) && (swapFree >= 0)) {
float swapUsage = ((float) (swap - swapFree)) / ((float) swap);
host.getHealth().setSwapUsage(swapUsage);
}
// Calculate memory usage
if (debug) Gpr.debug("mem:" + mem + " memFree:" + memFree);
if ((mem > 0) && (memFree >= 0)) {
float memUsage = ((float) (mem - memFree)) / ((float) mem);
host.getHealth().setMemUsage(memUsage);
}
}
/**
* Parse command line arguments
*
* @param args
*/
public void parse(String[] args) {
// Nothing? Show command line options
if (args.length <= 0) usage(null);
programArgs = new ArrayList<String>();
bdsAction = BdsAction.RUN;
for (int i = 0; i < args.length; i++) {
String arg = args[i];
if (programFileName != null) {
// Everything after 'programFileName' is an command line
// argument for the BigDataScript program
programArgs.add(arg);
} else if (isOpt(arg)) {
switch (arg.toLowerCase()) {
case "-checkpidregex":
checkPidRegex = true;
break;
case "-c":
case "-config":
// Checkpoint restore
if ((i + 1) < args.length) configFile = args[++i];
else usage("Option '-c' without restore file argument");
break;
case "-d":
case "-debug":
debug = verbose = true; // Debug implies verbose
break;
case "-download":
if ((i + 2) < args.length) {
config();
boolean ok = download(args[++i], args[++i]);
System.exit(ok ? 0 : 1);
} else usage("Option '-download' requires two parameters (URL and file)");
break;
case "-dryrun":
dryRun = true;
noRmOnExit = true; // Not running, so don't delete files
reportHtml = reportYaml = false;
break;
case "-extractsource":
extractSource = true;
break;
case "-h":
case "-help":
case "--help":
usage(null);
break;
case "-i":
case "-info":
// Checkpoint info
if ((i + 1) < args.length) chekcpointRestoreFile = args[++i];
else usage("Option '-i' without checkpoint file argument");
bdsAction = BdsAction.INFO_CHECKPOINT;
break;
case "-l":
case "-log":
log = true;
break;
case "-nochp":
noCheckpoint = true;
break;
case "-noreport":
reportHtml = reportYaml = false;
break;
case "-noreporthtml":
reportHtml = false;
break;
case "-noreportyaml":
reportYaml = false;
break;
case "-normonexit":
noRmOnExit = true;
break;
case "-pid":
// PID file
if ((i + 1) < args.length) pidFile = args[++i];
else usage("Option '-pid' without file argument");
break;
case "-q":
case "-queue":
// Queue name
if ((i + 1) < args.length) queue = args[++i];
else usage("Option '-queue' without file argument");
break;
case "-quiet":
verbose = false;
debug = false;
quiet = true;
break;
case "-r":
case "-restore":
// Checkpoint restore
if ((i + 1) < args.length) chekcpointRestoreFile = args[++i];
else usage("Option '-r' without checkpoint file argument");
bdsAction = BdsAction.RUN_CHECKPOINT;
break;
case "-reporthtml":
reportHtml = true;
break;
case "-reportname":
if ((i + 1) < args.length) reportFileName = args[++i];
else usage("Option '-reportName' without name argument");
break;
case "-reportyaml":
case "-yaml":
reportYaml = true;
break;
case "-s":
case "-system":
// System type
if ((i + 1) < args.length) system = args[++i];
else usage("Option '-system' without file argument");
break;
case "-t":
case "-test":
bdsAction = BdsAction.TEST;
break;
case "-upload":
if ((i + 2) < args.length) {
config();
boolean ok = upload(args[++i], args[++i]);
System.exit(ok ? 0 : 1);
} else usage("Option '-upload' requires two parameters (file and URL)");
break;
case "-v":
case "-verbose":
verbose = true;
break;
case "-version":
System.out.println(VERSION);
System.exit(0);
break;
case "-y":
case "-retry":
// Number of retries
if ((i + 1) < args.length) taskFailCount = Gpr.parseIntSafe(args[++i]);
else usage("Option '-t' without number argument");
break;
default:
usage("Unknown command line option " + arg);
}
} else if (programFileName == null) programFileName = arg; // Get program file name
}
// Sanity checks
if (checkPidRegex) {
// OK: Nothing to chek
} else if ((programFileName == null) && (chekcpointRestoreFile == null)) {
// No file name => Error
usage("Missing program file name.");
}
}
/** Add symbols to global scope */
void initilaizeGlobalScope() {
if (debug) log("Initialize global scope.");
// Reset Global scope
Scope.resetGlobalScope();
Scope globalScope = Scope.getGlobalScope();
// --
// Get default veluas from command line or config file
// ---
// Command line parameters override defaults
String cpusStr =
config.getString(ExpressionTask.TASK_OPTION_CPUS, "1"); // Default number of cpus: 1
long cpus = Gpr.parseIntSafe(cpusStr);
if (cpus <= 0)
throw new RuntimeException("Number of cpus must be a positive number ('" + cpusStr + "')");
long mem =
Gpr.parseMemSafe(
config.getString(
ExpressionTask.TASK_OPTION_MEM,
"-1")); // Default amount of memory: -1 (unrestricted)
String node = config.getString(ExpressionTask.TASK_OPTION_NODE, "");
if (queue == null) queue = config.getString(ExpressionTask.TASK_OPTION_QUEUE, "");
if (system == null)
system =
config.getString(
ExpressionTask.TASK_OPTION_SYSTEM, ExecutionerType.LOCAL.toString().toLowerCase());
if (taskFailCount < 0)
taskFailCount = Gpr.parseIntSafe(config.getString(ExpressionTask.TASK_OPTION_RETRY, "0"));
long oneDay = 1L * 24 * 60 * 60;
long timeout =
Gpr.parseLongSafe(config.getString(ExpressionTask.TASK_OPTION_TIMEOUT, "" + oneDay));
long wallTimeout =
Gpr.parseLongSafe(config.getString(ExpressionTask.TASK_OPTION_WALL_TIMEOUT, "" + oneDay));
// ---
// Add global symbols
// ---
globalScope.add(
new ScopeSymbol(
Scope.GLOBAL_VAR_PROGRAM_NAME,
Type.STRING,
"")); // Now is empty, but they are assigned later
globalScope.add(new ScopeSymbol(Scope.GLOBAL_VAR_PROGRAM_PATH, Type.STRING, ""));
// Task related variables: Default values
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_SYSTEM,
Type.STRING,
system)); // System type: "local", "ssh", "cluster", "aws", etc.
globalScope.add(
new ScopeSymbol(ExpressionTask.TASK_OPTION_CPUS, Type.INT, cpus)); // Default number of cpus
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_MEM,
Type.INT,
mem)); // Default amount of memory (unrestricted)
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_QUEUE, Type.STRING, queue)); // Default queue: none
globalScope.add(
new ScopeSymbol(ExpressionTask.TASK_OPTION_NODE, Type.STRING, node)); // Default node: none
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_CAN_FAIL,
Type.BOOL,
false)); // Task fail triggers checkpoint & exit (a task cannot fail)
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_ALLOW_EMPTY,
Type.BOOL,
false)); // Tasks are allowed to have empty output file/s
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_RETRY,
Type.INT,
(long) taskFailCount)); // Task fail can be re-tried (re-run) N times before considering
// failed.
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_TIMEOUT, Type.INT, timeout)); // Task default timeout
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_WALL_TIMEOUT,
Type.INT,
wallTimeout)); // Task default wall-timeout
// Number of local CPUs
// Kilo, Mega, Giga, Tera, Peta.
LinkedList<ScopeSymbol> constants = new LinkedList<ScopeSymbol>();
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_LOCAL_CPUS, Type.INT, (long) Gpr.NUM_CORES));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_K, Type.INT, 1024L));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_M, Type.INT, 1024L * 1024L));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_G, Type.INT, 1024L * 1024L * 1024L));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_T, Type.INT, 1024L * 1024L * 1024L * 1024L));
constants.add(
new ScopeSymbol(Scope.GLOBAL_VAR_P, Type.INT, 1024L * 1024L * 1024L * 1024L * 1024L));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_MINUTE, Type.INT, 60L));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_HOUR, Type.INT, (long) (60 * 60)));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_DAY, Type.INT, (long) (24 * 60 * 60)));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_WEEK, Type.INT, (long) (7 * 24 * 60 * 60)));
// Math constants
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_E, Type.REAL, Math.E));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_PI, Type.REAL, Math.PI));
// Add all constants
for (ScopeSymbol ss : constants) {
ss.setConstant(true);
globalScope.add(ss);
}
// Set "physical" path
String path;
try {
path = new File(".").getCanonicalPath();
} catch (IOException e) {
throw new RuntimeException("Cannot get cannonical path for current dir");
}
globalScope.add(new ScopeSymbol(ExpressionTask.TASK_OPTION_PHYSICAL_PATH, Type.STRING, path));
// Set all environment variables
Map<String, String> envMap = System.getenv();
for (String varName : envMap.keySet()) {
String varVal = envMap.get(varName);
globalScope.add(new ScopeSymbol(varName, Type.STRING, varVal));
}
// Command line arguments (default: empty list)
// This is properly set in 'initializeArgs()' method, but
// we have to set something now, otherwise we'll get a "variable
// not found" error at compiler time, if the program attempts
// to use 'args'.
Scope.getGlobalScope()
.add(
new ScopeSymbol(
Scope.GLOBAL_VAR_ARGS_LIST, TypeList.get(Type.STRING), new ArrayList<String>()));
}
/**
* Set command line arguments as global variables
*
* <p>How it works: - Program is executes as something like:
*
* <p>java -jar BigDataScript.jar [options] programFile.bds [programOptions]
*
* <p>- Any command line argument AFTER "programFile.bds" is considered a command line argument
* for the BigDataScript program. E.g.
*
* <p>java -jar BigDataScript.jar -v program.bds -file myFile.txt -verbose -num 7
*
* <p>So our program "program.bds" has command line options: -file myFile.txt -verbose -num 7
* (notice that "-v" is a command line option for loudScript.jar and not for "program.bds")
*
* <p>- We look for variables in ProgramUnit that match the name of these command line arguments
*
* <p>- Then we add those values to the variable initialization. Thus overriding any
* initialization values provided in the program. E.g.
*
* <p>Our program has the following variable declarations: string file = "default_file.txt" int
* num = 3 bool verbose = false
*
* <p>We execute the program: java -jar BigDataScript.jar -v program.bds -file myFile.txt -verbose
* -num 7
*
* <p>The variable declarations are replaced as follows: string file = "myFile.txt" int num = 7
* bool verbose = true
*
* <p>- Note: Only primitive types are supported (i.e.: string, bool, int & real)
*
* <p>- Note: Unmatched variables names will be silently ignored (same for variables that match,
* but are non-primitive)
*
* <p>- Note: If a variable is matched, is primitive, but cannot be converted. An error is thrown.
* E.g.: Program: int num = 1
*
* <p>Command line: java -jar BigDataScript.jar program.bds -num "hello" <- This is an error
* because 'num' is an int
*
* <p>- Note: Unprocessed arguments will be available to the program as an 'args' list
*/
void initializeArgs() {
// Set program arguments as global variables
for (int argNum = 0; argNum < programArgs.size(); argNum++) {
String arg = programArgs.get(argNum);
// Parse '-OPT' option
if (arg.equalsIgnoreCase("-h")
|| arg.equalsIgnoreCase("-help")
|| arg.equalsIgnoreCase("--help")) {
if (debug) Timer.showStdErr("Activating 'show help' mode");
showHelp = true;
} else if (arg.startsWith("-")) {
// Get variable name and value
String varName = arg.substring(1);
// Find all variable declarations that match this command line argument
for (VarDeclaration varDecl : programUnit.varDeclarations(true)) {
Type varType = varDecl.getType();
// Is is a primitive variable or a primitive list?
if (varType.isPrimitiveType() || varType.isList()) {
// Find an initialization that matches the command line argument
for (VariableInit varInit : varDecl.getVarInit())
if (varInit.getVarName().equals(varName)) { // Name matches?
int argNumOri = argNum;
boolean useVal = false;
if (varType.isList()) {
// Create a list of arguments and use them to initialize the variable (list)
ArrayList<String> vals = new ArrayList<String>();
for (int i = argNum + 1; i < programArgs.size(); i++)
if (programArgs.get(i).startsWith("-")) break;
else vals.add(programArgs.get(i));
useVal =
initializeArgs(
varType, varInit,
vals); // Found variable, try to replace or add LITERAL to this VarInit
} else if (varType.isBool()) {
String valStr = "true";
// Booleans may not have a value (just '-varName' sets them to 'true')
if (programArgs.size() > (argNum + 1)) {
// Is the next argument 'true' or 'false'? => Set argument
String boolVal = programArgs.get(argNum + 1);
if (valStr.equalsIgnoreCase("true") || valStr.equalsIgnoreCase("false"))
valStr = boolVal;
}
initializeArgs(varType, varInit, valStr);
} else {
String val =
(argNum < programArgs.size()
? programArgs.get(++argNum)
: ""); // Get one argument and use it to initialize the variable
useVal =
initializeArgs(
varType, varInit,
val); // Found variable, try to replace or add LITERAL to this VarInit
}
if (!useVal) argNum = argNumOri; // We did not use the arguments
}
}
}
}
}
// Make all unprocessed arguments available for the program (in 'args' list)
Scope.getGlobalScope()
.add(new ScopeSymbol(Scope.GLOBAL_VAR_ARGS_LIST, TypeList.get(Type.STRING), programArgs));
// Initialize program name
String programPath = programUnit.getFileName();
String progName = Gpr.baseName(programPath);
Scope.getGlobalScope()
.add(new ScopeSymbol(Scope.GLOBAL_VAR_PROGRAM_NAME, Type.STRING, progName));
Scope.getGlobalScope()
.add(new ScopeSymbol(Scope.GLOBAL_VAR_PROGRAM_PATH, Type.STRING, programPath));
}
/** Resolve include statements */
private static boolean resolveIncludes(
ParseTree tree, boolean debug, Set<String> alreadyIncluded) {
boolean changed = false;
if (tree instanceof IncludeFileContext) {
// Parent file: The one that is including the other file
File parentFile =
new File(((IncludeFileContext) tree).getStart().getInputStream().getSourceName());
// Included file name
String includedFilename = StatementInclude.includeFileName(tree.getChild(1).getText());
// Find file (look into all include paths)
File includedFile = StatementInclude.includeFile(includedFilename, parentFile);
if (includedFile == null) {
CompilerMessages.get()
.add(
tree,
parentFile,
"\n\tIncluded file not found: '"
+ includedFilename
+ "'\n\tSearch path: "
+ Config.get().getIncludePath(),
MessageType.ERROR);
return false;
}
// Already included? don't bother
String canonicalFileName = Gpr.getCanonicalFileName(includedFile);
if (alreadyIncluded.contains(canonicalFileName)) {
if (debug)
Gpr.debug(
"File already included: '"
+ includedFilename
+ "'\tCanonical path: '"
+ canonicalFileName
+ "'");
return false;
}
if (!includedFile.canRead()) {
CompilerMessages.get()
.add(
tree,
parentFile,
"\n\tCannot read included file: '" + includedFilename + "'",
MessageType.ERROR);
return false;
}
// Parse
ParseTree treeinc = createAst(includedFile, debug, alreadyIncluded);
if (treeinc == null) {
CompilerMessages.get()
.add(
tree,
parentFile,
"\n\tFatal error including file '" + includedFilename + "'",
MessageType.ERROR);
return false;
}
// Is a child always a RuleContext?
for (int i = 0; i < treeinc.getChildCount(); i++) {
((IncludeFileContext) tree).addChild((RuleContext) treeinc.getChild(i));
}
} else {
for (int i = 0; i < tree.getChildCount(); i++)
changed |= resolveIncludes(tree.getChild(i), debug, alreadyIncluded);
}
return changed;
}
/**
* Create an AST from a program (using ANTLR lexer & parser) Returns null if error Use
* 'alreadyIncluded' to keep track of from 'include' statements
*/
public static ParseTree createAst(File file, boolean debug, Set<String> alreadyIncluded) {
alreadyIncluded.add(Gpr.getCanonicalFileName(file));
String fileName = file.toString();
String filePath = fileName;
BigDataScriptLexer lexer = null;
BigDataScriptParser parser = null;
try {
filePath = file.getCanonicalPath();
// Input stream
if (!Gpr.canRead(filePath)) {
CompilerMessages.get().addError("Can't read file '" + filePath + "'");
return null;
}
// Create a CharStream that reads from standard input
ANTLRFileStream input = new ANTLRFileStream(fileName);
// ---
// Lexer: Create a lexer that feeds off of input CharStream
// ---
lexer =
new BigDataScriptLexer(input) {
@Override
public void recover(LexerNoViableAltException e) {
throw new RuntimeException(e); // Bail out
}
};
// ---
// Parser
// ---
CommonTokenStream tokens = new CommonTokenStream(lexer);
parser = new BigDataScriptParser(tokens);
// Parser error handling
parser.setErrorHandler(
new CompileErrorStrategy()); // Bail out with exception if errors in parser
parser.addErrorListener(new CompilerErrorListener()); // Catch some other error messages that
// 'CompileErrorStrategy' fails to catch
// Begin parsing at main rule
ParseTree tree = parser.programUnit();
// Error loading file?
if (tree == null) {
System.err.println("Can't parse file '" + filePath + "'");
return null;
}
// Show main nodes
if (debug) {
Timer.showStdErr("AST:");
for (int childNum = 0; childNum < tree.getChildCount(); childNum++) {
Tree child = tree.getChild(childNum);
System.err.println(
"\t\tChild " + childNum + ":\t" + child + "\tTree:'" + child.toStringTree() + "'");
}
}
// Included files
boolean resolveIncludePending = true;
while (resolveIncludePending)
resolveIncludePending = resolveIncludes(tree, debug, alreadyIncluded);
return tree;
} catch (Exception e) {
String msg = e.getMessage();
CompilerMessages.get()
.addError(
"Could not compile "
+ filePath //
+ (msg != null ? " :" + e.getMessage() : "") //
);
return null;
}
}