@Override
public void parse(String cmdResult[]) {
long swap, swapFree, mem, memFree;
swap = swapFree = mem = memFree = -1;
for (int line = 0; line < cmdResult.length; line++) {
String fields[] = cmdResult[line].replace(':', ' ').split("\\s+");
if (fields[0].equalsIgnoreCase("MemTotal")) {
mem = Gpr.parseIntSafe(fields[1]);
host.getResources().setMem(mem);
}
if (fields[0].equalsIgnoreCase("MemFree")) memFree = Gpr.parseLongSafe(fields[1]);
if (fields[0].equalsIgnoreCase("SwapTotal")) swap = Gpr.parseLongSafe(fields[1]);
if (fields[0].equalsIgnoreCase("SwapFree")) swapFree = Gpr.parseLongSafe(fields[1]);
if (debug) {
Gpr.debug("line[" + line + "]: " + cmdResult[line]);
for (int i = 0; i < fields.length; i++)
Gpr.debug("\tfields[" + i + "]: '" + fields[i] + "'");
}
}
// Calculate swap usage
if (debug) Gpr.debug("swap" + swap + " swapFree:" + swapFree);
if ((swap > 0) && (swapFree >= 0)) {
float swapUsage = ((float) (swap - swapFree)) / ((float) swap);
host.getHealth().setSwapUsage(swapUsage);
}
// Calculate memory usage
if (debug) Gpr.debug("mem:" + mem + " memFree:" + memFree);
if ((mem > 0) && (memFree >= 0)) {
float memUsage = ((float) (mem - memFree)) / ((float) mem);
host.getHealth().setMemUsage(memUsage);
}
}
/**
* Parse command line arguments
*
* @param args
*/
public void parse(String[] args) {
// Nothing? Show command line options
if (args.length <= 0) usage(null);
programArgs = new ArrayList<String>();
bdsAction = BdsAction.RUN;
for (int i = 0; i < args.length; i++) {
String arg = args[i];
if (programFileName != null) {
// Everything after 'programFileName' is an command line
// argument for the BigDataScript program
programArgs.add(arg);
} else if (isOpt(arg)) {
switch (arg.toLowerCase()) {
case "-checkpidregex":
checkPidRegex = true;
break;
case "-c":
case "-config":
// Checkpoint restore
if ((i + 1) < args.length) configFile = args[++i];
else usage("Option '-c' without restore file argument");
break;
case "-d":
case "-debug":
debug = verbose = true; // Debug implies verbose
break;
case "-download":
if ((i + 2) < args.length) {
config();
boolean ok = download(args[++i], args[++i]);
System.exit(ok ? 0 : 1);
} else usage("Option '-download' requires two parameters (URL and file)");
break;
case "-dryrun":
dryRun = true;
noRmOnExit = true; // Not running, so don't delete files
reportHtml = reportYaml = false;
break;
case "-extractsource":
extractSource = true;
break;
case "-h":
case "-help":
case "--help":
usage(null);
break;
case "-i":
case "-info":
// Checkpoint info
if ((i + 1) < args.length) chekcpointRestoreFile = args[++i];
else usage("Option '-i' without checkpoint file argument");
bdsAction = BdsAction.INFO_CHECKPOINT;
break;
case "-l":
case "-log":
log = true;
break;
case "-nochp":
noCheckpoint = true;
break;
case "-noreport":
reportHtml = reportYaml = false;
break;
case "-noreporthtml":
reportHtml = false;
break;
case "-noreportyaml":
reportYaml = false;
break;
case "-normonexit":
noRmOnExit = true;
break;
case "-pid":
// PID file
if ((i + 1) < args.length) pidFile = args[++i];
else usage("Option '-pid' without file argument");
break;
case "-q":
case "-queue":
// Queue name
if ((i + 1) < args.length) queue = args[++i];
else usage("Option '-queue' without file argument");
break;
case "-quiet":
verbose = false;
debug = false;
quiet = true;
break;
case "-r":
case "-restore":
// Checkpoint restore
if ((i + 1) < args.length) chekcpointRestoreFile = args[++i];
else usage("Option '-r' without checkpoint file argument");
bdsAction = BdsAction.RUN_CHECKPOINT;
break;
case "-reporthtml":
reportHtml = true;
break;
case "-reportname":
if ((i + 1) < args.length) reportFileName = args[++i];
else usage("Option '-reportName' without name argument");
break;
case "-reportyaml":
case "-yaml":
reportYaml = true;
break;
case "-s":
case "-system":
// System type
if ((i + 1) < args.length) system = args[++i];
else usage("Option '-system' without file argument");
break;
case "-t":
case "-test":
bdsAction = BdsAction.TEST;
break;
case "-upload":
if ((i + 2) < args.length) {
config();
boolean ok = upload(args[++i], args[++i]);
System.exit(ok ? 0 : 1);
} else usage("Option '-upload' requires two parameters (file and URL)");
break;
case "-v":
case "-verbose":
verbose = true;
break;
case "-version":
System.out.println(VERSION);
System.exit(0);
break;
case "-y":
case "-retry":
// Number of retries
if ((i + 1) < args.length) taskFailCount = Gpr.parseIntSafe(args[++i]);
else usage("Option '-t' without number argument");
break;
default:
usage("Unknown command line option " + arg);
}
} else if (programFileName == null) programFileName = arg; // Get program file name
}
// Sanity checks
if (checkPidRegex) {
// OK: Nothing to chek
} else if ((programFileName == null) && (chekcpointRestoreFile == null)) {
// No file name => Error
usage("Missing program file name.");
}
}
/** Add symbols to global scope */
void initilaizeGlobalScope() {
if (debug) log("Initialize global scope.");
// Reset Global scope
Scope.resetGlobalScope();
Scope globalScope = Scope.getGlobalScope();
// --
// Get default veluas from command line or config file
// ---
// Command line parameters override defaults
String cpusStr =
config.getString(ExpressionTask.TASK_OPTION_CPUS, "1"); // Default number of cpus: 1
long cpus = Gpr.parseIntSafe(cpusStr);
if (cpus <= 0)
throw new RuntimeException("Number of cpus must be a positive number ('" + cpusStr + "')");
long mem =
Gpr.parseMemSafe(
config.getString(
ExpressionTask.TASK_OPTION_MEM,
"-1")); // Default amount of memory: -1 (unrestricted)
String node = config.getString(ExpressionTask.TASK_OPTION_NODE, "");
if (queue == null) queue = config.getString(ExpressionTask.TASK_OPTION_QUEUE, "");
if (system == null)
system =
config.getString(
ExpressionTask.TASK_OPTION_SYSTEM, ExecutionerType.LOCAL.toString().toLowerCase());
if (taskFailCount < 0)
taskFailCount = Gpr.parseIntSafe(config.getString(ExpressionTask.TASK_OPTION_RETRY, "0"));
long oneDay = 1L * 24 * 60 * 60;
long timeout =
Gpr.parseLongSafe(config.getString(ExpressionTask.TASK_OPTION_TIMEOUT, "" + oneDay));
long wallTimeout =
Gpr.parseLongSafe(config.getString(ExpressionTask.TASK_OPTION_WALL_TIMEOUT, "" + oneDay));
// ---
// Add global symbols
// ---
globalScope.add(
new ScopeSymbol(
Scope.GLOBAL_VAR_PROGRAM_NAME,
Type.STRING,
"")); // Now is empty, but they are assigned later
globalScope.add(new ScopeSymbol(Scope.GLOBAL_VAR_PROGRAM_PATH, Type.STRING, ""));
// Task related variables: Default values
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_SYSTEM,
Type.STRING,
system)); // System type: "local", "ssh", "cluster", "aws", etc.
globalScope.add(
new ScopeSymbol(ExpressionTask.TASK_OPTION_CPUS, Type.INT, cpus)); // Default number of cpus
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_MEM,
Type.INT,
mem)); // Default amount of memory (unrestricted)
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_QUEUE, Type.STRING, queue)); // Default queue: none
globalScope.add(
new ScopeSymbol(ExpressionTask.TASK_OPTION_NODE, Type.STRING, node)); // Default node: none
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_CAN_FAIL,
Type.BOOL,
false)); // Task fail triggers checkpoint & exit (a task cannot fail)
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_ALLOW_EMPTY,
Type.BOOL,
false)); // Tasks are allowed to have empty output file/s
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_RETRY,
Type.INT,
(long) taskFailCount)); // Task fail can be re-tried (re-run) N times before considering
// failed.
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_TIMEOUT, Type.INT, timeout)); // Task default timeout
globalScope.add(
new ScopeSymbol(
ExpressionTask.TASK_OPTION_WALL_TIMEOUT,
Type.INT,
wallTimeout)); // Task default wall-timeout
// Number of local CPUs
// Kilo, Mega, Giga, Tera, Peta.
LinkedList<ScopeSymbol> constants = new LinkedList<ScopeSymbol>();
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_LOCAL_CPUS, Type.INT, (long) Gpr.NUM_CORES));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_K, Type.INT, 1024L));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_M, Type.INT, 1024L * 1024L));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_G, Type.INT, 1024L * 1024L * 1024L));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_T, Type.INT, 1024L * 1024L * 1024L * 1024L));
constants.add(
new ScopeSymbol(Scope.GLOBAL_VAR_P, Type.INT, 1024L * 1024L * 1024L * 1024L * 1024L));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_MINUTE, Type.INT, 60L));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_HOUR, Type.INT, (long) (60 * 60)));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_DAY, Type.INT, (long) (24 * 60 * 60)));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_WEEK, Type.INT, (long) (7 * 24 * 60 * 60)));
// Math constants
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_E, Type.REAL, Math.E));
constants.add(new ScopeSymbol(Scope.GLOBAL_VAR_PI, Type.REAL, Math.PI));
// Add all constants
for (ScopeSymbol ss : constants) {
ss.setConstant(true);
globalScope.add(ss);
}
// Set "physical" path
String path;
try {
path = new File(".").getCanonicalPath();
} catch (IOException e) {
throw new RuntimeException("Cannot get cannonical path for current dir");
}
globalScope.add(new ScopeSymbol(ExpressionTask.TASK_OPTION_PHYSICAL_PATH, Type.STRING, path));
// Set all environment variables
Map<String, String> envMap = System.getenv();
for (String varName : envMap.keySet()) {
String varVal = envMap.get(varName);
globalScope.add(new ScopeSymbol(varName, Type.STRING, varVal));
}
// Command line arguments (default: empty list)
// This is properly set in 'initializeArgs()' method, but
// we have to set something now, otherwise we'll get a "variable
// not found" error at compiler time, if the program attempts
// to use 'args'.
Scope.getGlobalScope()
.add(
new ScopeSymbol(
Scope.GLOBAL_VAR_ARGS_LIST, TypeList.get(Type.STRING), new ArrayList<String>()));
}