/**
* Queries the command line options for an action to perform.
*
* <pre>
* <code>
* > java -jar scalaris.jar --help
* usage: scalaris [Options]
* -h,--help print this message
* -v,--verbose print verbose information,
* e.g. the properties read
* -lh,--localhost gets the local host's name as
* known to Java (for debugging
* purposes)
* -b,--minibench <[ops]> <[tpn]> <[benchs]> run selected mini
* benchmark(s) [1|...|18|all]
* (default: all benchmarks, 500
* operations, 10 threads per
* Scalaris node)
* -m,--monitor <node> print monitoring information
* -r,--read <key> read an item
* -w,--write <key> <value> write an item
* --test-and-set <key> <old> <new> atomic test and set, i.e.
* write <key> to <new> if the
* current value is <old>
* -d,--delete <key> <[timeout]> delete an item (default
* timeout: 2000ms)
* WARNING: This function can
* lead to inconsistent data
* (e.g. deleted items can
* re-appear). Also when
* re-creating an item the
* version before the delete can
* re-appear.
* -jmx,--jmxservice <node> starts a service exposing
* Scalaris monitoring values
* via JMX
* </code>
* </pre>
*
* In order to override node and cookie to use for a connection, specify the
* <tt>scalaris.node</tt> or <tt>scalaris.cookie</tt> system properties. Their values will be used
* instead of the values defined in the config file!
*
* @param args command line arguments
*/
public static void main(final String[] args) {
boolean verbose = false;
final CommandLineParser parser = new GnuParser();
CommandLine line = null;
final Options options = getOptions();
try {
line = parser.parse(options, args);
} catch (final ParseException e) {
printException("Parsing failed", e, false);
return; // will not be reached since printException exits
}
if (line.hasOption("verbose")) {
verbose = true;
ConnectionFactory.getInstance().printProperties();
}
if (line.hasOption("minibench")) {
final String[] optionValues = line.getOptionValues("minibench");
int nrOperations = 500;
int threadsPerNode = 10;
final HashSet<Integer> benchmarks = new HashSet<Integer>(10);
boolean all = false;
if (optionValues != null) {
checkArguments(optionValues, 0, options, "b");
if (optionValues.length >= 1) {
nrOperations = Integer.parseInt(optionValues[0]);
}
if (optionValues.length >= 2) {
threadsPerNode = Integer.parseInt(optionValues[1]);
}
if (optionValues.length >= 3) {
for (int i = 2; i < Math.min(20, optionValues.length); ++i) {
final String benchmarks_str = optionValues[i];
if (benchmarks_str.equals("all")) {
all = true;
} else {
benchmarks.add(Integer.parseInt(benchmarks_str));
}
}
}
} else {
all = true;
}
if (all) {
for (int i = 1; i <= 18; ++i) {
benchmarks.add(i);
}
}
Benchmark.minibench(nrOperations, threadsPerNode, benchmarks);
} else if (line.hasOption("r")) { // read
final String key = line.getOptionValue("read");
checkArguments(key, options, "r");
try {
final TransactionSingleOp sc = new TransactionSingleOp();
final String value = sc.read(key).value().toString();
System.out.println("read(" + key + ") == " + value);
} catch (final ConnectionException e) {
printException("read failed with connection error", e, verbose);
} catch (final NotFoundException e) {
printException("read failed with not found", e, verbose);
} catch (final UnknownException e) {
printException("read failed with unknown", e, verbose);
}
} else if (line.hasOption("w")) { // write
final String[] optionValues = line.getOptionValues("write");
checkArguments(optionValues, 2, options, "w");
final String key = optionValues[0];
final String value = optionValues[1];
try {
final TransactionSingleOp sc = new TransactionSingleOp();
sc.write(key, value);
System.out.println("write(" + key + ", " + value + "): ok");
} catch (final ConnectionException e) {
printException("write failed with connection error", e, verbose);
} catch (final AbortException e) {
printException("write failed with abort", e, verbose);
} catch (final UnknownException e) {
printException("write failed with unknown", e, verbose);
}
} else if (line.hasOption("test-and-set")) { // test_and_set
final String[] optionValues = line.getOptionValues("test-and-set");
checkArguments(optionValues, 3, options, "test-and-set");
final String key = optionValues[0];
final String oldValue = optionValues[1];
final String newValue = optionValues[2];
try {
final TransactionSingleOp sc = new TransactionSingleOp();
sc.testAndSet(key, oldValue, newValue);
System.out.println("testAndSet(" + key + ", " + oldValue + ", " + newValue + "): ok");
} catch (final ConnectionException e) {
printException("testAndSet failed with connection error", e, verbose);
} catch (final AbortException e) {
printException("testAndSet failed with abort", e, verbose);
} catch (final UnknownException e) {
printException("testAndSet failed with unknown", e, verbose);
} catch (final NotFoundException e) {
printException("testAndSet failed with not found", e, verbose);
} catch (final KeyChangedException e) {
printException(
"testAndSet failed with key changed (current value: "
+ e.getOldValue().toString()
+ ")",
e,
verbose);
}
} else if (line.hasOption("d")) { // delete
final String[] optionValues = line.getOptionValues("delete");
checkArguments(optionValues, 1, options, "d");
final String key = optionValues[0];
int timeout = 2000;
if (optionValues.length >= 2) {
try {
timeout = Integer.parseInt(optionValues[1]);
} catch (final Exception e) {
printException(
"Parsing failed",
new ParseException(
"wrong type for timeout parameter of option d"
+ " (parameters: <"
+ options.getOption("d").getArgName()
+ ">)"),
verbose);
}
}
try {
final ReplicatedDHT sc = new ReplicatedDHT();
final DeleteResult deleteResult = sc.delete(key, timeout);
System.out.println(
"delete("
+ key
+ ", "
+ timeout
+ "): "
+ deleteResult.ok
+ " ok, "
+ deleteResult.locks_set
+ " locks_set, "
+ deleteResult.undef
+ " undef");
} catch (final ConnectionException e) {
printException("delete failed with connection error", e, verbose);
} catch (final TimeoutException e) {
printException("delete failed with timeout", e, verbose);
} catch (final UnknownException e) {
printException("delete failed with unknown error", e, verbose);
}
} else if (line.hasOption("lh")) { // get local host name
System.out.println(ConnectionFactory.getLocalhostName());
} else if (line.hasOption("monitor")) { // print monitoring data
final String node = line.getOptionValue("monitor");
checkArguments(node, options, "monitor");
try {
final Monitor monitor = new Monitor(node);
final GetNodeInfoResult nodeInfo = monitor.getNodeInfo();
final GetNodePerformanceResult nodePerf = monitor.getNodePerformance();
final GetServiceInfoResult srvInfo = monitor.getServiceInfo();
final GetServicePerformanceResult srvPerf = monitor.getServicePerformance();
final Double nodePerfCurLatAvg = Monitor.getCurrentPerfValue(nodePerf.latencyAvg);
final Double nodePerfCurLatStddev = Monitor.getCurrentPerfValue(nodePerf.latencyStddev);
final Double srvPerfCurLatAvg = Monitor.getCurrentPerfValue(srvPerf.latencyAvg);
final Double srcPerfCurLatStddev = Monitor.getCurrentPerfValue(srvPerf.latencyStddev);
final DecimalFormat df = new DecimalFormat("0.##");
System.out.println("== Node Info ==");
System.out.println("Scalaris version: " + nodeInfo.scalarisVersion);
System.out.println("Erlang version: " + nodeInfo.erlangVersion);
System.out.println("# of DHT nodes : " + nodeInfo.dhtNodes);
System.out.println("== Service Info (from mgmt_server) ==");
System.out.println("Total # of nodes: " + srvInfo.nodes);
System.out.println("Total load : " + srvInfo.totalLoad);
System.out.println("== Node Performance ==");
System.out.println(
"Current latency : "
+ (nodePerfCurLatAvg == null ? "n/a" : df.format(nodePerfCurLatAvg)));
System.out.println(
"Current stddev : "
+ (nodePerfCurLatStddev == null ? "n/a" : df.format(nodePerfCurLatStddev)));
System.out.println("== Service Performance ==");
System.out.println(
"Current latency : "
+ (srvPerfCurLatAvg == null ? "n/a" : df.format(srvPerfCurLatAvg)));
System.out.println(
"Current stddev : "
+ (srcPerfCurLatStddev == null ? "n/a" : df.format(srcPerfCurLatStddev)));
} catch (final ConnectionException e) {
printException("monitor failed with connection error", e, verbose);
} catch (final UnknownException e) {
printException("monitor failed with unknown error", e, verbose);
}
} else if (line.hasOption("jmx")) { // start JMX monitoring service
final String node = line.getOptionValue("jmx");
checkArguments(node, options, "jmx");
startJmxService(node, verbose);
} else {
// print help if no other option was given
// if (line.hasOption("help")) {
final HelpFormatter formatter = new HelpFormatter();
formatter.setOptionComparator(
new Comparator<Option>() {
private int optionToInt(final Option option) {
if (option.getLongOpt().equals("help")) {
return 1;
} else if (option.getLongOpt().equals("verbose")) {
return 2;
} else if (option.getLongOpt().equals("localhost")) {
return 3;
} else if (option.getLongOpt().equals("minibench")) {
return 4;
} else if (option.getLongOpt().equals("monitor")) {
return 5;
} else if (option.getLongOpt().equals("read")) {
return 6;
} else if (option.getLongOpt().equals("write")) {
return 7;
} else if (option.getLongOpt().equals("test-and-set")) {
return 8;
} else if (option.getLongOpt().equals("add-del-on-list")) {
return 9;
} else if (option.getLongOpt().equals("add-on-nr")) {
return 10;
} else if (option.getLongOpt().equals("delete")) {
return 11;
} else if (option.getLongOpt().equals("jmxservice")) {
return 12;
} else {
return 13;
}
}
public int compare(final Option arg0, final Option arg1) {
final int arg0_i = optionToInt(arg0);
final int arg1_i = optionToInt(arg1);
if (arg0_i < arg1_i) {
return -1;
} else if (arg0_i == arg1_i) {
return 0;
} else {
return 1;
}
}
});
formatter.printHelp("scalaris [Options]", getOptions());
if (!line.hasOption("help")) {
System.exit(1);
}
}
}
/** Hadoop {@link Tool} implementation */
@Override
public int run(String[] args) throws Exception {
Options options = new Options();
configureOptions(options);
CommandLineParser parser = new GnuParser();
try {
CommandLine commandLine = parser.parse(options, args);
if (commandLine.hasOption(VERBOSE)) {
Logger.getGlobal().setLevel(Level.FINEST);
}
if (commandLine.hasOption(QUIET)) {
Logger.getGlobal().setLevel(Level.OFF);
}
String transformationLocation = commandLine.getOptionValue(TRANSFORMATION);
String sourcemmLocation = commandLine.getOptionValue(SOURCE_PACKAGE);
String targetmmLocation = commandLine.getOptionValue(TARGET_PACKAGE);
String recordsLocation = commandLine.getOptionValue(RECORDS_FILE);
String inputLocation = commandLine.getOptionValue(INPUT_MODEL);
String outputLocation =
commandLine.getOptionValue(
OUTPUT_MODEL, new Path(inputLocation).suffix(".out.xmi").toString());
int recommendedMappers = 1;
if (commandLine.hasOption(RECOMMENDED_MAPPERS)) {
recommendedMappers =
((Number) commandLine.getParsedOptionValue(RECOMMENDED_MAPPERS)).intValue();
}
Configuration conf = this.getConf();
Job job = Job.getInstance(conf, JOB_NAME);
// Configure classes
job.setJarByClass(ATLMRMaster.class);
job.setMapperClass(ATLMRMapper.class);
job.setReducerClass(ATLMRReducer.class);
job.setInputFormatClass(NLineInputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(Text.class);
job.setNumReduceTasks(1);
// Configure MapReduce input/outputs
Path recordsPath = new Path(recordsLocation);
FileInputFormat.setInputPaths(job, recordsPath);
String timestamp = new SimpleDateFormat("yyyyMMddhhmm").format(new Date());
String outDirName = "atlmr-out-" + timestamp + "-" + UUID.randomUUID();
FileOutputFormat.setOutputPath(
job, new Path(job.getWorkingDirectory().suffix(Path.SEPARATOR + outDirName).toUri()));
// Configure records per map
FileSystem fileSystem = FileSystem.get(recordsPath.toUri(), conf);
InputStream inputStream = fileSystem.open(recordsPath);
long linesPerMap =
(long) Math.ceil((double) countLines(inputStream) / (double) recommendedMappers);
job.getConfiguration().setLong(NLineInputFormat.LINES_PER_MAP, linesPerMap);
// Configure ATL related inputs/outputs
job.getConfiguration().set(TRANSFORMATION, transformationLocation);
job.getConfiguration().set(SOURCE_PACKAGE, sourcemmLocation);
job.getConfiguration().set(TARGET_PACKAGE, targetmmLocation);
job.getConfiguration().set(INPUT_MODEL, inputLocation);
job.getConfiguration().set(OUTPUT_MODEL, outputLocation);
Logger.getGlobal().log(Level.INFO, "Starting Job execution");
long begin = System.currentTimeMillis();
int returnValue = job.waitForCompletion(true) ? STATUS_OK : STATUS_ERROR;
long end = System.currentTimeMillis();
Logger.getGlobal()
.log(
Level.INFO,
MessageFormat.format(
"Job execution ended in {0}s with status code {1}",
(end - begin) / 1000, returnValue));
return returnValue;
} catch (ParseException e) {
System.err.println(e.getLocalizedMessage());
HelpFormatter formatter = new HelpFormatter();
formatter.setOptionComparator(new OptionComarator<>());
try {
formatter.setWidth(Math.max(Terminal.getTerminal().getTerminalWidth(), 80));
} catch (Throwable t) {
// Nothing to do...
}
;
formatter.printHelp("yarn jar <this-file.jar>", options, true);
return STATUS_ERROR;
}
}