Exemple #1
0
  /**
   * Queries the command line options for an action to perform.
   *
   * <pre>
   * <code>
   * > java -jar scalaris.jar --help
   * usage: scalaris [Options]
   *  -h,--help                                   print this message
   *  -v,--verbose                                print verbose information,
   *                                              e.g. the properties read
   *  -lh,--localhost                             gets the local host's name as
   *                                              known to Java (for debugging
   *                                              purposes)
   *  -b,--minibench <[ops]> <[tpn]> <[benchs]>   run selected mini
   *                                              benchmark(s) [1|...|18|all]
   *                                              (default: all benchmarks, 500
   *                                              operations, 10 threads per
   *                                              Scalaris node)
   *  -m,--monitor <node>                         print monitoring information
   *  -r,--read <key>                             read an item
   *  -w,--write <key> <value>                    write an item
   *     --test-and-set <key> <old> <new>         atomic test and set, i.e.
   *                                              write <key> to <new> if the
   *                                              current value is <old>
   *  -d,--delete <key> <[timeout]>               delete an item (default
   *                                              timeout: 2000ms)
   *                                              WARNING: This function can
   *                                              lead to inconsistent data
   *                                              (e.g. deleted items can
   *                                              re-appear). Also when
   *                                              re-creating an item the
   *                                              version before the delete can
   *                                              re-appear.
   *  -jmx,--jmxservice <node>                    starts a service exposing
   *                                              Scalaris monitoring values
   *                                              via JMX
   * </code>
   * </pre>
   *
   * In order to override node and cookie to use for a connection, specify the
   * <tt>scalaris.node</tt> or <tt>scalaris.cookie</tt> system properties. Their values will be used
   * instead of the values defined in the config file!
   *
   * @param args command line arguments
   */
  public static void main(final String[] args) {
    boolean verbose = false;
    final CommandLineParser parser = new GnuParser();
    CommandLine line = null;
    final Options options = getOptions();
    try {
      line = parser.parse(options, args);
    } catch (final ParseException e) {
      printException("Parsing failed", e, false);
      return; // will not be reached since printException exits
    }

    if (line.hasOption("verbose")) {
      verbose = true;
      ConnectionFactory.getInstance().printProperties();
    }

    if (line.hasOption("minibench")) {
      final String[] optionValues = line.getOptionValues("minibench");
      int nrOperations = 500;
      int threadsPerNode = 10;
      final HashSet<Integer> benchmarks = new HashSet<Integer>(10);
      boolean all = false;
      if (optionValues != null) {
        checkArguments(optionValues, 0, options, "b");
        if (optionValues.length >= 1) {
          nrOperations = Integer.parseInt(optionValues[0]);
        }
        if (optionValues.length >= 2) {
          threadsPerNode = Integer.parseInt(optionValues[1]);
        }
        if (optionValues.length >= 3) {
          for (int i = 2; i < Math.min(20, optionValues.length); ++i) {
            final String benchmarks_str = optionValues[i];
            if (benchmarks_str.equals("all")) {
              all = true;
            } else {
              benchmarks.add(Integer.parseInt(benchmarks_str));
            }
          }
        }
      } else {
        all = true;
      }
      if (all) {
        for (int i = 1; i <= 18; ++i) {
          benchmarks.add(i);
        }
      }
      Benchmark.minibench(nrOperations, threadsPerNode, benchmarks);
    } else if (line.hasOption("r")) { // read
      final String key = line.getOptionValue("read");
      checkArguments(key, options, "r");
      try {
        final TransactionSingleOp sc = new TransactionSingleOp();
        final String value = sc.read(key).value().toString();
        System.out.println("read(" + key + ") == " + value);
      } catch (final ConnectionException e) {
        printException("read failed with connection error", e, verbose);
      } catch (final NotFoundException e) {
        printException("read failed with not found", e, verbose);
      } catch (final UnknownException e) {
        printException("read failed with unknown", e, verbose);
      }
    } else if (line.hasOption("w")) { // write
      final String[] optionValues = line.getOptionValues("write");
      checkArguments(optionValues, 2, options, "w");
      final String key = optionValues[0];
      final String value = optionValues[1];
      try {
        final TransactionSingleOp sc = new TransactionSingleOp();
        sc.write(key, value);
        System.out.println("write(" + key + ", " + value + "): ok");
      } catch (final ConnectionException e) {
        printException("write failed with connection error", e, verbose);
      } catch (final AbortException e) {
        printException("write failed with abort", e, verbose);
      } catch (final UnknownException e) {
        printException("write failed with unknown", e, verbose);
      }
    } else if (line.hasOption("test-and-set")) { // test_and_set
      final String[] optionValues = line.getOptionValues("test-and-set");
      checkArguments(optionValues, 3, options, "test-and-set");
      final String key = optionValues[0];
      final String oldValue = optionValues[1];
      final String newValue = optionValues[2];
      try {
        final TransactionSingleOp sc = new TransactionSingleOp();
        sc.testAndSet(key, oldValue, newValue);
        System.out.println("testAndSet(" + key + ", " + oldValue + ", " + newValue + "): ok");
      } catch (final ConnectionException e) {
        printException("testAndSet failed with connection error", e, verbose);
      } catch (final AbortException e) {
        printException("testAndSet failed with abort", e, verbose);
      } catch (final UnknownException e) {
        printException("testAndSet failed with unknown", e, verbose);
      } catch (final NotFoundException e) {
        printException("testAndSet failed with not found", e, verbose);
      } catch (final KeyChangedException e) {
        printException(
            "testAndSet failed with key changed (current value: "
                + e.getOldValue().toString()
                + ")",
            e,
            verbose);
      }
    } else if (line.hasOption("d")) { // delete
      final String[] optionValues = line.getOptionValues("delete");
      checkArguments(optionValues, 1, options, "d");
      final String key = optionValues[0];
      int timeout = 2000;
      if (optionValues.length >= 2) {
        try {
          timeout = Integer.parseInt(optionValues[1]);
        } catch (final Exception e) {
          printException(
              "Parsing failed",
              new ParseException(
                  "wrong type for timeout parameter of option d"
                      + " (parameters: <"
                      + options.getOption("d").getArgName()
                      + ">)"),
              verbose);
        }
      }
      try {
        final ReplicatedDHT sc = new ReplicatedDHT();
        final DeleteResult deleteResult = sc.delete(key, timeout);
        System.out.println(
            "delete("
                + key
                + ", "
                + timeout
                + "): "
                + deleteResult.ok
                + " ok, "
                + deleteResult.locks_set
                + " locks_set, "
                + deleteResult.undef
                + " undef");
      } catch (final ConnectionException e) {
        printException("delete failed with connection error", e, verbose);
      } catch (final TimeoutException e) {
        printException("delete failed with timeout", e, verbose);
      } catch (final UnknownException e) {
        printException("delete failed with unknown error", e, verbose);
      }
    } else if (line.hasOption("lh")) { // get local host name
      System.out.println(ConnectionFactory.getLocalhostName());
    } else if (line.hasOption("monitor")) { // print monitoring data
      final String node = line.getOptionValue("monitor");
      checkArguments(node, options, "monitor");
      try {
        final Monitor monitor = new Monitor(node);
        final GetNodeInfoResult nodeInfo = monitor.getNodeInfo();
        final GetNodePerformanceResult nodePerf = monitor.getNodePerformance();
        final GetServiceInfoResult srvInfo = monitor.getServiceInfo();
        final GetServicePerformanceResult srvPerf = monitor.getServicePerformance();

        final Double nodePerfCurLatAvg = Monitor.getCurrentPerfValue(nodePerf.latencyAvg);
        final Double nodePerfCurLatStddev = Monitor.getCurrentPerfValue(nodePerf.latencyStddev);
        final Double srvPerfCurLatAvg = Monitor.getCurrentPerfValue(srvPerf.latencyAvg);
        final Double srcPerfCurLatStddev = Monitor.getCurrentPerfValue(srvPerf.latencyStddev);

        final DecimalFormat df = new DecimalFormat("0.##");
        System.out.println("== Node Info ==");
        System.out.println("Scalaris version: " + nodeInfo.scalarisVersion);
        System.out.println("Erlang   version: " + nodeInfo.erlangVersion);
        System.out.println("# of DHT nodes  : " + nodeInfo.dhtNodes);
        System.out.println("== Service Info (from mgmt_server) ==");
        System.out.println("Total # of nodes: " + srvInfo.nodes);
        System.out.println("Total load      : " + srvInfo.totalLoad);
        System.out.println("== Node Performance ==");
        System.out.println(
            "Current latency : "
                + (nodePerfCurLatAvg == null ? "n/a" : df.format(nodePerfCurLatAvg)));
        System.out.println(
            "Current stddev  : "
                + (nodePerfCurLatStddev == null ? "n/a" : df.format(nodePerfCurLatStddev)));
        System.out.println("== Service Performance ==");
        System.out.println(
            "Current latency : "
                + (srvPerfCurLatAvg == null ? "n/a" : df.format(srvPerfCurLatAvg)));
        System.out.println(
            "Current stddev  : "
                + (srcPerfCurLatStddev == null ? "n/a" : df.format(srcPerfCurLatStddev)));
      } catch (final ConnectionException e) {
        printException("monitor failed with connection error", e, verbose);
      } catch (final UnknownException e) {
        printException("monitor failed with unknown error", e, verbose);
      }
    } else if (line.hasOption("jmx")) { // start JMX monitoring service
      final String node = line.getOptionValue("jmx");
      checkArguments(node, options, "jmx");
      startJmxService(node, verbose);
    } else {
      // print help if no other option was given
      //        if (line.hasOption("help")) {
      final HelpFormatter formatter = new HelpFormatter();
      formatter.setOptionComparator(
          new Comparator<Option>() {
            private int optionToInt(final Option option) {
              if (option.getLongOpt().equals("help")) {
                return 1;
              } else if (option.getLongOpt().equals("verbose")) {
                return 2;
              } else if (option.getLongOpt().equals("localhost")) {
                return 3;
              } else if (option.getLongOpt().equals("minibench")) {
                return 4;
              } else if (option.getLongOpt().equals("monitor")) {
                return 5;
              } else if (option.getLongOpt().equals("read")) {
                return 6;
              } else if (option.getLongOpt().equals("write")) {
                return 7;
              } else if (option.getLongOpt().equals("test-and-set")) {
                return 8;
              } else if (option.getLongOpt().equals("add-del-on-list")) {
                return 9;
              } else if (option.getLongOpt().equals("add-on-nr")) {
                return 10;
              } else if (option.getLongOpt().equals("delete")) {
                return 11;
              } else if (option.getLongOpt().equals("jmxservice")) {
                return 12;
              } else {
                return 13;
              }
            }

            public int compare(final Option arg0, final Option arg1) {
              final int arg0_i = optionToInt(arg0);
              final int arg1_i = optionToInt(arg1);
              if (arg0_i < arg1_i) {
                return -1;
              } else if (arg0_i == arg1_i) {
                return 0;
              } else {
                return 1;
              }
            }
          });
      formatter.printHelp("scalaris [Options]", getOptions());
      if (!line.hasOption("help")) {
        System.exit(1);
      }
    }
  }
Exemple #2
0
  /** Hadoop {@link Tool} implementation */
  @Override
  public int run(String[] args) throws Exception {

    Options options = new Options();

    configureOptions(options);

    CommandLineParser parser = new GnuParser();

    try {
      CommandLine commandLine = parser.parse(options, args);

      if (commandLine.hasOption(VERBOSE)) {
        Logger.getGlobal().setLevel(Level.FINEST);
      }

      if (commandLine.hasOption(QUIET)) {
        Logger.getGlobal().setLevel(Level.OFF);
      }

      String transformationLocation = commandLine.getOptionValue(TRANSFORMATION);
      String sourcemmLocation = commandLine.getOptionValue(SOURCE_PACKAGE);
      String targetmmLocation = commandLine.getOptionValue(TARGET_PACKAGE);
      String recordsLocation = commandLine.getOptionValue(RECORDS_FILE);
      String inputLocation = commandLine.getOptionValue(INPUT_MODEL);
      String outputLocation =
          commandLine.getOptionValue(
              OUTPUT_MODEL, new Path(inputLocation).suffix(".out.xmi").toString());

      int recommendedMappers = 1;
      if (commandLine.hasOption(RECOMMENDED_MAPPERS)) {
        recommendedMappers =
            ((Number) commandLine.getParsedOptionValue(RECOMMENDED_MAPPERS)).intValue();
      }

      Configuration conf = this.getConf();
      Job job = Job.getInstance(conf, JOB_NAME);

      // Configure classes
      job.setJarByClass(ATLMRMaster.class);
      job.setMapperClass(ATLMRMapper.class);
      job.setReducerClass(ATLMRReducer.class);
      job.setInputFormatClass(NLineInputFormat.class);
      job.setOutputFormatClass(SequenceFileOutputFormat.class);
      job.setMapOutputKeyClass(LongWritable.class);
      job.setMapOutputValueClass(Text.class);
      job.setNumReduceTasks(1);

      // Configure MapReduce input/outputs
      Path recordsPath = new Path(recordsLocation);
      FileInputFormat.setInputPaths(job, recordsPath);
      String timestamp = new SimpleDateFormat("yyyyMMddhhmm").format(new Date());
      String outDirName = "atlmr-out-" + timestamp + "-" + UUID.randomUUID();
      FileOutputFormat.setOutputPath(
          job, new Path(job.getWorkingDirectory().suffix(Path.SEPARATOR + outDirName).toUri()));

      // Configure records per map
      FileSystem fileSystem = FileSystem.get(recordsPath.toUri(), conf);
      InputStream inputStream = fileSystem.open(recordsPath);
      long linesPerMap =
          (long) Math.ceil((double) countLines(inputStream) / (double) recommendedMappers);
      job.getConfiguration().setLong(NLineInputFormat.LINES_PER_MAP, linesPerMap);

      // Configure ATL related inputs/outputs
      job.getConfiguration().set(TRANSFORMATION, transformationLocation);
      job.getConfiguration().set(SOURCE_PACKAGE, sourcemmLocation);
      job.getConfiguration().set(TARGET_PACKAGE, targetmmLocation);
      job.getConfiguration().set(INPUT_MODEL, inputLocation);
      job.getConfiguration().set(OUTPUT_MODEL, outputLocation);

      Logger.getGlobal().log(Level.INFO, "Starting Job execution");
      long begin = System.currentTimeMillis();
      int returnValue = job.waitForCompletion(true) ? STATUS_OK : STATUS_ERROR;
      long end = System.currentTimeMillis();
      Logger.getGlobal()
          .log(
              Level.INFO,
              MessageFormat.format(
                  "Job execution ended in {0}s with status code {1}",
                  (end - begin) / 1000, returnValue));

      return returnValue;

    } catch (ParseException e) {
      System.err.println(e.getLocalizedMessage());
      HelpFormatter formatter = new HelpFormatter();
      formatter.setOptionComparator(new OptionComarator<>());
      try {
        formatter.setWidth(Math.max(Terminal.getTerminal().getTerminalWidth(), 80));
      } catch (Throwable t) {
        // Nothing to do...
      }
      ;
      formatter.printHelp("yarn jar <this-file.jar>", options, true);
      return STATUS_ERROR;
    }
  }