Example #1
0
  public static void main(String[] args) throws ConfigurationException, ParseException {
    Configuration config = new PropertiesConfiguration("application.properties");

    // Options
    Options options = setUpOptions();

    CommandLineParser parser = new DefaultParser();
    CommandLine cmd = parser.parse(options, args);

    // TODO Anatoli: it makes sense to add strict validation
    for (Option option : cmd.getOptions()) {
      if (StringUtils.isNotBlank(option.getValue())) {
        config.addProperty(option.getLongOpt(), option.getValue());
      }
    }
    if (cmd.hasOption('h')) {
      HelpFormatter formatter = new HelpFormatter();
      formatter.setWidth(120);
      formatter.printHelp(
          "java -Dlogback.configurationFile=logback.xml -jar whatIsOilPriceNow.jar [options]",
          options);
      return;
    }

    RockNRoller roller = new RockNRoller(config);
    roller.startMainLoop();
  }
Example #2
0
 private void printUsage(Options options, Boolean andExit) {
   String HEADER = "";
   String USAGE = "java -jar QuickFormatter.jar -i inputfile [-o outputfile] [-d delimiter] [-x]";
   String FOOTER = "\n\n";
   HelpFormatter helpFormatter = new HelpFormatter();
   helpFormatter.setWidth(100);
   helpFormatter.printHelp(USAGE, HEADER, options, FOOTER);
   if (andExit) System.exit(0);
 }
Example #3
0
    private void usage(ParseException e, boolean exit) {
      if (e != null) {
        System.err.println(e.getMessage());
        System.err.println();
        System.err.println();
      }

      PrintWriter pw = new PrintWriter(System.err, true);
      HelpFormatter formatter = new HelpFormatter();
      formatter.setWidth(80);
      formatter.printHelp(
          pw,
          formatter.getWidth(),
          getCommandUsage(),
          null,
          getOptions(false),
          formatter.getLeftPadding(),
          formatter.getDescPadding(),
          null);
      pw.flush();

      System.err.println();
      System.err.println("term filter: ");
      System.err.println(
          "    - each f[n], t[n] pair represents a term filter, f[n], t[n] don't have to both exist");
      System.err.println("    - f[n] represents a term field name, t[n] represents a term text");
      System.err.println("    - the final filter is formed by ORing all term filters together");
      System.err.println("    - maximum of " + NUM_TERM_FILTERS + " term filters are allowed");

      System.err.println();
      System.err.println("    examples:");
      System.err.println("        -f1 l.content -t1 foo" + " (term l.content=foo)");
      System.err.println(
          "        -f2 from" + "              (all terms with from as the field name)");
      System.err.println("        -t3 bar" + "               (all terms with bar as the text)");

      System.err.println();
      System.err.println();
      System.err.println("Sample command lines:");
      System.err.println(
          "zmjava com.zimbra.cs.index.LuceneViewer -a dump -i /opt/zimbra/index/0/2/index/0 -o /tmp/user1-index-dump.txt");
      System.err.println(
          "zmjava com.zimbra.cs.index.LuceneViewer -a dump -v -f1 l.content -t1 jay -f2 subject -t2 howdy -i /opt/zimbra/index/0/2/index/0 -o /tmp/user1-index-dump.txt");
      System.err.println(
          "zmjava com.zimbra.cs.index.LuceneViewer -a dump -f1 from [email protected] -i /opt/zimbra/index/0/2/index/0 -o /tmp/user1-index-dump.txt");

      if (exit) {
        System.exit(1);
      }
    }
Example #4
0
  public static CommandLine parseCmdLine(
      final String appName, String[] args, Options options, CommandLineParser parser) {
    HelpFormatter hf = new HelpFormatter();
    hf.setWidth(110);
    CommandLine commandLine = null;
    try {
      commandLine = parser.parse(options, args);
      if (commandLine.hasOption('h')) {
        hf.printHelp(appName, options, true);
        return null;
      }
    } catch (ParseException e) {
      hf.printHelp(appName, options, true);
    }

    return commandLine;
  }
  /** Runs this tool. */
  @SuppressWarnings({"static-access"})
  public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(
        OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INDEX));
    options.addOption(
        OptionBuilder.withArgName("path")
            .hasArg()
            .withDescription("output path")
            .create(COLLECTION));

    CommandLine cmdline = null;
    CommandLineParser parser = new GnuParser();

    try {
      cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
      System.err.println("Error parsing command line: " + exp.getMessage());
      System.exit(-1);
    }

    if (!cmdline.hasOption(INDEX) || !cmdline.hasOption(COLLECTION)) {
      System.out.println("args: " + Arrays.toString(args));
      HelpFormatter formatter = new HelpFormatter();
      formatter.setWidth(120);
      formatter.printHelp(LookupPostingsCompressed.class.getName(), options);
      ToolRunner.printGenericCommandUsage(System.out);
      System.exit(-1);
    }

    String indexPath = cmdline.getOptionValue(INDEX);
    String collectionPath = cmdline.getOptionValue(COLLECTION);

    if (collectionPath.endsWith(".gz")) {
      System.out.println("gzipped collection is not seekable: use compressed version!");
      System.exit(-1);
    }

    Configuration config = new Configuration();
    FileSystem fs = FileSystem.get(config);
    MapFile.Reader reader = new MapFile.Reader(new Path(indexPath + "/part-r-00000"), config);

    FSDataInputStream collection = fs.open(new Path(collectionPath));
    BufferedReader d = new BufferedReader(new InputStreamReader(collection));

    Text key = new Text();
    ArrayListWritable<PairOfInts> postings;
    BytesWritable bytesValue = new BytesWritable();

    System.out.println("Looking up postings for the term \"starcross'd\"");
    key.set("starcross'd");

    reader.get(key, bytesValue);
    postings = deserializePosting(bytesValue);

    // ArrayListWritable<PairOfVInts> postings = value;
    for (PairOfInts pair : postings) {
      System.out.println(pair);
      collection.seek(pair.getLeftElement());
      System.out.println(d.readLine());
    }

    bytesValue = new BytesWritable();
    key.set("gold");
    reader.get(key, bytesValue);
    postings = deserializePosting(bytesValue);
    System.out.println(
        "Complete postings list for 'gold': (" + postings.size() + ", " + postings + ")");

    Int2IntFrequencyDistribution goldHist = new Int2IntFrequencyDistributionEntry();
    // postings = value;
    for (PairOfInts pair : postings) {
      goldHist.increment(pair.getRightElement());
    }

    System.out.println("histogram of tf values for gold");
    for (PairOfInts pair : goldHist) {
      System.out.println(pair.getLeftElement() + "\t" + pair.getRightElement());
    }

    bytesValue = new BytesWritable();
    key.set("silver");
    reader.get(key, bytesValue);
    postings = deserializePosting(bytesValue);
    System.out.println(
        "Complete postings list for 'silver': (" + postings.size() + ", " + postings + ")");

    Int2IntFrequencyDistribution silverHist = new Int2IntFrequencyDistributionEntry();
    // postings = value;
    for (PairOfInts pair : postings) {
      silverHist.increment(pair.getRightElement());
    }

    System.out.println("histogram of tf values for silver");
    for (PairOfInts pair : silverHist) {
      System.out.println(pair.getLeftElement() + "\t" + pair.getRightElement());
    }

    bytesValue = new BytesWritable();
    key.set("bronze");
    Writable w = reader.get(key, bytesValue);

    if (w == null) {
      System.out.println("the term bronze does not appear in the collection");
    }

    collection.close();
    reader.close();

    return 0;
  }
Example #6
0
 public static void printCommandLineHelp(final String appName, final Options options) {
   HelpFormatter hf = new HelpFormatter();
   hf.setWidth(110);
   hf.printHelp(appName, options, true);
 }
  @SuppressWarnings("static-access")
  @Override
  public int run(String[] args) throws Exception {
    Options options = new Options();
    options.addOption(
        OptionBuilder.withArgName("path").hasArg().withDescription("bz2 input path").create(INPUT));
    options.addOption(
        OptionBuilder.withArgName("path").hasArg().withDescription("output path").create(OUTPUT));
    options.addOption(
        OptionBuilder.withArgName("integer")
            .hasArg()
            .withDescription("number of samples")
            .create(nSamplesOption));

    CommandLine cmdline;
    CommandLineParser parser = new GnuParser();
    try {
      cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
      System.err.println("Error parsing command line: " + exp.getMessage());
      return -1;
    }

    if (!cmdline.hasOption(INPUT)
        || !cmdline.hasOption(OUTPUT)
        || !cmdline.hasOption(nSamplesOption)) {
      HelpFormatter formatter = new HelpFormatter();
      formatter.setWidth(120);
      formatter.printHelp(this.getClass().getName(), options);
      ToolRunner.printGenericCommandUsage(System.out);
      return -1;
    }

    String inputPath = cmdline.getOptionValue(INPUT);
    String outputPath = cmdline.getOptionValue(OUTPUT);
    String nSamplesIn = cmdline.getOptionValue(nSamplesOption);
    int reduceTasks = 1;

    LOG.info("Tool name: " + this.getClass().getName());
    LOG.info(" - bz2 file: " + inputPath);
    LOG.info(" - output file: " + outputPath);

    JobConf conf = new JobConf(getConf(), DedupCLIRMHPairs.class);
    conf.setJobName(
        String.format("DedupSentencePairs[%s: %s, %s: %s]", INPUT, inputPath, OUTPUT, outputPath));

    conf.setNumMapTasks(4);
    conf.setNumReduceTasks(reduceTasks);

    conf.setInt("nSamples", Integer.parseInt(nSamplesIn));

    FileInputFormat.setInputPaths(conf, new Path(inputPath));
    FileOutputFormat.setOutputPath(conf, new Path(outputPath));

    // Set heap space - using old API
    conf.set("mapred.job.map.memory.mb", "2048");
    conf.set("mapred.map.child.java.opts", "-Xmx2048m");
    conf.set("mapred.job.reduce.memory.mb", "4096");
    conf.set("mapred.reduce.child.java.opts", "-Xmx4096m");

    conf.setMapperClass(DedupMapper.class);
    conf.setReducerClass(DedupReducer.class);

    conf.setInputFormat(SequenceFileInputFormat.class);
    conf.setOutputFormat(MapFileOutputFormat.class);

    conf.setMapOutputKeyClass(PairOfInts.class);
    conf.setMapOutputValueClass(IntWritable.class);

    conf.setOutputKeyClass(PairOfInts.class);
    conf.setOutputValueClass(IntWritable.class);

    // Delete the output directory if it exists already.
    Path outputDir = new Path(outputPath);
    FileSystem.get(conf).delete(outputDir, true);

    JobClient.runJob(conf);

    return 0;
  }
Example #8
0
 private static void printUsage(@NotNull CliWrapper cliOptions) {
   System.out.println(HEADER);
   HelpFormatter helpFormatter = new HelpFormatter();
   helpFormatter.setWidth(WIDTH);
   helpFormatter.printHelp(SPUTNIK, cliOptions.getOptions(), true);
 }
 /**
  * Prints a usage help message.
  *
  * @param options command-line options
  */
 private void printHelpMessage(Options options) {
   HelpFormatter formatter = new HelpFormatter();
   formatter.setWidth(HELP_FORMATTER_WIDTH);
   formatter.setArgName("Test");
   formatter.printHelp(getApplicationName(), options, true);
 }
  /** Runs this tool. */
  @SuppressWarnings({"static-access"})
  public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(new Option(RANGE, "use range partitioner"));

    options.addOption(
        OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INPUT));
    options.addOption(
        OptionBuilder.withArgName("path").hasArg().withDescription("output path").create(OUTPUT));
    options.addOption(
        OptionBuilder.withArgName("num")
            .hasArg()
            .withDescription("number of nodes")
            .create(NUM_NODES));
    options.addOption(
        OptionBuilder.withArgName("num")
            .hasArg()
            .withDescription("number of partitions")
            .create(NUM_PARTITIONS));

    CommandLine cmdline;
    CommandLineParser parser = new GnuParser();

    try {
      cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
      System.err.println("Error parsing command line: " + exp.getMessage());
      return -1;
    }

    if (!cmdline.hasOption(INPUT)
        || !cmdline.hasOption(OUTPUT)
        || !cmdline.hasOption(NUM_NODES)
        || !cmdline.hasOption(NUM_PARTITIONS)) {
      System.out.println("args: " + Arrays.toString(args));
      HelpFormatter formatter = new HelpFormatter();
      formatter.setWidth(120);
      formatter.printHelp(this.getClass().getName(), options);
      ToolRunner.printGenericCommandUsage(System.out);
      return -1;
    }

    String inPath = cmdline.getOptionValue(INPUT);
    String outPath = cmdline.getOptionValue(OUTPUT);
    int nodeCount = Integer.parseInt(cmdline.getOptionValue(NUM_NODES));
    int numParts = Integer.parseInt(cmdline.getOptionValue(NUM_PARTITIONS));
    boolean useRange = cmdline.hasOption(RANGE);

    LOG.info("Tool name: " + PartitionGraph.class.getSimpleName());
    LOG.info(" - input dir: " + inPath);
    LOG.info(" - output dir: " + outPath);
    LOG.info(" - num partitions: " + numParts);
    LOG.info(" - node cnt: " + nodeCount);
    LOG.info(" - use range partitioner: " + useRange);

    Configuration conf = getConf();
    conf.setInt("NodeCount", nodeCount);

    Job job = Job.getInstance(conf);
    job.setJobName(PartitionGraph.class.getSimpleName() + ":" + inPath);
    job.setJarByClass(PartitionGraph.class);

    job.setNumReduceTasks(numParts);

    FileInputFormat.setInputPaths(job, new Path(inPath));
    FileOutputFormat.setOutputPath(job, new Path(outPath));

    job.setInputFormatClass(NonSplitableSequenceFileInputFormat.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);

    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(PageRankNode.class);

    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(PageRankNode.class);

    FileSystem.get(conf).delete(new Path(outPath), true);

    job.waitForCompletion(true);

    return 0;
  }
Example #11
0
  @SuppressWarnings("static-access")
  @Override
  public int run(String[] args) throws Exception {
    Options options = new Options();
    options.addOption(
        OptionBuilder.withArgName("path")
            .hasArg()
            .withDescription("output path")
            .create(matchOutput));
    options.addOption(
        OptionBuilder.withArgName("path")
            .hasArg()
            .withDescription("output path")
            .create(nomatchOutput));
    options.addOption(
        OptionBuilder.withArgName("integer")
            .hasArg()
            .withDescription("number of samples")
            .create(nSamplesOption));

    CommandLine cmdline;
    CommandLineParser parser = new GnuParser();
    try {
      cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
      System.err.println("Error parsing command line: " + exp.getMessage());
      return -1;
    }

    if (!cmdline.hasOption(matchOutput)
        || !cmdline.hasOption(nomatchOutput)
        || !cmdline.hasOption(nSamplesOption)) {
      HelpFormatter formatter = new HelpFormatter();
      formatter.setWidth(120);
      formatter.printHelp(this.getClass().getName(), options);
      ToolRunner.printGenericCommandUsage(System.out);
      return -1;
    }

    String matchOutputPath = cmdline.getOptionValue(matchOutput);
    String nomatchOutputPath = cmdline.getOptionValue(nomatchOutput);
    String nSamplesIn = cmdline.getOptionValue(nSamplesOption);

    LOG.info("Tool name: " + this.getClass().getName());
    // LOG.info(" - input file: " + inputPath);
    // LOG.info(" - output file: " + outputPath);

    JobConf conf = new JobConf(getConf(), JaccardCompare.class);
    conf.setJobName(String.format("JaccardCompare"));

    //        FileInputFormat.setInputPaths(conf, new Path(inputPath));
    //        FileOutputFormat.setOutputPath(conf, new Path(outputPath));

    int nSentences = 1000;
    int nSamples = Integer.parseInt(nSamplesIn);
    try {

      File matchFile = new File(matchOutputPath);
      File nomatchFile = new File(nomatchOutputPath);
      FileOutputStream fosM = null, fosNM = null;
      BufferedWriter dosM = null, dosNM = null;

      fosM = new FileOutputStream(matchFile);
      fosNM = new FileOutputStream(nomatchFile);
      dosM = new BufferedWriter(new OutputStreamWriter(fosM));
      dosNM = new BufferedWriter(new OutputStreamWriter(fosNM));

      MapFile.Reader id2sentenceReader =
          new MapFile.Reader(new Path("id2sentence.map/part-00000"), conf);
      HashMap<Integer, ArrayListWritable<Text>> id2sentence =
          new HashMap<Integer, ArrayListWritable<Text>>();
      IntWritable key = new IntWritable();
      ArrayListWritable<Text> val = new ArrayListWritable<Text>();
      while (id2sentenceReader.next(key, val)) {
        id2sentence.put(key.get(), val);
        val = new ArrayListWritable<Text>();
      }

      MapFile.Reader sentence2translationReader =
          new MapFile.Reader(new Path("sentence2translation.map/part-00000"), conf);
      HashMap<Integer, ArrayListOfIntsWritable> sentence2translation =
          new HashMap<Integer, ArrayListOfIntsWritable>();
      IntWritable key2 = new IntWritable();
      ArrayListOfIntsWritable val2 = new ArrayListOfIntsWritable();
      while (sentence2translationReader.next(key2, val2)) {
        sentence2translation.put(key2.get(), val2);
        val2 = new ArrayListOfIntsWritable();
      }

      MapFile.Reader sentencematchReader =
          new MapFile.Reader(new Path("sentencematchpairs.map/part-00000"), conf);
      HashSet<PairOfInts> sentencematchpairs = new HashSet<PairOfInts>();
      PairOfInts key3 = new PairOfInts();
      IntWritable val3 = new IntWritable();
      while (sentencematchReader.next(key3, val3)) {
        sentencematchpairs.add(key3);
        key3 = new PairOfInts();
      }

      System.out.println("Done reading");
      PairOfInts p = new PairOfInts();
      IntWritable match;
      IntWritable eLineNum = new IntWritable();
      IntWritable eLineId = new IntWritable();
      ArrayListWritable<Text> eSentence = new ArrayListWritable<Text>();
      for (int i = 0; i < nSentences; i++) {
        if (i % 100 == 0) System.out.println("eLine " + i);
        // eLineNum.set(2*i);
        ArrayListOfIntsWritable transIdList = sentence2translation.get(2 * i);
        // ArrayListOfIntsWritable transIdList = new ArrayListOfIntsWritable();
        // sentence2translationReader.get(eLineNum, transIdList);
        // System.out.println("transIdList " + transIdList);
        for (int j = 0; j < nSentences; j++) {
          // System.out.println("fLine " + j);
          ArrayListWritable<Text> fSentence = id2sentence.get((2 * j + 1) * nSamples);
          // ArrayListWritable<Text> fSentence = new ArrayListWritable<Text>();
          // IntWritable fLineId = new IntWritable();
          // fLineId.set((2*j+1)*nSamples);
          // id2sentenceReader.get(fLineId, fSentence);
          // System.out.println("fLineId " + (2*j+1)*nSamples + " FSentence " + fSentence);
          float jsimMax = -1.0f;
          float jsimAvg = 0.0f;
          for (int id : transIdList) {

            eSentence = id2sentence.get(id);
            // eLineId.set(id);
            // id2sentenceReader.get(eLineId, eSentence);
            float jsim = JaccardSim.jaccardSim(eSentence, fSentence);
            // System.out.println("\teSentence " + eSentence + " " + jsim);
            jsimAvg += jsim;
            if (jsim > jsimMax) {
              jsimMax = jsim;
            }
          }
          jsimAvg = jsimAvg / transIdList.size();
          if (2 * i < 2 * j + 1) {
            p.set(2 * i, 2 * j + 1);
          } else {
            p.set(2 * j + 1, 2 * i);
          }
          // match = new IntWritable();
          // match = (IntWritable) sentencematchReader.get(p, match);
          // if(match != null){
          if (sentencematchpairs.contains(p)) {
            if (jsimMax < .5) {
              System.out.println("Low match: ");
              System.out.println("\teSentence: " + i + " " + eSentence);
              System.out.println("\tfSentence: " + j + " " + fSentence);
            }
            // System.out.println("match");
            dosM.write(Float.toString(jsimMax));
            // dosM.write(Float.toString(jsimAvg));
            dosM.write("\n");
          } else {
            // System.out.println("no match");
            dosNM.write(Float.toString(jsimMax));
            // dosNM.write(Float.toString(jsimAvg));
            dosNM.write("\n");
          }
        }
      }
      sentencematchReader.close();
      sentence2translationReader.close();
      id2sentenceReader.close();
      dosM.close();
      dosNM.close();

    } catch (IOException e2) {
      // TODO Auto-generated catch block
      e2.printStackTrace();
    }

    // Delete the output directory if it exists already.
    //        Path outputDir = new Path(outputPath);
    // FileSystem.get(conf).delete(outputDir, true);

    // JobClient.runJob(conf);

    return 0;
  }
Example #12
0
  /** Runs this tool. */
  @SuppressWarnings({"static-access"})
  public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(new Option(COMBINER, "use combiner"));
    options.addOption(new Option(INMAPPER_COMBINER, "user in-mapper combiner"));
    options.addOption(new Option(RANGE, "use range partitioner"));

    options.addOption(
        OptionBuilder.withArgName("path").hasArg().withDescription("base path").create(BASE));
    options.addOption(
        OptionBuilder.withArgName("num").hasArg().withDescription("start iteration").create(START));
    options.addOption(
        OptionBuilder.withArgName("num").hasArg().withDescription("end iteration").create(END));
    options.addOption(
        OptionBuilder.withArgName("num")
            .hasArg()
            .withDescription("number of nodes")
            .create(NUM_NODES));

    CommandLine cmdline;
    CommandLineParser parser = new GnuParser();

    try {
      cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
      System.err.println("Error parsing command line: " + exp.getMessage());
      return -1;
    }

    if (!cmdline.hasOption(BASE)
        || !cmdline.hasOption(START)
        || !cmdline.hasOption(END)
        || !cmdline.hasOption(NUM_NODES)) {
      System.out.println("args: " + Arrays.toString(args));
      HelpFormatter formatter = new HelpFormatter();
      formatter.setWidth(120);
      formatter.printHelp(this.getClass().getName(), options);
      ToolRunner.printGenericCommandUsage(System.out);
      return -1;
    }

    String basePath = cmdline.getOptionValue(BASE);
    int n = Integer.parseInt(cmdline.getOptionValue(NUM_NODES));
    int s = Integer.parseInt(cmdline.getOptionValue(START));
    int e = Integer.parseInt(cmdline.getOptionValue(END));
    boolean useCombiner = cmdline.hasOption(COMBINER);
    boolean useInmapCombiner = cmdline.hasOption(INMAPPER_COMBINER);
    boolean useRange = cmdline.hasOption(RANGE);

    LOG.info("Tool name: RunPageRank");
    LOG.info(" - base path: " + basePath);
    LOG.info(" - num nodes: " + n);
    LOG.info(" - start iteration: " + s);
    LOG.info(" - end iteration: " + e);
    LOG.info(" - use combiner: " + useCombiner);
    LOG.info(" - use in-mapper combiner: " + useInmapCombiner);
    LOG.info(" - user range partitioner: " + useRange);

    // Iterate PageRank.
    for (int i = s; i < e; i++) {
      iteratePageRank(i, i + 1, basePath, n, useCombiner, useInmapCombiner);
    }

    return 0;
  }
 public void usage(String msg) {
   HelpFormatter helpFormatter = new HelpFormatter();
   helpFormatter.setWidth(80);
   helpFormatter.printHelp(syntax, msg + header, options, footer);
 }
Example #14
0
  /** Hadoop {@link Tool} implementation */
  @Override
  public int run(String[] args) throws Exception {

    Options options = new Options();

    configureOptions(options);

    CommandLineParser parser = new GnuParser();

    try {
      CommandLine commandLine = parser.parse(options, args);

      if (commandLine.hasOption(VERBOSE)) {
        Logger.getGlobal().setLevel(Level.FINEST);
      }

      if (commandLine.hasOption(QUIET)) {
        Logger.getGlobal().setLevel(Level.OFF);
      }

      String transformationLocation = commandLine.getOptionValue(TRANSFORMATION);
      String sourcemmLocation = commandLine.getOptionValue(SOURCE_PACKAGE);
      String targetmmLocation = commandLine.getOptionValue(TARGET_PACKAGE);
      String recordsLocation = commandLine.getOptionValue(RECORDS_FILE);
      String inputLocation = commandLine.getOptionValue(INPUT_MODEL);
      String outputLocation =
          commandLine.getOptionValue(
              OUTPUT_MODEL, new Path(inputLocation).suffix(".out.xmi").toString());

      int recommendedMappers = 1;
      if (commandLine.hasOption(RECOMMENDED_MAPPERS)) {
        recommendedMappers =
            ((Number) commandLine.getParsedOptionValue(RECOMMENDED_MAPPERS)).intValue();
      }

      Configuration conf = this.getConf();
      Job job = Job.getInstance(conf, JOB_NAME);

      // Configure classes
      job.setJarByClass(ATLMRMaster.class);
      job.setMapperClass(ATLMRMapper.class);
      job.setReducerClass(ATLMRReducer.class);
      job.setInputFormatClass(NLineInputFormat.class);
      job.setOutputFormatClass(SequenceFileOutputFormat.class);
      job.setMapOutputKeyClass(LongWritable.class);
      job.setMapOutputValueClass(Text.class);
      job.setNumReduceTasks(1);

      // Configure MapReduce input/outputs
      Path recordsPath = new Path(recordsLocation);
      FileInputFormat.setInputPaths(job, recordsPath);
      String timestamp = new SimpleDateFormat("yyyyMMddhhmm").format(new Date());
      String outDirName = "atlmr-out-" + timestamp + "-" + UUID.randomUUID();
      FileOutputFormat.setOutputPath(
          job, new Path(job.getWorkingDirectory().suffix(Path.SEPARATOR + outDirName).toUri()));

      // Configure records per map
      FileSystem fileSystem = FileSystem.get(recordsPath.toUri(), conf);
      InputStream inputStream = fileSystem.open(recordsPath);
      long linesPerMap =
          (long) Math.ceil((double) countLines(inputStream) / (double) recommendedMappers);
      job.getConfiguration().setLong(NLineInputFormat.LINES_PER_MAP, linesPerMap);

      // Configure ATL related inputs/outputs
      job.getConfiguration().set(TRANSFORMATION, transformationLocation);
      job.getConfiguration().set(SOURCE_PACKAGE, sourcemmLocation);
      job.getConfiguration().set(TARGET_PACKAGE, targetmmLocation);
      job.getConfiguration().set(INPUT_MODEL, inputLocation);
      job.getConfiguration().set(OUTPUT_MODEL, outputLocation);

      Logger.getGlobal().log(Level.INFO, "Starting Job execution");
      long begin = System.currentTimeMillis();
      int returnValue = job.waitForCompletion(true) ? STATUS_OK : STATUS_ERROR;
      long end = System.currentTimeMillis();
      Logger.getGlobal()
          .log(
              Level.INFO,
              MessageFormat.format(
                  "Job execution ended in {0}s with status code {1}",
                  (end - begin) / 1000, returnValue));

      return returnValue;

    } catch (ParseException e) {
      System.err.println(e.getLocalizedMessage());
      HelpFormatter formatter = new HelpFormatter();
      formatter.setOptionComparator(new OptionComarator<>());
      try {
        formatter.setWidth(Math.max(Terminal.getTerminal().getTerminalWidth(), 80));
      } catch (Throwable t) {
        // Nothing to do...
      }
      ;
      formatter.printHelp("yarn jar <this-file.jar>", options, true);
      return STATUS_ERROR;
    }
  }
 @Override
 protected void usage(Options options) {
   HelpFormatter formatter = new HelpFormatter();
   formatter.setWidth(120);
   formatter.printHelp("config-import-export-tool.sh [file...]", options, true);
 }
Example #16
0
 private void help() {
   HelpFormatter formatter = new HelpFormatter();
   formatter.setWidth(90);
   formatter.printHelp("PerfTimes.jar", options);
   System.exit(0);
 }