Beispiel #1
0
  public static void main(String[] args) throws Exception {

    Conf conf = new Conf(args, "-i input -o output");
    conf.setMapMemoryMB(4096);
    conf.setTaskTimeout(180000);
    conf.setMapSpeculativeExecution(false);
    conf.setMaxMapAttempts(1);

    String input = conf.get("input");
    Path out = new Path(conf.get("output"));

    Job job = new Job(conf, input, out);
    InputFormat.setNonSplitable(job);
    setupInput(job, input);

    job.setNumReduceTasks(1);
    job.setMapperClass(VocabularyMap.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(LongWritable.class);
    job.setReducerClass(VocabularyReduce.class);
    job.setGroupingComparatorClass(Text.Comparator.class);
    job.setOutputFormatClass(NullOutputFormat.class);

    job.waitForCompletion(true);
  }
Beispiel #2
0
  /**
   * @param args the command line arguments
   * @throws java.io.IOException
   */
  public static void main(String[] args) throws IOException, Exception {
    Conf conf = new Conf();
    Job job = new Job(conf, "TweetsLanguage");

    Scan scan = new Scan();

    TableMapReduceUtil.initTableMapperJob(
        "hhscyber:tweets", scan, LanguageMapper.class, null, null, job);
    job.setNumReduceTasks(0);

    TableMapReduceUtil.initTableReducerJob("hhscyber:tweets_lang", null, job);

    job.waitForCompletion(true);
  }