示例#1
0
  /**
   * The main driver for sort program. Invoke this method to submit the map/reduce job.
   *
   * @throws IOException When there is communication problems with the job tracker.
   */
  public static void main(String[] args) throws Exception {

    JobConf jobConf = new JobConf();

    jobConf.setMapperClass(IdentityMapper.class);
    jobConf.setReducerClass(IdentityReducer.class);

    jobConf.setNumReduceTasks(1);

    jobConf.setMapOutputKeySchema(SchemaUtils.fromString("key:bigint"));
    jobConf.setMapOutputValueSchema(SchemaUtils.fromString("value:bigint"));

    InputUtils.addTable(TableInfo.builder().tableName(args[0]).build(), jobConf);
    OutputUtils.addTable(TableInfo.builder().tableName(args[1]).build(), jobConf);

    Date startTime = new Date();
    System.out.println("Job started: " + startTime);

    JobClient.runJob(jobConf);

    Date end_time = new Date();
    System.out.println("Job ended: " + end_time);
    System.out.println(
        "The job took " + (end_time.getTime() - startTime.getTime()) / 1000 + " seconds.");
  }
示例#2
0
  public static void main(String[] args) throws Exception {
    String testDateYmd = "20140102";
    String extraPartitions = "";

    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
    MapreduceConfigInfo conf = parseConfig(extraPartitions);
    JobConf job = makeMapreduceJobConf(conf, sdf.parse(testDateYmd));
    if (job == null) {
      throw new Exception("Create mapreduce job failed");
    }

    Account account = new AliyunAccount("x", "x");
    Odps odps = new Odps(account);
    odps.setDefaultProject("local");
    SessionState sessionState = SessionState.get();
    sessionState.setLocalRun(true);
    sessionState.setOdps(odps);

    JobClient.runJob(job);
  }