Exemplo n.º 1
0
 // 使用者 时间 preurl cururl
 public void map(
     LongWritable key, Text value, OutputCollector<Text, LongWritable> out, Reporter reporter)
     throws IOException {
   String line = value.toString();
   String[] _allCols = StringUtils.splitPreserveAllTokens(line, TAB);
   if (_allCols.length < 41) {
     return;
   }
   String gmtCreated = _allCols[AuctionAuctions.OLD_STARTS];
   if (gmtCreated != null && gmtCreated.indexOf(date) >= 0) {
     String status = _allCols[AuctionAuctions.AUCTION_STATUS];
     String attr = _allCols[AuctionAuctions.FEATURES];
     String from = TaobaoPath.getValue(attr, "source", null);
     out.collect(new Text(from + "\t" + status), one);
     out.collect(new Text("cat\t" + _allCols[AuctionAuctions.CATEGORY]), one);
   }
 }
Exemplo n.º 2
0
  public int run(String[] args) throws Exception {

    if (args.length < 1) {
      args = new String[] {TaobaoPath.now()};
      System.out.println(
          "ERROR: Please Enter Date , eg. 20100507  now use default!" + TaobaoPath.now());
    }

    JobConf conf = new JobConf(getConf(), NewItemDailyFrom.class);
    conf.setJobName("NewItemDailyFrom-" + System.currentTimeMillis());

    String date = args[0];
    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(TaobaoPath.getOutput("new_item_daily_from", date))) {
      System.out.println(
          "ERROR: You has finish this job at this day :  "
              + date
              + " [ "
              + TaobaoPath.getOutput("new_item_daily_from", date)
              + " ] ");
      return -1;
    }

    conf.set("user.date", date);
    conf.setNumReduceTasks(1);
    conf.setMapperClass(MapClass.class);
    conf.setReducerClass(LongSumReducer.class);
    conf.setInputFormat(TextInputFormat.class);
    conf.setMapOutputKeyClass(Text.class);
    conf.setMapOutputValueClass(LongWritable.class);
    TextInputFormat.addInputPath(conf, TaobaoPath.hiveAuctionAuctions(date));

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(LongWritable.class);

    FileOutputFormat.setOutputPath(conf, TaobaoPath.getOutput("new_item_daily_from", date));

    JobClient.runJob(conf);

    return JobClient.SUCCESS;
  }