Example #1
0
  /**
   * @param args
   * @throws java.io.IOException
   */
  public static void main(String[] args) throws IOException {
    String input = args[0];
    if (!new File(input).exists()) {
      return;
    }
    Map<String, Long> statusMap = getMap(input);
    if (statusMap == null) return;
    Report report = Report.newReport("闲置新发布商品来源跟踪");
    if (true) {
      Table table = report.newGroupTable("all", "商品发布来源分布", "来源", "数量");

      for (String key : SOURCE) {
        table.addCol(key, sum(statusMap, key));
      }
    }

    if (true) {
      for (String key : SOURCE) {
        Table table = report.newGroupTable("lite_" + key, key + "发布来源分布", "状态", "数量");
        for (String status : ItemUtils.allStatus) {
          Long _num = statusMap.get(key + "^" + status);
          if (_num == null) {
            _num = 0L;
          }
          table.addCol(status, ItemUtils.getItemStatusName(status), String.valueOf(_num));
        }
      }
    }

    if (true) {
      Map<String, List<String[]>> today = MapUtils.map(Utils.read(input, (String[]) null));
      if (true && today.containsKey("cat")) {
        Map<String, Long> _countMap = new HashMap<String, Long>();
        for (String[] array : today.get("cat")) {
          String catId = array[0];
          if (null != Category.getCategory(catId)) {
            catId = Category.getCategory(catId).getRootId();
          }
          long value = NumberUtils.toLong(array[1]);
          Long _v = _countMap.get(catId);
          if (_v == null) {
            _v = 0L;
          }
          _v = _v + value;
          _countMap.put(catId, _v);
        }
        Table table = report.newGroupTable("cat", "今天发布商品的各个类目分布");
        for (Map.Entry<String, Long> entry : _countMap.entrySet()) {
          table.addCol(
              entry.getKey(),
              Category.getCategoryName(entry.getKey()),
              String.valueOf(entry.getValue()));
        }
        table.sort(Table.SORT_VALUE);
      }
    }
    XmlReportFactory.dump(report, new FileOutputStream(args[0] + ".xml"));
  }
Example #2
0
  @Override
  protected void doWork(String line, OutputCollector<Text, LongWritable> output)
      throws IOException {
    String[] _allCols = StringUtils.splitPreserveAllTokens(line, TAB);
    if (_allCols.length < 47) {
      return;
    }
    String queryDate = DateStringUtils.format(inputArgs[0]);
    String gmtCreated = _allCols[TcBizOrder.GMT_CREATE];
    String payTime = _allCols[TcBizOrder.PAY_TIME];
    String gmtModified = _allCols[TcBizOrder.GMT_MODIFIED];
    if (Utils.isSameDay(queryDate, gmtCreated)) {
      output.collect(Utils.mergeKey("t", "sys", "created"), ONE);
      output.collect(Utils.mergeKey("d", "sys", "created", "m", getMinutes(gmtCreated)), ONE);
      output.collect(Utils.mergeKey("d", "sys", "created", "s", gmtCreated), ONE);
    }
    if (Utils.isSameDay(queryDate, gmtModified)) {
      output.collect(Utils.mergeKey("t", "sys", "modified"), ONE);
    }

    // filter not effective order
    if (!TcBizOrder.isEffective(_allCols)) {
      return;
    }

    if (TcBizOrder.isMain(_allCols)) {
      commonMonitor("all", _allCols, output, queryDate);
      if (TcBizOrder.isB2C(_allCols)) {
        commonMonitor("b2c", _allCols, output, queryDate);
      } else {
        commonMonitor("c2c", _allCols, output, queryDate);
      }
      if (TcBizOrder.isFromTgroupon(_allCols)) {
        commonMonitor("jhs", _allCols, output, queryDate);
      }
      if (Utils.isSameDay(payTime, queryDate)) {
        long fee = TcBizOrder.getTotalFee(_allCols);
        String key = alipayTradeArea.getArea(fee);
        output.collect(Utils.mergeKey("t", "alipay_area", key), ONE);
      }
    }
    if (TcBizOrder.isDetail(_allCols)) {
      String rootCatId = Utils.getValue(_allCols[TcBizOrder.ATTRIBUTES], "realRootCat", "NULL");
      commonTotalMonitor("cat", rootCatId, _allCols, output, queryDate);
    }
  }
  protected String[] getInputPath(String[] args) {
    // String[] input = new String[]{HadoopTable.atpanel(args[0])};

    String[] input =
        new String[] {"/group/tbdev/xiaodu/suoni/user_view_muli_host_setp_two/" + args[0]};
    if (args.length > 1 && "false".equals(args[1])) {
      input =
          new String[] {"/group/tbdev/xiaodu/suoni/user_view_muli_host_setp_two_all/" + args[0]};
    }
    System.out.println("input path => " + Utils.asString(input));
    return input;
  }
Example #4
0
  /**
   * @param args
   * @throws java.io.IOException
   */
  public static void main(String[] args) throws IOException {
    String input = args[0];
    if (!new File(input).exists()) {
      System.out.println("File Not Exist ! => " + input);
      return;
    }

    String name = "";
    if (args.length > 1) {
      name = args[1];
    }

    Map<String, List<String[]>> today = MapUtils.map(Utils.read(input), CTRL_A);

    Map<String, String[]> lsMap = MapUtils.toMap(today.get("ls"));
    //  Map<String, String[]> ipvMap = MapUtils.toMap(today.get("ipv"));

    Report report = Report.newReport(name + "统计");

    Map<String, Table> marketMap = new HashMap<String, Table>();

    for (Entry<String, String[]> entry : lsMap.entrySet()) {

      String key = entry.getKey(); // eg: 针织-针织衫市场

      String[] ls = entry.getValue(); // value[0]:pv,value[1]:uv,value[2]:mid;

      String[] keys = StringUtils.split(key, "-");

      Table market = marketMap.get(keys[1]);

      if (market == null) {
        market = report.newViewTable(keys[1], keys[1]);
        market
            .addCol("活动页面")
            .addCol("PV")
            .addCol("UV")
            .addCol("UV(MID)")
            .addCol(Report.BREAK_VALUE);
        marketMap.put(keys[1], market);
      }
      market.addCol(keys[0]).addCol(ls[0]).addCol(ls[1]).addCol(ls[2]).addCol(Report.BREAK_VALUE);
    }

    XmlReportFactory.dump(report, new FileOutputStream(args[0] + ".xml"));
  }
  public int run(String[] args) throws Exception {
    if (args.length < 1) {
      args = new String[] {DateStringUtils.now()};
      System.out.println(
          "ERROR: Please Enter Date , eg. 20101010 ! now use default => " + DateStringUtils.now());
    }

    JobConf config = new JobConf(getConf(), getClass());
    config.set("user.args", Utils.asString(args));

    config.setJobName(getClass() + "-" + System.currentTimeMillis());
    config.setNumReduceTasks(100);
    config.setMapperClass(getClass());
    config.setReducerClass(getClass());
    config.setInputFormat(getInputFormat());
    config.setMapOutputKeyClass(Text.class);
    config.setMapOutputValueClass(Text.class);

    // add input paths
    for (String path : getInputPath(args)) {
      if (TextInputFormat.class.equals(getInputFormat())) {
        TextInputFormat.addInputPath(config, new Path(path));
      } else if (SequenceFileInputFormat.class.equals(getInputFormat())) {
        SequenceFileInputFormat.addInputPath(config, new Path(path));
      }
    }

    config.setOutputKeyClass(Text.class);
    config.setOutputValueClass(Text.class);

    // if output path exists then return
    FileSystem fs = FileSystem.get(config);
    Path outputPath = new Path(getOutputPath(args));
    FileOutputFormat.setOutputPath(config, outputPath);

    if (!fs.exists(outputPath)) {
      JobClient.runJob(config);
    } else {
      System.out.println("You has finished this job today ! " + outputPath);
    }

    return JobClient.SUCCESS;
  }
Example #6
0
 private static void commonTotalMonitor(
     String firstKey,
     String secondKey,
     String[] _allCols,
     OutputCollector<Text, LongWritable> output,
     String queryDate)
     throws IOException {
   String gmtCreated = _allCols[TcBizOrder.GMT_CREATE];
   String payTime = _allCols[TcBizOrder.PAY_TIME];
   String gmtModified = _allCols[TcBizOrder.GMT_MODIFIED];
   if (TcBizOrder.isDetail(_allCols)) {
     long fee = TcBizOrder.getTotalFee(_allCols);
     LongWritable totalFee = new LongWritable(fee);
     // 统计GMV 和支付宝交易
     if (Utils.isSameDay(gmtCreated, queryDate)) {
       output.collect(Utils.mergeKey("t", firstKey, "gmv", secondKey), totalFee);
       output.collect(Utils.mergeKey("t", firstKey, "gmv_num", secondKey), ONE);
     }
     if (Utils.isSameDay(payTime, queryDate)) {
       output.collect(Utils.mergeKey("t", firstKey, "alipay", secondKey), totalFee);
       output.collect(Utils.mergeKey("t", firstKey, "alipay_num", secondKey), ONE);
     }
   }
 }
Example #7
0
 @Override
 protected String[] getInputPath(String[] args) {
   String[] input = HadoopTable.orderDelta(args[0]).getInputPath();
   System.out.println("input path => " + Utils.asString(input));
   return input;
 }
Example #8
0
  /**
   * @Title: main @Description: TODO(这里用一句话描述这个方法的作用)
   *
   * @param @param args 设定文件
   * @return void 返回类型
   * @throws
   */
  public static void main(String[] args) throws IOException {
    //		String input    = "d:\\20111202";      //文件路径
    //      String mainName ="淘宝房产";      //报表名称
    String input = args[0]; // 文件路径
    String mainName = args[1]; // 报表名称
    String tableName = "fangpvuv";
    String tableTitle = "PVUV";

    String zufang = "zufang",
        ershoufang = "ershoufang",
        all = "all",
        detail = "detail",
        qmfq = "qmfq",
        alluv = "unrepeatuv";

    if (!new File(input).exists()) {
      System.out.println("File Not Exist ! => " + input);
      return;
    }
    Report report = Report.newReport(mainName + "每日PV、UV数据");
    List<String> lines = Utils.readWithCharset(input, "utf-8");
    Table table = null;
    int allPV = 0;
    if (lines != null && lines.size() > 0) {
      for (String line : lines) {
        if (line.startsWith(zufang)) {
          tableName = "zufangpvuv";
          tableTitle = "租房频道当日PV、UV";
        } else if (line.startsWith(ershoufang)) {
          tableName = "ershoufangpvuv";
          tableTitle = "二手房频道当日PV、UV";
        } else if (line.startsWith(all)) {
          tableName = "houseall";
          tableTitle = "House下当日PV、UV";
        } else if (line.startsWith(detail)) {
          tableName = "detailpvuv";
          tableTitle = "Detail当日PV、UV";
        } else if (line.startsWith(qmfq)) {
          tableName = "qmfqpvuv";
          tableTitle = "全民疯抢当日PV、UV";
        } else if (line.startsWith(alluv)) {
          tableName = "allpvuv";
          tableTitle = "整站去重UV";
        }
        String[] _cols = StringUtils.splitPreserveAllTokens(line, CTRL_A);
        if (_cols.length > 0) {
          if (tableName.equals("allpvuv")) {
            table = report.newTable(tableName, mainName + "总PV、UV(去重)");
            table.addCol("allpv", "总PV", String.valueOf(allPV));
            table.addCol("vpv", "访客UV", _cols[1]);
            table.addCol("uuv", "会员UV", _cols[2]);
            table.addCol(
                "alluv",
                "总UV",
                String.valueOf(Integer.valueOf(_cols[1]) + Integer.valueOf(_cols[2])));
            break;
          } else {
            table = report.newTable(tableName, mainName + tableTitle);
            table.addCol("vpv", "访客PV", _cols[1]);
            table.addCol("upv", "会员PV", _cols[2]);
            if (!tableName.equals("qmfqpvuv") && !tableName.equals("alluv")) {
              allPV = allPV + Integer.valueOf(_cols[1]) + Integer.valueOf(_cols[2]);
            }
            table.addCol("vuv", "访客UV", _cols[3]);
            table.addCol("uuv", "会员UV", _cols[4]);
          }
        }
        table.sort(Table.SORT_VALUE);
      }
    }
    XmlReportFactory.dump(report, new FileOutputStream(input + ".xml"));
  }
 public void configure(JobConf job) {
   this.inputArgs = Utils.asArray(job.get("user.args"));
 }
  public void reduce(
      Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter reporter)
      throws IOException {
    List<String[]> _temp = new ArrayList<String[]>();
    int count = 0;
    while (values.hasNext()) {
      Text _out = values.next();
      String[] tokens = StringUtils.splitPreserveAllTokens(_out.toString(), TAB);
      _temp.add(tokens);
      if (count++ > 100000) break;
    }

    if (count > 10000) {
      Set<String> ipSet = new HashSet<String>();
      for (int posI = 0; posI < _temp.size(); posI++) {
        String[] array = _temp.get(posI);
        if (array == null) continue;

        String mid = array[2];
        String ip = array[3];
        ipSet.add(ip);
      }
      output.collect(
          key, Utils.mergeKey(String.valueOf(ipSet.size()), StringUtils.join(ipSet, '|')));
      return;
    }

    /**
     * ·Ö×éËã·¨ FOREACH ALL_DATA IF IN INDEX THEN UPDATE INDEX AND INSERT DATA ELSE FOREACH SUB_DATA
     * MAKE INDEX AND SET FIND'S DATA AS NULL
     */
    // List<List<String[]>> dataList = new ArrayList<List<String[]>>();
    List<StringBuffer> indexList = new ArrayList<StringBuffer>();
    Set<String> ipSet = new HashSet<String>();
    boolean muliHost = false;
    for (int posI = 0; posI < _temp.size(); posI++) {
      String[] array = _temp.get(posI);
      if (array == null) continue;

      String mid = array[2];
      String ip = array[3];
      ipSet.add(ip);
      boolean hasIndex = false;
      for (int i = 0; i < indexList.size(); i++) {
        StringBuffer index = indexList.get(i);
        if (index.indexOf("|" + mid + "|") >= 0 || index.indexOf("|" + ip + "|") >= 0) {
          if (index.indexOf("|" + mid + "|") < 0) {
            index.append('|').append(mid).append('|');
          }

          if (index.indexOf("|" + ip + "|") < 0) {
            index.append('|').append(ip).append('|');
          }
          // dataList.get(i).add(array);
          hasIndex = true;
          break;
        }
      }
      if (!hasIndex) {
        StringBuffer index = new StringBuffer("|" + mid + "|" + ip + "|");
        // List<String[]> _tmp = new ArrayList<String[]>();
        // _tmp.add(array);
        for (int k = posI + 1; k < _temp.size(); k++) {
          String[] _newArray = _temp.get(k);
          if (_newArray == null) {
            continue;
          }
          String _mid = _newArray[2];
          String _ip = _newArray[3];
          if (index.indexOf("|" + _mid + "|") >= 0 || index.indexOf("|" + _ip + "|") >= 0) {
            if (index.indexOf("|" + _mid + "|") < 0) {
              index.append('|').append(_mid).append('|');
            }

            if (index.indexOf("|" + _ip + "|") < 0) {
              index.append('|').append(_ip).append('|');
            }
            // _tmp.add(_newArray);
            _temp.set(k, null);
          }
        }
        indexList.add(index);
        // dataList.add(_tmp);
      }
    }
    //        for(String[] _array : _temp){
    //            output.collect(key,Utils.mergeKey(_array[1],_array[2],_array[3],_array[4]));
    //        }

    StringBuffer allIndex = new StringBuffer();
    for (StringBuffer index : indexList) {
      allIndex.append(index).append(';');
    }
    if (allIndex.length() > 0) {
      allIndex.deleteCharAt(allIndex.length() - 1);
    }
    output.collect(
        key, Utils.mergeKey(String.valueOf(indexList.size()), StringUtils.join(ipSet, '|')));
  }