示例#1
0
  @Override
  protected void doWork(String line, OutputCollector<Text, LongWritable> output)
      throws IOException {
    String[] _allCols = StringUtils.splitPreserveAllTokens(line, TAB);
    if (_allCols.length < 47) {
      return;
    }
    String queryDate = DateStringUtils.format(inputArgs[0]);
    String gmtCreated = _allCols[TcBizOrder.GMT_CREATE];
    String payTime = _allCols[TcBizOrder.PAY_TIME];
    String gmtModified = _allCols[TcBizOrder.GMT_MODIFIED];
    if (Utils.isSameDay(queryDate, gmtCreated)) {
      output.collect(Utils.mergeKey("t", "sys", "created"), ONE);
      output.collect(Utils.mergeKey("d", "sys", "created", "m", getMinutes(gmtCreated)), ONE);
      output.collect(Utils.mergeKey("d", "sys", "created", "s", gmtCreated), ONE);
    }
    if (Utils.isSameDay(queryDate, gmtModified)) {
      output.collect(Utils.mergeKey("t", "sys", "modified"), ONE);
    }

    // filter not effective order
    if (!TcBizOrder.isEffective(_allCols)) {
      return;
    }

    if (TcBizOrder.isMain(_allCols)) {
      commonMonitor("all", _allCols, output, queryDate);
      if (TcBizOrder.isB2C(_allCols)) {
        commonMonitor("b2c", _allCols, output, queryDate);
      } else {
        commonMonitor("c2c", _allCols, output, queryDate);
      }
      if (TcBizOrder.isFromTgroupon(_allCols)) {
        commonMonitor("jhs", _allCols, output, queryDate);
      }
      if (Utils.isSameDay(payTime, queryDate)) {
        long fee = TcBizOrder.getTotalFee(_allCols);
        String key = alipayTradeArea.getArea(fee);
        output.collect(Utils.mergeKey("t", "alipay_area", key), ONE);
      }
    }
    if (TcBizOrder.isDetail(_allCols)) {
      String rootCatId = Utils.getValue(_allCols[TcBizOrder.ATTRIBUTES], "realRootCat", "NULL");
      commonTotalMonitor("cat", rootCatId, _allCols, output, queryDate);
    }
  }
示例#2
0
 private static void commonTotalMonitor(
     String firstKey,
     String secondKey,
     String[] _allCols,
     OutputCollector<Text, LongWritable> output,
     String queryDate)
     throws IOException {
   String gmtCreated = _allCols[TcBizOrder.GMT_CREATE];
   String payTime = _allCols[TcBizOrder.PAY_TIME];
   String gmtModified = _allCols[TcBizOrder.GMT_MODIFIED];
   if (TcBizOrder.isDetail(_allCols)) {
     long fee = TcBizOrder.getTotalFee(_allCols);
     LongWritable totalFee = new LongWritable(fee);
     // 统计GMV 和支付宝交易
     if (Utils.isSameDay(gmtCreated, queryDate)) {
       output.collect(Utils.mergeKey("t", firstKey, "gmv", secondKey), totalFee);
       output.collect(Utils.mergeKey("t", firstKey, "gmv_num", secondKey), ONE);
     }
     if (Utils.isSameDay(payTime, queryDate)) {
       output.collect(Utils.mergeKey("t", firstKey, "alipay", secondKey), totalFee);
       output.collect(Utils.mergeKey("t", firstKey, "alipay_num", secondKey), ONE);
     }
   }
 }
  public void reduce(
      Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter reporter)
      throws IOException {
    List<String[]> _temp = new ArrayList<String[]>();
    int count = 0;
    while (values.hasNext()) {
      Text _out = values.next();
      String[] tokens = StringUtils.splitPreserveAllTokens(_out.toString(), TAB);
      _temp.add(tokens);
      if (count++ > 100000) break;
    }

    if (count > 10000) {
      Set<String> ipSet = new HashSet<String>();
      for (int posI = 0; posI < _temp.size(); posI++) {
        String[] array = _temp.get(posI);
        if (array == null) continue;

        String mid = array[2];
        String ip = array[3];
        ipSet.add(ip);
      }
      output.collect(
          key, Utils.mergeKey(String.valueOf(ipSet.size()), StringUtils.join(ipSet, '|')));
      return;
    }

    /**
     * ·Ö×éËã·¨ FOREACH ALL_DATA IF IN INDEX THEN UPDATE INDEX AND INSERT DATA ELSE FOREACH SUB_DATA
     * MAKE INDEX AND SET FIND'S DATA AS NULL
     */
    // List<List<String[]>> dataList = new ArrayList<List<String[]>>();
    List<StringBuffer> indexList = new ArrayList<StringBuffer>();
    Set<String> ipSet = new HashSet<String>();
    boolean muliHost = false;
    for (int posI = 0; posI < _temp.size(); posI++) {
      String[] array = _temp.get(posI);
      if (array == null) continue;

      String mid = array[2];
      String ip = array[3];
      ipSet.add(ip);
      boolean hasIndex = false;
      for (int i = 0; i < indexList.size(); i++) {
        StringBuffer index = indexList.get(i);
        if (index.indexOf("|" + mid + "|") >= 0 || index.indexOf("|" + ip + "|") >= 0) {
          if (index.indexOf("|" + mid + "|") < 0) {
            index.append('|').append(mid).append('|');
          }

          if (index.indexOf("|" + ip + "|") < 0) {
            index.append('|').append(ip).append('|');
          }
          // dataList.get(i).add(array);
          hasIndex = true;
          break;
        }
      }
      if (!hasIndex) {
        StringBuffer index = new StringBuffer("|" + mid + "|" + ip + "|");
        // List<String[]> _tmp = new ArrayList<String[]>();
        // _tmp.add(array);
        for (int k = posI + 1; k < _temp.size(); k++) {
          String[] _newArray = _temp.get(k);
          if (_newArray == null) {
            continue;
          }
          String _mid = _newArray[2];
          String _ip = _newArray[3];
          if (index.indexOf("|" + _mid + "|") >= 0 || index.indexOf("|" + _ip + "|") >= 0) {
            if (index.indexOf("|" + _mid + "|") < 0) {
              index.append('|').append(_mid).append('|');
            }

            if (index.indexOf("|" + _ip + "|") < 0) {
              index.append('|').append(_ip).append('|');
            }
            // _tmp.add(_newArray);
            _temp.set(k, null);
          }
        }
        indexList.add(index);
        // dataList.add(_tmp);
      }
    }
    //        for(String[] _array : _temp){
    //            output.collect(key,Utils.mergeKey(_array[1],_array[2],_array[3],_array[4]));
    //        }

    StringBuffer allIndex = new StringBuffer();
    for (StringBuffer index : indexList) {
      allIndex.append(index).append(';');
    }
    if (allIndex.length() > 0) {
      allIndex.deleteCharAt(allIndex.length() - 1);
    }
    output.collect(
        key, Utils.mergeKey(String.valueOf(indexList.size()), StringUtils.join(ipSet, '|')));
  }