@Override
protected void doWork(String line, OutputCollector<Text, LongWritable> output)
throws IOException {
String[] _allCols = StringUtils.splitPreserveAllTokens(line, TAB);
if (_allCols.length < 47) {
return;
}
String queryDate = DateStringUtils.format(inputArgs[0]);
String gmtCreated = _allCols[TcBizOrder.GMT_CREATE];
String payTime = _allCols[TcBizOrder.PAY_TIME];
String gmtModified = _allCols[TcBizOrder.GMT_MODIFIED];
if (Utils.isSameDay(queryDate, gmtCreated)) {
output.collect(Utils.mergeKey("t", "sys", "created"), ONE);
output.collect(Utils.mergeKey("d", "sys", "created", "m", getMinutes(gmtCreated)), ONE);
output.collect(Utils.mergeKey("d", "sys", "created", "s", gmtCreated), ONE);
}
if (Utils.isSameDay(queryDate, gmtModified)) {
output.collect(Utils.mergeKey("t", "sys", "modified"), ONE);
}
// filter not effective order
if (!TcBizOrder.isEffective(_allCols)) {
return;
}
if (TcBizOrder.isMain(_allCols)) {
commonMonitor("all", _allCols, output, queryDate);
if (TcBizOrder.isB2C(_allCols)) {
commonMonitor("b2c", _allCols, output, queryDate);
} else {
commonMonitor("c2c", _allCols, output, queryDate);
}
if (TcBizOrder.isFromTgroupon(_allCols)) {
commonMonitor("jhs", _allCols, output, queryDate);
}
if (Utils.isSameDay(payTime, queryDate)) {
long fee = TcBizOrder.getTotalFee(_allCols);
String key = alipayTradeArea.getArea(fee);
output.collect(Utils.mergeKey("t", "alipay_area", key), ONE);
}
}
if (TcBizOrder.isDetail(_allCols)) {
String rootCatId = Utils.getValue(_allCols[TcBizOrder.ATTRIBUTES], "realRootCat", "NULL");
commonTotalMonitor("cat", rootCatId, _allCols, output, queryDate);
}
}
private static void commonTotalMonitor(
String firstKey,
String secondKey,
String[] _allCols,
OutputCollector<Text, LongWritable> output,
String queryDate)
throws IOException {
String gmtCreated = _allCols[TcBizOrder.GMT_CREATE];
String payTime = _allCols[TcBizOrder.PAY_TIME];
String gmtModified = _allCols[TcBizOrder.GMT_MODIFIED];
if (TcBizOrder.isDetail(_allCols)) {
long fee = TcBizOrder.getTotalFee(_allCols);
LongWritable totalFee = new LongWritable(fee);
// 统计GMV 和支付宝交易
if (Utils.isSameDay(gmtCreated, queryDate)) {
output.collect(Utils.mergeKey("t", firstKey, "gmv", secondKey), totalFee);
output.collect(Utils.mergeKey("t", firstKey, "gmv_num", secondKey), ONE);
}
if (Utils.isSameDay(payTime, queryDate)) {
output.collect(Utils.mergeKey("t", firstKey, "alipay", secondKey), totalFee);
output.collect(Utils.mergeKey("t", firstKey, "alipay_num", secondKey), ONE);
}
}
}
public void reduce(
Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter reporter)
throws IOException {
List<String[]> _temp = new ArrayList<String[]>();
int count = 0;
while (values.hasNext()) {
Text _out = values.next();
String[] tokens = StringUtils.splitPreserveAllTokens(_out.toString(), TAB);
_temp.add(tokens);
if (count++ > 100000) break;
}
if (count > 10000) {
Set<String> ipSet = new HashSet<String>();
for (int posI = 0; posI < _temp.size(); posI++) {
String[] array = _temp.get(posI);
if (array == null) continue;
String mid = array[2];
String ip = array[3];
ipSet.add(ip);
}
output.collect(
key, Utils.mergeKey(String.valueOf(ipSet.size()), StringUtils.join(ipSet, '|')));
return;
}
/**
* ·Ö×éËã·¨ FOREACH ALL_DATA IF IN INDEX THEN UPDATE INDEX AND INSERT DATA ELSE FOREACH SUB_DATA
* MAKE INDEX AND SET FIND'S DATA AS NULL
*/
// List<List<String[]>> dataList = new ArrayList<List<String[]>>();
List<StringBuffer> indexList = new ArrayList<StringBuffer>();
Set<String> ipSet = new HashSet<String>();
boolean muliHost = false;
for (int posI = 0; posI < _temp.size(); posI++) {
String[] array = _temp.get(posI);
if (array == null) continue;
String mid = array[2];
String ip = array[3];
ipSet.add(ip);
boolean hasIndex = false;
for (int i = 0; i < indexList.size(); i++) {
StringBuffer index = indexList.get(i);
if (index.indexOf("|" + mid + "|") >= 0 || index.indexOf("|" + ip + "|") >= 0) {
if (index.indexOf("|" + mid + "|") < 0) {
index.append('|').append(mid).append('|');
}
if (index.indexOf("|" + ip + "|") < 0) {
index.append('|').append(ip).append('|');
}
// dataList.get(i).add(array);
hasIndex = true;
break;
}
}
if (!hasIndex) {
StringBuffer index = new StringBuffer("|" + mid + "|" + ip + "|");
// List<String[]> _tmp = new ArrayList<String[]>();
// _tmp.add(array);
for (int k = posI + 1; k < _temp.size(); k++) {
String[] _newArray = _temp.get(k);
if (_newArray == null) {
continue;
}
String _mid = _newArray[2];
String _ip = _newArray[3];
if (index.indexOf("|" + _mid + "|") >= 0 || index.indexOf("|" + _ip + "|") >= 0) {
if (index.indexOf("|" + _mid + "|") < 0) {
index.append('|').append(_mid).append('|');
}
if (index.indexOf("|" + _ip + "|") < 0) {
index.append('|').append(_ip).append('|');
}
// _tmp.add(_newArray);
_temp.set(k, null);
}
}
indexList.add(index);
// dataList.add(_tmp);
}
}
// for(String[] _array : _temp){
// output.collect(key,Utils.mergeKey(_array[1],_array[2],_array[3],_array[4]));
// }
StringBuffer allIndex = new StringBuffer();
for (StringBuffer index : indexList) {
allIndex.append(index).append(';');
}
if (allIndex.length() > 0) {
allIndex.deleteCharAt(allIndex.length() - 1);
}
output.collect(
key, Utils.mergeKey(String.valueOf(indexList.size()), StringUtils.join(ipSet, '|')));
}