public void reduce( Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { List<String[]> _temp = new ArrayList<String[]>(); int count = 0; while (values.hasNext()) { Text _out = values.next(); String[] tokens = StringUtils.splitPreserveAllTokens(_out.toString(), TAB); _temp.add(tokens); if (count++ > 100000) break; } if (count > 10000) { Set<String> ipSet = new HashSet<String>(); for (int posI = 0; posI < _temp.size(); posI++) { String[] array = _temp.get(posI); if (array == null) continue; String mid = array[2]; String ip = array[3]; ipSet.add(ip); } output.collect( key, Utils.mergeKey(String.valueOf(ipSet.size()), StringUtils.join(ipSet, '|'))); return; } /** * ·Ö×éËã·¨ FOREACH ALL_DATA IF IN INDEX THEN UPDATE INDEX AND INSERT DATA ELSE FOREACH SUB_DATA * MAKE INDEX AND SET FIND'S DATA AS NULL */ // List<List<String[]>> dataList = new ArrayList<List<String[]>>(); List<StringBuffer> indexList = new ArrayList<StringBuffer>(); Set<String> ipSet = new HashSet<String>(); boolean muliHost = false; for (int posI = 0; posI < _temp.size(); posI++) { String[] array = _temp.get(posI); if (array == null) continue; String mid = array[2]; String ip = array[3]; ipSet.add(ip); boolean hasIndex = false; for (int i = 0; i < indexList.size(); i++) { StringBuffer index = indexList.get(i); if (index.indexOf("|" + mid + "|") >= 0 || index.indexOf("|" + ip + "|") >= 0) { if (index.indexOf("|" + mid + "|") < 0) { index.append('|').append(mid).append('|'); } if (index.indexOf("|" + ip + "|") < 0) { index.append('|').append(ip).append('|'); } // dataList.get(i).add(array); hasIndex = true; break; } } if (!hasIndex) { StringBuffer index = new StringBuffer("|" + mid + "|" + ip + "|"); // List<String[]> _tmp = new ArrayList<String[]>(); // _tmp.add(array); for (int k = posI + 1; k < _temp.size(); k++) { String[] _newArray = _temp.get(k); if (_newArray == null) { continue; } String _mid = _newArray[2]; String _ip = _newArray[3]; if (index.indexOf("|" + _mid + "|") >= 0 || index.indexOf("|" + _ip + "|") >= 0) { if (index.indexOf("|" + _mid + "|") < 0) { index.append('|').append(_mid).append('|'); } if (index.indexOf("|" + _ip + "|") < 0) { index.append('|').append(_ip).append('|'); } // _tmp.add(_newArray); _temp.set(k, null); } } indexList.add(index); // dataList.add(_tmp); } } // for(String[] _array : _temp){ // output.collect(key,Utils.mergeKey(_array[1],_array[2],_array[3],_array[4])); // } StringBuffer allIndex = new StringBuffer(); for (StringBuffer index : indexList) { allIndex.append(index).append(';'); } if (allIndex.length() > 0) { allIndex.deleteCharAt(allIndex.length() - 1); } output.collect( key, Utils.mergeKey(String.valueOf(indexList.size()), StringUtils.join(ipSet, '|'))); }