@Test
public void groupFile() {
long startTiem = System.currentTimeMillis();
String map =
"function() { "
+ "var category; "
+ "if ( this.secretStatus = 2 ) "
+ "category = '标密状态位2'; "
+ "else "
+ "category = '标密状态位其它'; "
+ "emit(category, {secretPerson: this.secretPerson});}";
String reduce =
"function(key, values) { "
+ "var sum = 0; "
+ "values.forEach(function(doc) { "
+ "sum += 1; "
+ "}); "
+ "return {副本总数: sum};} ";
MapReduceCommand cmd =
new MapReduceCommand(bakFiles, map, reduce, null, MapReduceCommand.OutputType.INLINE, null);
MapReduceOutput out = bakFiles.mapReduce(cmd);
long endTime = System.currentTimeMillis();
System.out.println("本次操作消费 " + (endTime - startTiem) + " 毫秒");
for (DBObject o : out.results()) {
System.out.println(o.toString());
}
}
/** 使用mongodb的mapreduce分组统计</br> 日期:2014-3-13 上午11:47:08 */
@Test
public void mapreduce() {
String map =
" function() {"
+ "emit("
+ "{fileId:this.fileId,clientId:this.clientId},"
+ "{count:0}"
+ ");"
+ "}";
String reduce =
"function (key, values) {"
+ "var ss = {count:0};"
+ "values.count.forEach(val){"
+ "ss.count += = val.count;"
+ "}"
+ "return ss;"
+ "}";
MapReduceCommand cmd =
new MapReduceCommand(bakFiles, map, reduce, null, MapReduceCommand.OutputType.INLINE, null);
MapReduceOutput out = bakFiles.mapReduce(cmd);
System.out.println(out.toString());
}
public List<Double> getTimeStatsForUser(User u) {
DBObject queryObject = new BasicDBObject("authorName", u.getUsername());
String map =
"function(){"
+ "var hours = this.datePublished.getHours();"
+ "var zone;"
+ "if(hours >= 0 && hours < 4){"
+ "zone = 0;"
+ "}"
+ "else if(hours >= 4 && hours < 8){"
+ "zone = 1;"
+ "}"
+ "else if(hours >= 8 && hours < 12){"
+ "zone = 2;"
+ "}"
+ "else if(hours >= 12 && hours < 16){"
+ "zone = 3;"
+ "}"
+ "else if(hours >= 16 && hours < 20){"
+ "zone = 4;"
+ "}"
+ "else {"
+ "zone = 5;"
+ "}"
+ "emit(zone, 1);"
+ "}";
String reduce = "function(k, v){" + "return Array.sum(v);" + "}";
DBCollection collection = datastore.getCollection(Message.class);
MapReduceCommand mrc =
new MapReduceCommand(
collection, map, reduce, null, MapReduceCommand.OutputType.INLINE, queryObject);
MapReduceOutput output = collection.mapReduce(mrc);
Integer count = collection.find(queryObject).count();
List<Double> result = new ArrayList<>();
for (int i = 0; i <= 5; i++) {
result.add(0.0);
}
result.add(count.doubleValue());
output
.results()
.forEach(
(DBObject o) ->
result.set(((Double) o.get("_id")).intValue(), (Double) o.get("value")));
return result;
}
public Map<String, Double> getPlaceStatsForUser(User u) {
DBObject queryObject = new BasicDBObject("authorName", u.getUsername());
String map =
"function(){"
+ "var place = this.place;"
+ "if(place != null){"
+ "if(place === ''){"
+ "place = 'Empty';"
+ "}"
+ "else if(place !== 'Sofia' && place !== 'Plovdiv' && place !== 'Varna' && place !== 'Burgas'){"
+ "place = 'Other';"
+ "}"
+ "}"
+ "else {"
+ "place = 'Empty';"
+ "}"
+ "emit(place, 1);"
+ "}";
String reduce = "function(k, v){ return Array.sum(v); }";
DBCollection collection = datastore.getCollection(Message.class);
MapReduceCommand mrc =
new MapReduceCommand(
collection, map, reduce, null, MapReduceCommand.OutputType.INLINE, queryObject);
MapReduceOutput output = collection.mapReduce(mrc);
Integer count = collection.find(queryObject).count();
Map<String, Double> result = new HashMap<>();
result.put("Total", count.doubleValue());
result.put("Burgas", 0.0);
result.put("Varna", 0.0);
result.put("Sofia", 0.0);
result.put("Plovdiv", 0.0);
result.put("Other", 0.0);
result.put("Empty", 0.0);
output
.results()
.forEach((DBObject o) -> result.replace((String) o.get("_id"), (Double) o.get("value")));
return result;
}
/**
* 查找数据sex1={name1};sex1={name2};sex1={name3};sex2={name1};sex2={name2};sex2={name3};sex3={name1};sex3={name2};sex3={name3};
* String map = "function(){emit(this.sex,this.name);}";
* 处理成sex1={name1,name2,name3};sex2={name1,name2,name3};sex3={name1,name2,name3}; String
* reduce="function(key,values){ var ret={sex:key,names:values}; return ret;}";
*
* @param collectionName
* @param mapFunction
* @param reduceFunction
* @param targetCollection
* @param keyNameOfReduceValue
*/
public void mapReduce(
String collectionName,
String mapFunction,
String reduceFunction,
String targetCollection,
String keyNameOfReduceValue) {
try {
DBCollection collection = getCollection(collectionName);
// 执行mapReduce命令
MapReduceOutput result =
collection.mapReduce(
mapFunction, reduceFunction, targetCollection, OutputType.INLINE, null);
logger.info(
"BaseMongoDAO.mapReduce返回的原始数据值， result=" + result.getCommandResult().toString());
// 获取mapreduce的结果
String reduceResult = result.getCommandResult().get("results").toString();
logger.info("BaseMongoDAO.mapReduce返回的有效数据值， reduceResult=" + reduceResult);
// 返回结果key=要处理的属性；value=聚合后的结果
Map<String, String> mapReduceResult = new HashMap<String, String>();
// 将返回结果用json序列化格式为[{ "_id" : null , "value" : { "sex" : null , "names" : [ "test3" , "test3"
// , "test3"]}} , { "_id" : "man" , "value" : { "sex" :
// "man" , "names" : [ "name" , "test6" , "test6" , "test3" , "test3" , "test3" , "test3" ,
// "test3" , "test3" , "test3" , "test3" , "test3" ,
// "test3" , "test3" , "test3"]}}]
List<Map<String, Object>> reduceMap = JsonUtil.fromJson(reduceResult, List.class);
// 序列化后的串[{_id=null, value={sex=null, num=3.0}}, {_id=man, value={sex=man, num=15.0}},
// {_id=woman, value={sex=woman, num=5.0}}, {_id=women,
// value={name=test2, num=1.0}}]
System.out.println(reduceMap);
logger.info("BaseMongoDAO.mapReduce方法json反序列化后的值， jsonToClassReduceMap=" + reduceMap);
// 遍历mapReduce的结果
for (Map<String, Object> mapInfo : reduceMap) {
// 定义reduce出的key对应的collection中的值
String keyValue = "null";
for (Entry<String, Object> mapInfoEntry : mapInfo.entrySet()) {
// System.out.println("111map key=====" +mapInfoEntry.getKey());
// 获取key对应的collection中的值
if (mapInfoEntry.getKey().equals("_id")) {
keyValue = processNull(mapInfoEntry.getValue());
}
/**
* 这个判断主要是为了集中处理value={sex=man, num=15.0}这类键值
* 绕开对key=NULL(即value为NULL)这类键值对的处理；例如{_id=null, value={sex=null,
* num=3.0}}中的_id和sex;如果为空则不做任何处理 因为下面使用的是MAP的强转，不适宜基础类型
*/
// 如果reduce的值不为空，而且不是基础类型则继续解析map
if (mapInfoEntry.getValue() != null
&& !isPrimitiveType(mapInfoEntry.getValue().getClass().getName())) {
// System.out.println("in in 111map value=====" +mapInfoEntry.getValue());
LinkedHashMap<Object, Object> realInfoMapList =
(LinkedHashMap<Object, Object>) mapInfoEntry.getValue();
for (Entry e : realInfoMapList.entrySet()) {
if (keyNameOfReduceValue.equals(e.getKey().toString())) {
mapReduceResult.put(keyValue, e.getValue().toString());
}
// System.out.println(e.getKey());
// System.out.println(e.getValue());
}
// 值不为NULL的基础类型的处理逻辑
} else if (mapInfoEntry.getValue() != null) {
if (keyNameOfReduceValue.equals(mapInfoEntry.getKey().toString())) {
mapReduceResult.put(keyValue, mapInfoEntry.getValue().toString());
}
// 值为NULL的处理，只打印日志
} else {
logger.error("BaseMongoDAO.mapReduce方法，查到的结果为NULL，不做任何处理" + mapInfoEntry.getValue());
}
}
}
// 输出reduce处理完成后的结果集
for (Entry<String, String> mapInfo : mapReduceResult.entrySet()) {
logger.info(
"BaseMongoDAO.mapReduce方法聚合后的结果key="
+ mapInfo.getKey()
+ "======"
+ mapInfo.getValue());
}
} catch (Exception e) {
logger.error(
"BaseMongoDAO.mapReduce聚合数据时发生异常，入参collectionName="
+ collectionName
+ "; mapFunction="
+ mapFunction
+ "; reduceFunction="
+ reduceFunction
+ "; targetCollection="
+ targetCollection
+ "; keyNameOfReduceValue="
+ keyNameOfReduceValue,
e);
}
}
@Test
public void test() {
DBObject o1 =
new BasicDBObject("name", "Rhys Lee")
.append("age", 29)
.append("sex", 1)
.append("type", 6)
.append("regDate", DateUtil.getToday("yyyyMMddHHmmss"));
DBObject o2 =
new BasicDBObject("name", "Ari")
.append("age", 23)
.append("sex", 0)
.append("type", 3)
.append("regDate", DateUtil.getToday("yyyyMMddHHmmss"));
DBObject o3 =
new BasicDBObject("name", "Atrox")
.append("age", 30)
.append("sex", 1)
.append("type", 5)
.append("regDate", DateUtil.getToday("yyyyMMddHHmmss"));
DBObject o4 =
new BasicDBObject("name", "Atrox1")
.append("age", 35)
.append("sex", 1)
.append("type", 5)
.append("regDate", DateUtil.getToday("yyyyMMddHHmmss"));
DBObject o5 =
new BasicDBObject("name", "Atrox2")
.append("age", 25)
.append("sex", 1)
.append("type", 5)
.append("regDate", DateUtil.getToday("yyyyMMddHHmmss"));
DBCollection c = MongoTemplate.getDBCollection("TEST");
c.drop(); // 테스트를 위한 초기화
c.insert(Arrays.asList(o1, o2, o3, o4, o5));
String map =
"function() { "
+ "var category; "
+ "if ( this.age >= 30 ) "
+ "category = 'Big'; "
+ "else "
+ "category = 'Small'; "
+ "emit(category, {name: this.name});}";
String reduce =
"function(key, values) { "
+ "var sum = 0; "
+ "values.forEach(function(doc) { sum += 1; "
+ "}); "
+ "return {cnt : sum};} ";
MapReduceCommand cmd =
new MapReduceCommand(c, map, reduce, null, MapReduceCommand.OutputType.INLINE, null);
MapReduceOutput out = c.mapReduce(cmd);
System.out.println("Mapreduce results");
for (DBObject o : out.results()) {
System.out.println(o.toString());
}
}
public MapReduceOutput postsStats() {
try {
Mongo mongo = new Mongo("localhost", 27017);
DB db = mongo.getDB("jciDb");
DBCollection posts = db.getCollection("post");
String map =
"function() { "
+ "var OLM; "
+ "switch ( this.olm ) {"
+ "case 'ENIT' : "
+ "OLM = 'ENIT'; "
+ "break;"
+ "case 'INSAT' : "
+ "OLM = 'INSAT'; "
+ "break;"
+ "case 'TUNIS' : "
+ "OLM = 'TUNIS'; "
+ "break;"
+ "case 'BEN AROUS' : "
+ "OLM = 'BEN AROUS'; "
+ "break;"
+ "case 'BIZERTE' : "
+ "OLM = 'BIZERTE'; "
+ "break;"
+ "case 'ARIANA' : "
+ "OLM = 'ARIANA'; "
+ "break;"
+ "case 'MANNOUBA' : "
+ "OLM = 'MANNOUBA'; "
+ "break;"
+ "case 'KELIBYA' : "
+ "OLM = 'KELIBYA'; "
+ "break;"
+ "case 'EZZAHRA' : "
+ "OLM = 'EZZAHRA'; "
+ "break;"
+ "case 'BARDO' : "
+ "OLM = 'BARDO'; "
+ "break;"
+ "case 'BOUMHEL' : "
+ "OLM = 'BOUMHEL'; "
+ "break;"
+ "case 'CARTHAGE' : "
+ "OLM = 'CARTHAGE'; "
+ "break;"
+ "case 'EL MENZAH' : "
+ "OLM = 'EL MENZAH'; "
+ "break;"
+ "case 'EL MOUROUJ' : "
+ "OLM = 'EL MOUROUJ'; "
+ "break;"
+ "case 'LA MARSA' : "
+ "OLM = 'LA MARSA'; "
+ "break;"
+ "case 'NABEUL' : "
+ "OLM = 'NABEUL'; "
+ "break;"
+ "case 'RADES' : "
+ "OLM = 'RADES'; "
+ "break;"
+ "case 'LA SOUKRA' : "
+ "OLM = 'LA SOUKRA'; "
+ "break;"
+ "case 'ULT' : "
+ "OLM = 'ULT'; "
+ "break;"
+ "case 'KORBA' : "
+ "OLM = 'KORBA'; "
+ "break;"
+ "case 'HAMMAMET' : "
+ "OLM = 'HAMMAMET'; "
+ "break;"
+ "case 'EL MANAR' : "
+ "OLM = 'EL MANAR'; "
+ "break;"
+ "default : "
+ "OLM = 'Autres OLMs'; "
+ "break;"
+ "}"
+ "emit(OLM, {contenu: this.contenu});}";
String reduce =
"function(key, values) { "
+ "var sum = 0; "
+ "values.forEach(function(doc) { "
+ "sum += 1; "
+ "}); "
+ "return {posts: sum};} ";
MapReduceCommand cmd =
new MapReduceCommand(posts, map, reduce, null, MapReduceCommand.OutputType.INLINE, null);
MapReduceOutput out = posts.mapReduce(cmd);
// for (DBObject o : out.results()) {
// String s = o.toString();
// String[] str1Array = s.split(":", 4);
// String[] str2Array = str1Array[3].split("}", 2);
// String[] str3Array = str2Array[0].split(".", 2);
// String[] str4Array = str1Array[1].split(",", 2);
//
// System.out.println(str4Array[0] + " : " + str3Array[1]);
// }
return (out);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
return (null);
}
}
public static void main(String[] args) {
// TODO Auto-generated method stub
try {
MongoClient mc = new MongoClient("localhost");
DB db = mc.getDB("mydb");
DBCollection dbc = db.getCollection("books");
/*BasicDBObject book=
new BasicDBObject();
book.put("name", "자바의 정석");
book.put("pages", 600);
dbc.insert(book);
book=
new BasicDBObject();
book.put("name", "스프링4");
book.put("pages", 250);
dbc.insert(book);
book=
new BasicDBObject();
book.put("name", "JQuery");
book.put("pages", 150);
dbc.insert(book);
book=
new BasicDBObject();
book.put("name", "오라클 프로그램");
book.put("pages", 800);
dbc.insert(book);
book=
new BasicDBObject();
book.put("name", "AngularJS");
book.put("pages", 130);
dbc.insert(book);
book=
new BasicDBObject();
book.put("name", "MongoDB");
book.put("pages", 650);
dbc.insert(book);
book=
new BasicDBObject();
book.put("name", "Hadoop");
book.put("pages", 300);
dbc.insert(book);
book=
new BasicDBObject();
book.put("name", "HIVE");
book.put("pages", 350);
dbc.insert(book);
book=
new BasicDBObject();
book.put("name", "PIG");
book.put("pages", 650);
dbc.insert(book);
book=
new BasicDBObject();
book.put("name", "HTML5");
book.put("pages", 360);
dbc.insert(book);*/
String map =
"function(){"
+ "var category;"
+ "if(this.pages>=300){"
+ "category='Big Books';"
+ "}else{"
+ "category='Small Books';}"
+ "emit(category,{name:this.name});}";
String reduce =
"function(key,values){"
+ "var sum=0;"
+ "values.forEach(function(doc){"
+ "sum+=1;"
+ "});"
+ "return {books:sum};}";
MapReduceCommand cmd =
new MapReduceCommand(dbc, map, reduce, null, MapReduceCommand.OutputType.INLINE, null);
MapReduceOutput out = dbc.mapReduce(cmd);
for (DBObject o : out.results()) {
System.out.println(o.toString());
}
} catch (Exception ex) {
System.out.println(ex.getMessage());
}
}