Exemplo n.º 1
0
  @Test
  public void groupFile() {
    long startTiem = System.currentTimeMillis();
    String map =
        "function() { "
            + "var category; "
            + "if ( this.secretStatus = 2 ) "
            + "category = '标密状态位2'; "
            + "else "
            + "category = '标密状态位其它'; "
            + "emit(category, {secretPerson: this.secretPerson});}";

    String reduce =
        "function(key, values) { "
            + "var sum = 0; "
            + "values.forEach(function(doc) { "
            + "sum += 1; "
            + "}); "
            + "return {副本总数: sum};} ";

    MapReduceCommand cmd =
        new MapReduceCommand(bakFiles, map, reduce, null, MapReduceCommand.OutputType.INLINE, null);

    MapReduceOutput out = bakFiles.mapReduce(cmd);

    long endTime = System.currentTimeMillis();
    System.out.println("本次操作消费   " + (endTime - startTiem) + "  毫秒");
    for (DBObject o : out.results()) {
      System.out.println(o.toString());
    }
  }
Exemplo n.º 2
0
  /** 使用mongodb的mapreduce分组统计</br> 日期:2014-3-13 上午11:47:08 */
  @Test
  public void mapreduce() {

    String map =
        " function() {"
            + "emit("
            + "{fileId:this.fileId,clientId:this.clientId},"
            + "{count:0}"
            + ");"
            + "}";

    String reduce =
        "function (key, values) {"
            + "var ss = {count:0};"
            + "values.count.forEach(val){"
            + "ss.count += = val.count;"
            + "}"
            + "return ss;"
            + "}";
    MapReduceCommand cmd =
        new MapReduceCommand(bakFiles, map, reduce, null, MapReduceCommand.OutputType.INLINE, null);

    MapReduceOutput out = bakFiles.mapReduce(cmd);
    System.out.println(out.toString());
  }
Exemplo n.º 3
0
  public List<Double> getTimeStatsForUser(User u) {
    DBObject queryObject = new BasicDBObject("authorName", u.getUsername());
    String map =
        "function(){"
            + "var hours = this.datePublished.getHours();"
            + "var zone;"
            + "if(hours >= 0 && hours < 4){"
            + "zone = 0;"
            + "}"
            + "else if(hours >= 4 && hours < 8){"
            + "zone = 1;"
            + "}"
            + "else if(hours >= 8 && hours < 12){"
            + "zone = 2;"
            + "}"
            + "else if(hours >= 12 && hours < 16){"
            + "zone = 3;"
            + "}"
            + "else if(hours >= 16 && hours < 20){"
            + "zone = 4;"
            + "}"
            + "else {"
            + "zone = 5;"
            + "}"
            + "emit(zone, 1);"
            + "}";

    String reduce = "function(k, v){" + "return Array.sum(v);" + "}";

    DBCollection collection = datastore.getCollection(Message.class);
    MapReduceCommand mrc =
        new MapReduceCommand(
            collection, map, reduce, null, MapReduceCommand.OutputType.INLINE, queryObject);

    MapReduceOutput output = collection.mapReduce(mrc);
    Integer count = collection.find(queryObject).count();

    List<Double> result = new ArrayList<>();
    for (int i = 0; i <= 5; i++) {
      result.add(0.0);
    }
    result.add(count.doubleValue());
    output
        .results()
        .forEach(
            (DBObject o) ->
                result.set(((Double) o.get("_id")).intValue(), (Double) o.get("value")));

    return result;
  }
Exemplo n.º 4
0
  public Map<String, Double> getPlaceStatsForUser(User u) {
    DBObject queryObject = new BasicDBObject("authorName", u.getUsername());
    String map =
        "function(){"
            + "var place = this.place;"
            + "if(place != null){"
            + "if(place === ''){"
            + "place = 'Empty';"
            + "}"
            + "else if(place !== 'Sofia' && place !== 'Plovdiv' && place !== 'Varna' && place !== 'Burgas'){"
            + "place = 'Other';"
            + "}"
            + "}"
            + "else {"
            + "place = 'Empty';"
            + "}"
            + "emit(place, 1);"
            + "}";

    String reduce = "function(k, v){ return Array.sum(v); }";
    DBCollection collection = datastore.getCollection(Message.class);
    MapReduceCommand mrc =
        new MapReduceCommand(
            collection, map, reduce, null, MapReduceCommand.OutputType.INLINE, queryObject);

    MapReduceOutput output = collection.mapReduce(mrc);
    Integer count = collection.find(queryObject).count();

    Map<String, Double> result = new HashMap<>();
    result.put("Total", count.doubleValue());
    result.put("Burgas", 0.0);
    result.put("Varna", 0.0);
    result.put("Sofia", 0.0);
    result.put("Plovdiv", 0.0);
    result.put("Other", 0.0);
    result.put("Empty", 0.0);
    output
        .results()
        .forEach((DBObject o) -> result.replace((String) o.get("_id"), (Double) o.get("value")));

    return result;
  }
Exemplo n.º 5
0
  /**
   * 查找数据sex1={name1};sex1={name2};sex1={name3};sex2={name1};sex2={name2};sex2={name3};sex3={name1};sex3={name2};sex3={name3};
   * String map = "function(){emit(this.sex,this.name);}";
   * 处理成sex1={name1,name2,name3};sex2={name1,name2,name3};sex3={name1,name2,name3}; String
   * reduce="function(key,values){ var ret={sex:key,names:values}; return ret;}";
   *
   * @param collectionName
   * @param mapFunction
   * @param reduceFunction
   * @param targetCollection
   * @param keyNameOfReduceValue
   */
  public void mapReduce(
      String collectionName,
      String mapFunction,
      String reduceFunction,
      String targetCollection,
      String keyNameOfReduceValue) {
    try {
      DBCollection collection = getCollection(collectionName);
      // 执行mapReduce命令
      MapReduceOutput result =
          collection.mapReduce(
              mapFunction, reduceFunction, targetCollection, OutputType.INLINE, null);
      logger.info(
          "BaseMongoDAO.mapReduce返回的原始数据值,  result=" + result.getCommandResult().toString());
      // 获取mapreduce的结果
      String reduceResult = result.getCommandResult().get("results").toString();
      logger.info("BaseMongoDAO.mapReduce返回的有效数据值,  reduceResult=" + reduceResult);

      // 返回结果key=要处理的属性;value=聚合后的结果
      Map<String, String> mapReduceResult = new HashMap<String, String>();
      // 将返回结果用json序列化格式为[{ "_id" : null , "value" : { "sex" : null , "names" : [ "test3" , "test3"
      // , "test3"]}} , { "_id" : "man" , "value" : { "sex" :
      // "man" , "names" : [ "name" , "test6" , "test6" , "test3" , "test3" , "test3" , "test3" ,
      // "test3" , "test3" , "test3" , "test3" , "test3" ,
      // "test3" , "test3" , "test3"]}}]
      List<Map<String, Object>> reduceMap = JsonUtil.fromJson(reduceResult, List.class);
      // 序列化后的串[{_id=null, value={sex=null, num=3.0}}, {_id=man, value={sex=man, num=15.0}},
      // {_id=woman, value={sex=woman, num=5.0}}, {_id=women,
      // value={name=test2, num=1.0}}]
      System.out.println(reduceMap);
      logger.info("BaseMongoDAO.mapReduce方法json反序列化后的值,  jsonToClassReduceMap=" + reduceMap);
      // 遍历mapReduce的结果
      for (Map<String, Object> mapInfo : reduceMap) {
        // 定义reduce出的key对应的collection中的值
        String keyValue = "null";
        for (Entry<String, Object> mapInfoEntry : mapInfo.entrySet()) {
          // System.out.println("111map key=====" +mapInfoEntry.getKey());
          // 获取key对应的collection中的值
          if (mapInfoEntry.getKey().equals("_id")) {
            keyValue = processNull(mapInfoEntry.getValue());
          }
          /**
           * 这个判断主要是为了集中处理value={sex=man, num=15.0}这类键值
           * 绕开对key=NULL(即value为NULL)这类键值对的处理;例如{_id=null, value={sex=null,
           * num=3.0}}中的_id和sex;如果为空则不做任何处理 因为下面使用的是MAP的强转,不适宜基础类型
           */
          // 如果reduce的值不为空,而且不是基础类型则继续解析map
          if (mapInfoEntry.getValue() != null
              && !isPrimitiveType(mapInfoEntry.getValue().getClass().getName())) {
            // System.out.println("in  in 111map value=====" +mapInfoEntry.getValue());
            LinkedHashMap<Object, Object> realInfoMapList =
                (LinkedHashMap<Object, Object>) mapInfoEntry.getValue();
            for (Entry e : realInfoMapList.entrySet()) {
              if (keyNameOfReduceValue.equals(e.getKey().toString())) {
                mapReduceResult.put(keyValue, e.getValue().toString());
              }
              // System.out.println(e.getKey());
              // System.out.println(e.getValue());
            }
            // 值不为NULL的基础类型的处理逻辑
          } else if (mapInfoEntry.getValue() != null) {
            if (keyNameOfReduceValue.equals(mapInfoEntry.getKey().toString())) {
              mapReduceResult.put(keyValue, mapInfoEntry.getValue().toString());
            }
            // 值为NULL的处理,只打印日志
          } else {
            logger.error("BaseMongoDAO.mapReduce方法,查到的结果为NULL,不做任何处理" + mapInfoEntry.getValue());
          }
        }
      }

      // 输出reduce处理完成后的结果集
      for (Entry<String, String> mapInfo : mapReduceResult.entrySet()) {
        logger.info(
            "BaseMongoDAO.mapReduce方法聚合后的结果key="
                + mapInfo.getKey()
                + "======"
                + mapInfo.getValue());
      }
    } catch (Exception e) {
      logger.error(
          "BaseMongoDAO.mapReduce聚合数据时发生异常,入参collectionName="
              + collectionName
              + ";  mapFunction="
              + mapFunction
              + ";  reduceFunction="
              + reduceFunction
              + ";  targetCollection="
              + targetCollection
              + ";  keyNameOfReduceValue="
              + keyNameOfReduceValue,
          e);
    }
  }
  @Test
  public void test() {

    DBObject o1 =
        new BasicDBObject("name", "Rhys Lee")
            .append("age", 29)
            .append("sex", 1)
            .append("type", 6)
            .append("regDate", DateUtil.getToday("yyyyMMddHHmmss"));

    DBObject o2 =
        new BasicDBObject("name", "Ari")
            .append("age", 23)
            .append("sex", 0)
            .append("type", 3)
            .append("regDate", DateUtil.getToday("yyyyMMddHHmmss"));

    DBObject o3 =
        new BasicDBObject("name", "Atrox")
            .append("age", 30)
            .append("sex", 1)
            .append("type", 5)
            .append("regDate", DateUtil.getToday("yyyyMMddHHmmss"));

    DBObject o4 =
        new BasicDBObject("name", "Atrox1")
            .append("age", 35)
            .append("sex", 1)
            .append("type", 5)
            .append("regDate", DateUtil.getToday("yyyyMMddHHmmss"));

    DBObject o5 =
        new BasicDBObject("name", "Atrox2")
            .append("age", 25)
            .append("sex", 1)
            .append("type", 5)
            .append("regDate", DateUtil.getToday("yyyyMMddHHmmss"));

    DBCollection c = MongoTemplate.getDBCollection("TEST");
    c.drop(); // 테스트를 위한 초기화
    c.insert(Arrays.asList(o1, o2, o3, o4, o5));

    String map =
        "function() { "
            + "var category; "
            + "if ( this.age >= 30 ) "
            + "category = 'Big'; "
            + "else "
            + "category = 'Small'; "
            + "emit(category, {name: this.name});}";

    String reduce =
        "function(key, values) { "
            + "var sum = 0; "
            + "values.forEach(function(doc) { sum += 1; "
            + "}); "
            + "return {cnt : sum};} ";

    MapReduceCommand cmd =
        new MapReduceCommand(c, map, reduce, null, MapReduceCommand.OutputType.INLINE, null);
    MapReduceOutput out = c.mapReduce(cmd);
    System.out.println("Mapreduce results");
    for (DBObject o : out.results()) {
      System.out.println(o.toString());
    }
  }
Exemplo n.º 7
0
  public MapReduceOutput postsStats() {
    try {
      Mongo mongo = new Mongo("localhost", 27017);

      DB db = mongo.getDB("jciDb");

      DBCollection posts = db.getCollection("post");

      String map =
          "function() { "
              + "var OLM; "
              + "switch ( this.olm ) {"
              + "case 'ENIT' : "
              + "OLM = 'ENIT'; "
              + "break;"
              + "case 'INSAT' : "
              + "OLM = 'INSAT'; "
              + "break;"
              + "case 'TUNIS' : "
              + "OLM = 'TUNIS'; "
              + "break;"
              + "case 'BEN AROUS' : "
              + "OLM = 'BEN AROUS'; "
              + "break;"
              + "case 'BIZERTE' : "
              + "OLM = 'BIZERTE'; "
              + "break;"
              + "case 'ARIANA' : "
              + "OLM = 'ARIANA'; "
              + "break;"
              + "case 'MANNOUBA' : "
              + "OLM = 'MANNOUBA'; "
              + "break;"
              + "case 'KELIBYA' : "
              + "OLM = 'KELIBYA'; "
              + "break;"
              + "case 'EZZAHRA' : "
              + "OLM = 'EZZAHRA'; "
              + "break;"
              + "case 'BARDO' : "
              + "OLM = 'BARDO'; "
              + "break;"
              + "case 'BOUMHEL' : "
              + "OLM = 'BOUMHEL'; "
              + "break;"
              + "case 'CARTHAGE' : "
              + "OLM = 'CARTHAGE'; "
              + "break;"
              + "case 'EL MENZAH' : "
              + "OLM = 'EL MENZAH'; "
              + "break;"
              + "case 'EL MOUROUJ' : "
              + "OLM = 'EL MOUROUJ'; "
              + "break;"
              + "case 'LA MARSA' : "
              + "OLM = 'LA MARSA'; "
              + "break;"
              + "case 'NABEUL' : "
              + "OLM = 'NABEUL'; "
              + "break;"
              + "case 'RADES' : "
              + "OLM = 'RADES'; "
              + "break;"
              + "case 'LA SOUKRA' : "
              + "OLM = 'LA SOUKRA'; "
              + "break;"
              + "case 'ULT' : "
              + "OLM = 'ULT'; "
              + "break;"
              + "case 'KORBA' : "
              + "OLM = 'KORBA'; "
              + "break;"
              + "case 'HAMMAMET' : "
              + "OLM = 'HAMMAMET'; "
              + "break;"
              + "case 'EL MANAR' : "
              + "OLM = 'EL MANAR'; "
              + "break;"
              + "default : "
              + "OLM = 'Autres OLMs'; "
              + "break;"
              + "}"
              + "emit(OLM, {contenu: this.contenu});}";

      String reduce =
          "function(key, values) { "
              + "var sum = 0; "
              + "values.forEach(function(doc) { "
              + "sum += 1; "
              + "}); "
              + "return {posts: sum};} ";

      MapReduceCommand cmd =
          new MapReduceCommand(posts, map, reduce, null, MapReduceCommand.OutputType.INLINE, null);

      MapReduceOutput out = posts.mapReduce(cmd);

      // for (DBObject o : out.results()) {
      // String s = o.toString();
      // String[] str1Array = s.split(":", 4);
      // String[] str2Array = str1Array[3].split("}", 2);
      // String[] str3Array = str2Array[0].split(".", 2);
      // String[] str4Array = str1Array[1].split(",", 2);
      //
      // System.out.println(str4Array[0] + " : " + str3Array[1]);
      // }
      return (out);

    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
      return (null);
    }
  }
Exemplo n.º 8
0
  public static void main(String[] args) {
    // TODO Auto-generated method stub
    try {
      MongoClient mc = new MongoClient("localhost");
      DB db = mc.getDB("mydb");
      DBCollection dbc = db.getCollection("books");
      /*BasicDBObject book=
      		new BasicDBObject();
      book.put("name", "자바의 정석");
      book.put("pages", 600);
      dbc.insert(book);

      book=
      		new BasicDBObject();
      book.put("name", "스프링4");
      book.put("pages", 250);
      dbc.insert(book);
      book=
      		new BasicDBObject();
      book.put("name", "JQuery");
      book.put("pages", 150);
      dbc.insert(book);
      book=
      		new BasicDBObject();
      book.put("name", "오라클 프로그램");
      book.put("pages", 800);
      dbc.insert(book);
      book=
      		new BasicDBObject();
      book.put("name", "AngularJS");
      book.put("pages", 130);
      dbc.insert(book);
      book=
      		new BasicDBObject();
      book.put("name", "MongoDB");
      book.put("pages", 650);
      dbc.insert(book);
      book=
      		new BasicDBObject();
      book.put("name", "Hadoop");
      book.put("pages", 300);
      dbc.insert(book);
      book=
      		new BasicDBObject();
      book.put("name", "HIVE");
      book.put("pages", 350);
      dbc.insert(book);
      book=
      		new BasicDBObject();
      book.put("name", "PIG");
      book.put("pages", 650);
      dbc.insert(book);
      book=
      		new BasicDBObject();
      book.put("name", "HTML5");
      book.put("pages", 360);
      dbc.insert(book);*/
      String map =
          "function(){"
              + "var category;"
              + "if(this.pages>=300){"
              + "category='Big Books';"
              + "}else{"
              + "category='Small Books';}"
              + "emit(category,{name:this.name});}";
      String reduce =
          "function(key,values){"
              + "var sum=0;"
              + "values.forEach(function(doc){"
              + "sum+=1;"
              + "});"
              + "return {books:sum};}";
      MapReduceCommand cmd =
          new MapReduceCommand(dbc, map, reduce, null, MapReduceCommand.OutputType.INLINE, null);
      MapReduceOutput out = dbc.mapReduce(cmd);

      for (DBObject o : out.results()) {
        System.out.println(o.toString());
      }
    } catch (Exception ex) {
      System.out.println(ex.getMessage());
    }
  }