Пример #1
0
  /**
   * All headers except collection and database are non available for this operation.
   *
   * @param exchange
   * @throws Exception
   */
  protected void doAggregate(Exchange exchange) throws Exception {
    DBCollection dbCol = calculateCollection(exchange);
    DBObject query = exchange.getIn().getMandatoryBody(DBObject.class);

    // Impossible with java driver to get the batch size and number to skip
    Iterable<DBObject> dbIterator = null;
    AggregationOutput aggregationResult = null;

    // Allow body to be a pipeline
    // @see http://docs.mongodb.org/manual/core/aggregation/
    if (query instanceof BasicDBList) {
      BasicDBList queryList = (BasicDBList) query;
      aggregationResult =
          dbCol.aggregate(
              (DBObject) queryList.get(0),
              queryList
                  .subList(1, queryList.size())
                  .toArray(new BasicDBObject[queryList.size() - 1]));
    } else {
      aggregationResult = dbCol.aggregate(query);
    }

    dbIterator = aggregationResult.results();
    Message resultMessage = prepareResponseMessage(exchange, MongoDbOperation.aggregate);
    resultMessage.setBody(dbIterator);
  }
Пример #2
0
  public static DBCollection createFeedsItemsCollectionMock(DB db) {
    DBCollection feedItemsCollection = mock(DBCollection.class);
    when(db.getCollection("feedItems")).thenReturn(feedItemsCollection);

    DBCursor feedsItemsCursor = createCursorMock(createFeedsItemDbObjectMock());
    when(feedsItemsCursor.sort(any(DBObject.class))).thenReturn(feedsItemsCursor);
    when(feedsItemsCursor.limit(any(Integer.class))).thenReturn(feedsItemsCursor);
    when(feedsItemsCursor.skip(any(Integer.class))).thenReturn(feedsItemsCursor);
    when(feedItemsCollection.find(any(BasicDBObject.class))).thenReturn(feedsItemsCursor);

    AggregationOutput aggregationOutput = Mockito.mock(AggregationOutput.class);
    List<DBObject> results = new ArrayList<DBObject>();
    DBObject result1 = new BasicDBObject();
    result1.put("_id", "tag1");
    result1.put("count", 5);
    results.add(result1);
    DBObject result2 = new BasicDBObject();
    result2.put("_id", "tag2");
    result2.put("count", 8);
    results.add(result2);

    when(aggregationOutput.results()).thenReturn(results);
    when(feedItemsCollection.aggregate(
            any(DBObject.class), any(DBObject.class), any(DBObject.class), any(DBObject.class)))
        .thenReturn(aggregationOutput);

    AggregationOutput countAggregate = Mockito.mock(AggregationOutput.class);
    List<DBObject> countResults = new ArrayList<DBObject>();
    DBObject countResult1 = new BasicDBObject();
    countResult1.put("_id", "12345");
    countResult1.put("count", 5);
    countResults.add(countResult1);
    when(countAggregate.results()).thenReturn(countResults);

    when(feedItemsCollection.aggregate(any(DBObject.class), any(DBObject.class)))
        .thenReturn(countAggregate);

    DBObject feedItem = new BasicDBObject();
    feedItem.put("_id", "52ffe096e81f7a9bb906b6f9");
    feedItem.put("categoryId", "5581e900d4c6cda2d04d0a98");
    feedItem.put("feedId", "55893938e4b0c822dc99fa2f");
    feedItem.put("delete", true);
    feedItem.put("description", "description");
    feedItem.put(
        "link", "http://thedailyedge.thejournal.ie/irish-supermarket-fails-2176781-Jun2015/");
    feedItem.put("imageUrl", "http://thedailyedge.thejournal.ie/image.jpg");
    feedItem.put("read", true);
    feedItem.put("source", "The Daily Edge");
    feedItem.put("title", "title");
    feedItem.put("username", "bob");
    feedItem.put("pubDate", new Date());

    when(feedItemsCollection.findOne(any(DBObject.class))).thenReturn(feedItem);

    return feedItemsCollection;
  }
Пример #3
0
  public List<DBObject> getSourceBrowsers() {

    DBObject groupFields = new BasicDBObject("_id", "$device");
    groupFields.put("sum", new BasicDBObject("$sum", "$sum"));
    DBObject group = new BasicDBObject("$group", groupFields);
    DBObject sort = new BasicDBObject("$sort", new BasicDBObject("sum", 1));

    List<DBObject> pipeline = Arrays.asList(group, sort);

    // allowDiskUse
    AggregationOptions options =
        AggregationOptions.builder().allowDiskUse(true).batchSize(10000).build();
    Cursor cursor = device.aggregate(pipeline, options);

    List<DBObject> list = new ArrayList<DBObject>();
    while (cursor.hasNext()) {
      DBObject object = cursor.next();
      DBObject newObj = new BasicDBObject();
      newObj.put("device", object.get("_id"));
      newObj.put("sum", object.get("sum"));
      list.add(newObj);
    }
    cursor.close();
    return list;
  }
Пример #4
0
  private List<String> selectViewCodes(String schema) throws UnknownHostException {
    // Initiate variables
    List<String> l = new ArrayList<String>();
    MongoDBConnectionManager mgr = MongoDBConnectionManager.getInstance();
    Mongo mongo = mgr.getMongo();
    DB db = mongo.getDB(schema);
    DBCollection collection = db.getCollection("views");

    // Compose NoSQL query
    BasicDBObject groupFiels = new BasicDBObject("_id", "$view_id");
    BasicDBObject group = new BasicDBObject("$group", groupFiels);
    BasicDBObject sortFiels = new BasicDBObject("_id", 1);
    BasicDBObject sort = new BasicDBObject("$sort", sortFiels);

    // Create the output
    AggregationOutput output = collection.aggregate(group, sort);
    Iterable<DBObject> results = output.results();
    for (DBObject result : results) {
      try {
        l.add(result.get("_id").toString());
      } catch (NullPointerException e) {
        System.out.println("Skipping NULL");
      }
    }

    // Return the result
    return l;
  }
  @Override
  public void execute() throws TranslatorException {
    this.visitor = new MongoDBSelectVisitor(this.executionFactory, this.metadata);
    this.visitor.visitNode(this.command);

    if (!this.visitor.exceptions.isEmpty()) {
      throw this.visitor.exceptions.get(0);
    }

    LogManager.logInfo(LogConstants.CTX_CONNECTOR, this.command);

    DBCollection collection =
        this.mongoDB.getCollection(this.visitor.mongoDoc.getTargetTable().getName());
    if (collection != null) {
      // TODO: check to see how to pass the hint
      ArrayList<DBObject> ops = new ArrayList<DBObject>();
      buildAggregate(ops, "$project", this.visitor.unwindProject); // $NON-NLS-1$

      if (this.visitor.project.isEmpty()) {
        throw new TranslatorException(
            MongoDBPlugin.Event.TEIID18025, MongoDBPlugin.Util.gs(MongoDBPlugin.Event.TEIID18025));
      }

      if (this.visitor.projectBeforeMatch) {
        buildAggregate(ops, "$project", this.visitor.project); // $NON-NLS-1$
      }

      if (!this.visitor.unwindTables.isEmpty()) {
        for (String ref : this.visitor.unwindTables) {
          buildAggregate(ops, "$unwind", "$" + ref); // $NON-NLS-1$ //$NON-NLS-2$
        }
      }
      buildAggregate(ops, "$match", this.visitor.match); // $NON-NLS-1$

      buildAggregate(ops, "$group", this.visitor.group); // $NON-NLS-1$
      buildAggregate(ops, "$match", this.visitor.having); // $NON-NLS-1$

      if (!this.visitor.projectBeforeMatch) {
        buildAggregate(ops, "$project", this.visitor.project); // $NON-NLS-1$
      }

      buildAggregate(ops, "$sort", this.visitor.sort); // $NON-NLS-1$
      buildAggregate(ops, "$skip", this.visitor.skip); // $NON-NLS-1$
      buildAggregate(ops, "$limit", this.visitor.limit); // $NON-NLS-1$

      try {
        AggregationOptions options =
            AggregationOptions.builder()
                .batchSize(this.executionContext.getBatchSize())
                .outputMode(AggregationOptions.OutputMode.CURSOR)
                .allowDiskUse(this.executionFactory.useDisk())
                .build();
        this.results = collection.aggregate(ops, options);
      } catch (MongoException e) {
        throw new TranslatorException(e);
      }
    }
  }
Пример #6
0
  @Override
  public MorphiaIterator<U, U> aggregate(
      final String collectionName,
      final Class<U> target,
      final AggregationOptions options,
      final ReadPreference readPreference) {
    LOG.debug("stages = " + stages);

    Cursor cursor = collection.aggregate(stages, options, readPreference);
    return new MorphiaIterator<U, U>(
        cursor, mapper, target, collection.getName(), mapper.createEntityCache());
  }
Пример #7
0
  @Override
  public void getHostsByRequestType() {
    final BasicDBObject match =
        new BasicDBObject("$match", new BasicDBObject("Request details.Request type", HEAD));
    final BasicDBObject group = new BasicDBObject("$group", new BasicDBObject("_id", "$Host"));

    final AggregationOutput result = table.aggregate(match, group);

    System.out.println("getHostsByRequestType: ");
    for (final DBObject obj : result.results()) {
      System.out.println(obj);
    }
  }
Пример #8
0
  // 获得一天的转换漏斗
  public List<DBObject> getConversionFunnel(String date) {
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
    DBObject match = null;
    try {
      match = new BasicDBObject("$match", new BasicDBObject("date", sdf.parse(date)));
    } catch (ParseException e) {

    }
    DBObject groupFields = new BasicDBObject("_id", "");
    groupFields.put("step1sum", new BasicDBObject("$sum", "$step1"));
    groupFields.put("step2sum", new BasicDBObject("$sum", "$step2"));
    groupFields.put("step3sum", new BasicDBObject("$sum", "$step3"));

    DBObject group = new BasicDBObject("$group", groupFields);
    // DBObject sort = new BasicDBObject("$sort", new BasicDBObject("sum", 1));

    List<DBObject> pipeline = Arrays.asList(match, group);

    // allowDiskUse
    AggregationOptions options =
        AggregationOptions.builder().allowDiskUse(true).batchSize(10000).build();
    Cursor cursor = CF.aggregate(pipeline, options);

    List<DBObject> list = new ArrayList<DBObject>();
    while (cursor.hasNext()) {
      DBObject object = cursor.next();
      DBObject newObj = new BasicDBObject();
      //            DBObject _id = (DBObject)object.get("_id");
      //            newObj.put("date",sdf.format(_id.get("$date")));
      DBObject newObj1 = new BasicDBObject();
      newObj1.put("step", "访问本站");
      newObj1.put("num", object.get("step1sum"));
      list.add(newObj1);
      DBObject newObj2 = new BasicDBObject();
      newObj2.put("step", "浏览商品");
      newObj2.put("num", object.get("step2sum"));
      list.add(newObj2);

      DBObject newObj3 = new BasicDBObject();
      newObj3.put("step", "完成交易");
      newObj3.put("num", object.get("step3sum"));
      list.add(newObj3);
    }
    cursor.close();
    return list;
  }
Пример #9
0
  public void getNumOfReplyCodeByRequestTypeAggr() {
    final BasicDBObject match = new BasicDBObject();
    match.put("Request details.Request type", GET);
    match.put("Request details.Reply code", SUCCESS_REPLY_CODE);

    final BasicDBObject group = new BasicDBObject();
    group.put("_id", 0);
    group.put("count", new BasicDBObject("$sum", 1));

    final BasicDBObject matchOp = new BasicDBObject("$match", match);
    final BasicDBObject unwindOp = new BasicDBObject("$unwind", "$Request details");
    final BasicDBObject groupOp = new BasicDBObject("$group", group);

    final AggregationOutput result = table.aggregate(matchOp, unwindOp, matchOp, groupOp);

    System.out.println("getNumOfReplyCodeByRequestType: ");
    for (DBObject obj : result.results()) {
      System.out.println(obj.get("count"));
    }
  }
  public static void main(String[] args) throws UnknownHostException {
    MongoClient client = new MongoClient();

    DB database = client.getDB("m101");
    DBCollection collection = database.getCollection("funnynumbers");

    // Not necessary yet to understand this.  It's just to prove that you
    // are able to run a command on a mongod server
    AggregationOutput output =
        collection.aggregate(
            new BasicDBObject(
                "$group",
                new BasicDBObject("_id", "$value").append("count", new BasicDBObject("$sum", 1))),
            new BasicDBObject("$match", new BasicDBObject("count", new BasicDBObject("$gt", 2))),
            new BasicDBObject("$sort", new BasicDBObject("_id", 1)));

    int answer = 0;
    for (DBObject doc : output.results()) {
      answer += (Double) doc.get("_id");
    }

    System.out.println("THE ANSWER IS: " + answer);
  }
Пример #11
0
  /** @param args */
  public static void main(String[] args) {
    // TODO Auto-generated method stub

    List<ServerAddress> seeds = new ArrayList<ServerAddress>();

    try {
      seeds.add(new ServerAddress("202.191.136.234", 27017));
      //			seeds.add(new ServerAddress("172.31.1.5", 27017));
    } catch (UnknownHostException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }

    MongoClient client = new MongoClient(seeds);

    DB db = client.getDB("netflowdb");
    DBCollection collection = db.getCollection("netflow_column_list");

    //		System.out.println(collection.findOne());

    DBObject record = collection.findOne();

    System.out.println(record.get("timestamp"));
    Date time = (Date) record.get("timestamp");
    System.out.println(time.getTime());

    Date time1 = new Date(time.getTime());
    System.out.println("new time " + time1);

    System.exit(1);
    DBCollection output = db.getCollection("agg_test");

    //		System.out.println("count : " + collection.count());

    Calendar startTime = Calendar.getInstance();
    startTime.add(Calendar.HOUR_OF_DAY, -1);
    startTime.set(Calendar.MINUTE, 0);
    startTime.set(Calendar.SECOND, 0);
    startTime.set(Calendar.MILLISECOND, 0);

    DBObject match_query =
        new BasicDBObject()
            .append("_id.dikey.routerip", "172.31.4.79")
            //		.append("_id.dikey.ifindex", 5)
            .append("_id.rd", startTime.getTime());

    DBObject match = new BasicDBObject("$match", match_query);
    System.out.println(startTime.getTime());

    System.out.println("find-match : " + collection.find(match_query).count());

    DBObject group =
        new BasicDBObject(
            "$group",
            new BasicDBObject()
                .append(
                    "_id",
                    new BasicDBObject()
                        .append("dikey", "$_id.dikey")
                        .append("customerid", "$_id.customerid")
                        .append("dstport", "$_id.dstport")
                        .append("protocol", "$_id.protocol")
                        .append("hourCount", "$_id.hourCount")
                        .append("rd", "$_id.rd"))
                .append("total", new BasicDBObject("$sum", "$value")));

    DBObject sort = new BasicDBObject("$sort", new BasicDBObject("total", -1));
    DBObject limit = new BasicDBObject("$limit", 3);

    List<DBObject> pipeline = Arrays.asList(group, sort);

    //		AggregationOutput result = collection.aggregate(match, (DBObject[]) pipeline.toArray());
    AggregationOutput result = collection.aggregate(match, group, sort, limit);
    System.out.println(result.getCommandResult());
    output.insert((List<DBObject>) result.results());
  }
Пример #12
0
  public List<MonthOperationResume> getAllMonthOperations() throws UnknownHostException {

    List<MonthOperationResume> resultList = new ArrayList<>();

    MongoClient client = null;
    try {
      client = getClient();
      DBCollection collection =
          super.getCollection(client, OperationConverter.OPERATION_COLLECTION_NAME);

      DBObject project0Fields = new BasicDBObject();
      BasicDBList add2HoursList = new BasicDBList();
      add2HoursList.add("$dateOperation");
      add2HoursList.add(2 * 60 * 60 * 1_000);
      DBObject add2Hours = new BasicDBObject("$add", add2HoursList);
      //			DBObject newDateOperation = new BasicDBObject("dateOperation", add2Hours);
      project0Fields.put("dateOperation", add2Hours);
      project0Fields.put("amount", "$amount");
      project0Fields.put("assistantOneAmount", "$assistantOneAmount");
      project0Fields.put("assistantTwoAmount", "$assistantTwoAmount");
      project0Fields.put("paymentRecievedDate", "$paymentRecievedDate");
      project0Fields.put("assistantsPaidDate", "$assistantsPaidDate");

      DBObject projectFields = new BasicDBObject("_id", 0);
      DBObject keyObject = new BasicDBObject();
      keyObject.put("month", new BasicDBObject("$month", "$dateOperation"));
      keyObject.put("year", new BasicDBObject("$year", "$dateOperation"));
      projectFields.put("key", keyObject);
      projectFields.put("amount", "$amount");
      projectFields.put("assistantOneAmount", "$assistantOneAmount");
      projectFields.put("assistantTwoAmount", "$assistantTwoAmount");

      BasicDBList conditionAssistantsPaidList = new BasicDBList();
      conditionAssistantsPaidList.add("$assistantsPaidDate");
      conditionAssistantsPaidList.add(1);
      conditionAssistantsPaidList.add(0);
      projectFields.put(
          "numberAssistantPaid", new BasicDBObject("$cond", conditionAssistantsPaidList));

      BasicDBList assistantOneGtList = new BasicDBList();
      assistantOneGtList.add("$assistantOneAmount");
      assistantOneGtList.add(0);

      BasicDBList assistantTwoGtList = new BasicDBList();
      assistantTwoGtList.add("$assistantTwoAmount");
      assistantTwoGtList.add(0);

      BasicDBList orList = new BasicDBList();
      orList.add(new BasicDBObject("$gt", assistantOneGtList));
      orList.add(new BasicDBObject("$gt", assistantTwoGtList));

      BasicDBObject orClause = new BasicDBObject();
      orClause.put("$or", orList);

      BasicDBList conditionNumberAssistedList = new BasicDBList();
      conditionNumberAssistedList.add(orClause);
      conditionNumberAssistedList.add(1);
      conditionNumberAssistedList.add(0);
      projectFields.put(
          "numberAssistedOperations", new BasicDBObject("$cond", conditionNumberAssistedList));

      BasicDBList conditionPaymentsRecievedList = new BasicDBList();
      conditionPaymentsRecievedList.add("$paymentRecievedDate");
      conditionPaymentsRecievedList.add(1);
      conditionPaymentsRecievedList.add(0);
      projectFields.put(
          "numberPaymentsRecieved", new BasicDBObject("$cond", conditionPaymentsRecievedList));

      DBObject project0 = new BasicDBObject();
      project0.put("$project", project0Fields);

      DBObject project = new BasicDBObject();
      project.put("$project", projectFields);

      DBObject groupFields = new BasicDBObject();
      groupFields.put("_id", "$key");
      groupFields.put("amount", new BasicDBObject("$sum", "$amount"));
      groupFields.put("assistantOneAmount", new BasicDBObject("$sum", "$assistantOneAmount"));
      groupFields.put("assistantTwoAmount", new BasicDBObject("$sum", "$assistantTwoAmount"));
      groupFields.put("numberAssistantPaid", new BasicDBObject("$sum", "$numberAssistantPaid"));
      groupFields.put(
          "numberAssistedOperations", new BasicDBObject("$sum", "$numberAssistedOperations"));
      groupFields.put("numberOperations", new BasicDBObject("$sum", 1));
      groupFields.put(
          "numberPaymentsRecieved", new BasicDBObject("$sum", "$numberPaymentsRecieved"));

      DBObject group = new BasicDBObject();
      group.put("$group", groupFields);

      DBObject sortFields = new BasicDBObject();
      sortFields.put("_id.year", Integer.valueOf(1));
      sortFields.put("_id.month", Integer.valueOf(1));
      DBObject sort = new BasicDBObject();
      sort.put("$sort", sortFields);

      List<DBObject> pipeline = Arrays.asList(project0, project, group, sort);
      AggregationOutput aggOutput = collection.aggregate(pipeline);

      for (DBObject result : aggOutput.results()) {
        resultList.add(MonthOperationResumeConverter.getMonthOperationResume(result));
      }
    } finally {
      if (client != null) {
        client.close();
      }
    }

    return resultList;
  }