/**
* All headers except collection and database are non available for this operation.
*
* @param exchange
* @throws Exception
*/
protected void doAggregate(Exchange exchange) throws Exception {
DBCollection dbCol = calculateCollection(exchange);
DBObject query = exchange.getIn().getMandatoryBody(DBObject.class);
// Impossible with java driver to get the batch size and number to skip
Iterable<DBObject> dbIterator = null;
AggregationOutput aggregationResult = null;
// Allow body to be a pipeline
// @see http://docs.mongodb.org/manual/core/aggregation/
if (query instanceof BasicDBList) {
BasicDBList queryList = (BasicDBList) query;
aggregationResult =
dbCol.aggregate(
(DBObject) queryList.get(0),
queryList
.subList(1, queryList.size())
.toArray(new BasicDBObject[queryList.size() - 1]));
} else {
aggregationResult = dbCol.aggregate(query);
}
dbIterator = aggregationResult.results();
Message resultMessage = prepareResponseMessage(exchange, MongoDbOperation.aggregate);
resultMessage.setBody(dbIterator);
}
public static DBCollection createFeedsItemsCollectionMock(DB db) {
DBCollection feedItemsCollection = mock(DBCollection.class);
when(db.getCollection("feedItems")).thenReturn(feedItemsCollection);
DBCursor feedsItemsCursor = createCursorMock(createFeedsItemDbObjectMock());
when(feedsItemsCursor.sort(any(DBObject.class))).thenReturn(feedsItemsCursor);
when(feedsItemsCursor.limit(any(Integer.class))).thenReturn(feedsItemsCursor);
when(feedsItemsCursor.skip(any(Integer.class))).thenReturn(feedsItemsCursor);
when(feedItemsCollection.find(any(BasicDBObject.class))).thenReturn(feedsItemsCursor);
AggregationOutput aggregationOutput = Mockito.mock(AggregationOutput.class);
List<DBObject> results = new ArrayList<DBObject>();
DBObject result1 = new BasicDBObject();
result1.put("_id", "tag1");
result1.put("count", 5);
results.add(result1);
DBObject result2 = new BasicDBObject();
result2.put("_id", "tag2");
result2.put("count", 8);
results.add(result2);
when(aggregationOutput.results()).thenReturn(results);
when(feedItemsCollection.aggregate(
any(DBObject.class), any(DBObject.class), any(DBObject.class), any(DBObject.class)))
.thenReturn(aggregationOutput);
AggregationOutput countAggregate = Mockito.mock(AggregationOutput.class);
List<DBObject> countResults = new ArrayList<DBObject>();
DBObject countResult1 = new BasicDBObject();
countResult1.put("_id", "12345");
countResult1.put("count", 5);
countResults.add(countResult1);
when(countAggregate.results()).thenReturn(countResults);
when(feedItemsCollection.aggregate(any(DBObject.class), any(DBObject.class)))
.thenReturn(countAggregate);
DBObject feedItem = new BasicDBObject();
feedItem.put("_id", "52ffe096e81f7a9bb906b6f9");
feedItem.put("categoryId", "5581e900d4c6cda2d04d0a98");
feedItem.put("feedId", "55893938e4b0c822dc99fa2f");
feedItem.put("delete", true);
feedItem.put("description", "description");
feedItem.put(
"link", "http://thedailyedge.thejournal.ie/irish-supermarket-fails-2176781-Jun2015/");
feedItem.put("imageUrl", "http://thedailyedge.thejournal.ie/image.jpg");
feedItem.put("read", true);
feedItem.put("source", "The Daily Edge");
feedItem.put("title", "title");
feedItem.put("username", "bob");
feedItem.put("pubDate", new Date());
when(feedItemsCollection.findOne(any(DBObject.class))).thenReturn(feedItem);
return feedItemsCollection;
}
public List<DBObject> getSourceBrowsers() {
DBObject groupFields = new BasicDBObject("_id", "$device");
groupFields.put("sum", new BasicDBObject("$sum", "$sum"));
DBObject group = new BasicDBObject("$group", groupFields);
DBObject sort = new BasicDBObject("$sort", new BasicDBObject("sum", 1));
List<DBObject> pipeline = Arrays.asList(group, sort);
// allowDiskUse
AggregationOptions options =
AggregationOptions.builder().allowDiskUse(true).batchSize(10000).build();
Cursor cursor = device.aggregate(pipeline, options);
List<DBObject> list = new ArrayList<DBObject>();
while (cursor.hasNext()) {
DBObject object = cursor.next();
DBObject newObj = new BasicDBObject();
newObj.put("device", object.get("_id"));
newObj.put("sum", object.get("sum"));
list.add(newObj);
}
cursor.close();
return list;
}
private List<String> selectViewCodes(String schema) throws UnknownHostException {
// Initiate variables
List<String> l = new ArrayList<String>();
MongoDBConnectionManager mgr = MongoDBConnectionManager.getInstance();
Mongo mongo = mgr.getMongo();
DB db = mongo.getDB(schema);
DBCollection collection = db.getCollection("views");
// Compose NoSQL query
BasicDBObject groupFiels = new BasicDBObject("_id", "$view_id");
BasicDBObject group = new BasicDBObject("$group", groupFiels);
BasicDBObject sortFiels = new BasicDBObject("_id", 1);
BasicDBObject sort = new BasicDBObject("$sort", sortFiels);
// Create the output
AggregationOutput output = collection.aggregate(group, sort);
Iterable<DBObject> results = output.results();
for (DBObject result : results) {
try {
l.add(result.get("_id").toString());
} catch (NullPointerException e) {
System.out.println("Skipping NULL");
}
}
// Return the result
return l;
}
@Override
public void execute() throws TranslatorException {
this.visitor = new MongoDBSelectVisitor(this.executionFactory, this.metadata);
this.visitor.visitNode(this.command);
if (!this.visitor.exceptions.isEmpty()) {
throw this.visitor.exceptions.get(0);
}
LogManager.logInfo(LogConstants.CTX_CONNECTOR, this.command);
DBCollection collection =
this.mongoDB.getCollection(this.visitor.mongoDoc.getTargetTable().getName());
if (collection != null) {
// TODO: check to see how to pass the hint
ArrayList<DBObject> ops = new ArrayList<DBObject>();
buildAggregate(ops, "$project", this.visitor.unwindProject); // $NON-NLS-1$
if (this.visitor.project.isEmpty()) {
throw new TranslatorException(
MongoDBPlugin.Event.TEIID18025, MongoDBPlugin.Util.gs(MongoDBPlugin.Event.TEIID18025));
}
if (this.visitor.projectBeforeMatch) {
buildAggregate(ops, "$project", this.visitor.project); // $NON-NLS-1$
}
if (!this.visitor.unwindTables.isEmpty()) {
for (String ref : this.visitor.unwindTables) {
buildAggregate(ops, "$unwind", "$" + ref); // $NON-NLS-1$ //$NON-NLS-2$
}
}
buildAggregate(ops, "$match", this.visitor.match); // $NON-NLS-1$
buildAggregate(ops, "$group", this.visitor.group); // $NON-NLS-1$
buildAggregate(ops, "$match", this.visitor.having); // $NON-NLS-1$
if (!this.visitor.projectBeforeMatch) {
buildAggregate(ops, "$project", this.visitor.project); // $NON-NLS-1$
}
buildAggregate(ops, "$sort", this.visitor.sort); // $NON-NLS-1$
buildAggregate(ops, "$skip", this.visitor.skip); // $NON-NLS-1$
buildAggregate(ops, "$limit", this.visitor.limit); // $NON-NLS-1$
try {
AggregationOptions options =
AggregationOptions.builder()
.batchSize(this.executionContext.getBatchSize())
.outputMode(AggregationOptions.OutputMode.CURSOR)
.allowDiskUse(this.executionFactory.useDisk())
.build();
this.results = collection.aggregate(ops, options);
} catch (MongoException e) {
throw new TranslatorException(e);
}
}
}
@Override
public MorphiaIterator<U, U> aggregate(
final String collectionName,
final Class<U> target,
final AggregationOptions options,
final ReadPreference readPreference) {
LOG.debug("stages = " + stages);
Cursor cursor = collection.aggregate(stages, options, readPreference);
return new MorphiaIterator<U, U>(
cursor, mapper, target, collection.getName(), mapper.createEntityCache());
}
@Override
public void getHostsByRequestType() {
final BasicDBObject match =
new BasicDBObject("$match", new BasicDBObject("Request details.Request type", HEAD));
final BasicDBObject group = new BasicDBObject("$group", new BasicDBObject("_id", "$Host"));
final AggregationOutput result = table.aggregate(match, group);
System.out.println("getHostsByRequestType: ");
for (final DBObject obj : result.results()) {
System.out.println(obj);
}
}
// 获得一天的转换漏斗
public List<DBObject> getConversionFunnel(String date) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
DBObject match = null;
try {
match = new BasicDBObject("$match", new BasicDBObject("date", sdf.parse(date)));
} catch (ParseException e) {
}
DBObject groupFields = new BasicDBObject("_id", "");
groupFields.put("step1sum", new BasicDBObject("$sum", "$step1"));
groupFields.put("step2sum", new BasicDBObject("$sum", "$step2"));
groupFields.put("step3sum", new BasicDBObject("$sum", "$step3"));
DBObject group = new BasicDBObject("$group", groupFields);
// DBObject sort = new BasicDBObject("$sort", new BasicDBObject("sum", 1));
List<DBObject> pipeline = Arrays.asList(match, group);
// allowDiskUse
AggregationOptions options =
AggregationOptions.builder().allowDiskUse(true).batchSize(10000).build();
Cursor cursor = CF.aggregate(pipeline, options);
List<DBObject> list = new ArrayList<DBObject>();
while (cursor.hasNext()) {
DBObject object = cursor.next();
DBObject newObj = new BasicDBObject();
// DBObject _id = (DBObject)object.get("_id");
// newObj.put("date",sdf.format(_id.get("$date")));
DBObject newObj1 = new BasicDBObject();
newObj1.put("step", "访问本站");
newObj1.put("num", object.get("step1sum"));
list.add(newObj1);
DBObject newObj2 = new BasicDBObject();
newObj2.put("step", "浏览商品");
newObj2.put("num", object.get("step2sum"));
list.add(newObj2);
DBObject newObj3 = new BasicDBObject();
newObj3.put("step", "完成交易");
newObj3.put("num", object.get("step3sum"));
list.add(newObj3);
}
cursor.close();
return list;
}
public void getNumOfReplyCodeByRequestTypeAggr() {
final BasicDBObject match = new BasicDBObject();
match.put("Request details.Request type", GET);
match.put("Request details.Reply code", SUCCESS_REPLY_CODE);
final BasicDBObject group = new BasicDBObject();
group.put("_id", 0);
group.put("count", new BasicDBObject("$sum", 1));
final BasicDBObject matchOp = new BasicDBObject("$match", match);
final BasicDBObject unwindOp = new BasicDBObject("$unwind", "$Request details");
final BasicDBObject groupOp = new BasicDBObject("$group", group);
final AggregationOutput result = table.aggregate(matchOp, unwindOp, matchOp, groupOp);
System.out.println("getNumOfReplyCodeByRequestType: ");
for (DBObject obj : result.results()) {
System.out.println(obj.get("count"));
}
}
public static void main(String[] args) throws UnknownHostException {
MongoClient client = new MongoClient();
DB database = client.getDB("m101");
DBCollection collection = database.getCollection("funnynumbers");
// Not necessary yet to understand this. It's just to prove that you
// are able to run a command on a mongod server
AggregationOutput output =
collection.aggregate(
new BasicDBObject(
"$group",
new BasicDBObject("_id", "$value").append("count", new BasicDBObject("$sum", 1))),
new BasicDBObject("$match", new BasicDBObject("count", new BasicDBObject("$gt", 2))),
new BasicDBObject("$sort", new BasicDBObject("_id", 1)));
int answer = 0;
for (DBObject doc : output.results()) {
answer += (Double) doc.get("_id");
}
System.out.println("THE ANSWER IS: " + answer);
}
/** @param args */
public static void main(String[] args) {
// TODO Auto-generated method stub
List<ServerAddress> seeds = new ArrayList<ServerAddress>();
try {
seeds.add(new ServerAddress("202.191.136.234", 27017));
// seeds.add(new ServerAddress("172.31.1.5", 27017));
} catch (UnknownHostException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
MongoClient client = new MongoClient(seeds);
DB db = client.getDB("netflowdb");
DBCollection collection = db.getCollection("netflow_column_list");
// System.out.println(collection.findOne());
DBObject record = collection.findOne();
System.out.println(record.get("timestamp"));
Date time = (Date) record.get("timestamp");
System.out.println(time.getTime());
Date time1 = new Date(time.getTime());
System.out.println("new time " + time1);
System.exit(1);
DBCollection output = db.getCollection("agg_test");
// System.out.println("count : " + collection.count());
Calendar startTime = Calendar.getInstance();
startTime.add(Calendar.HOUR_OF_DAY, -1);
startTime.set(Calendar.MINUTE, 0);
startTime.set(Calendar.SECOND, 0);
startTime.set(Calendar.MILLISECOND, 0);
DBObject match_query =
new BasicDBObject()
.append("_id.dikey.routerip", "172.31.4.79")
// .append("_id.dikey.ifindex", 5)
.append("_id.rd", startTime.getTime());
DBObject match = new BasicDBObject("$match", match_query);
System.out.println(startTime.getTime());
System.out.println("find-match : " + collection.find(match_query).count());
DBObject group =
new BasicDBObject(
"$group",
new BasicDBObject()
.append(
"_id",
new BasicDBObject()
.append("dikey", "$_id.dikey")
.append("customerid", "$_id.customerid")
.append("dstport", "$_id.dstport")
.append("protocol", "$_id.protocol")
.append("hourCount", "$_id.hourCount")
.append("rd", "$_id.rd"))
.append("total", new BasicDBObject("$sum", "$value")));
DBObject sort = new BasicDBObject("$sort", new BasicDBObject("total", -1));
DBObject limit = new BasicDBObject("$limit", 3);
List<DBObject> pipeline = Arrays.asList(group, sort);
// AggregationOutput result = collection.aggregate(match, (DBObject[]) pipeline.toArray());
AggregationOutput result = collection.aggregate(match, group, sort, limit);
System.out.println(result.getCommandResult());
output.insert((List<DBObject>) result.results());
}
public List<MonthOperationResume> getAllMonthOperations() throws UnknownHostException {
List<MonthOperationResume> resultList = new ArrayList<>();
MongoClient client = null;
try {
client = getClient();
DBCollection collection =
super.getCollection(client, OperationConverter.OPERATION_COLLECTION_NAME);
DBObject project0Fields = new BasicDBObject();
BasicDBList add2HoursList = new BasicDBList();
add2HoursList.add("$dateOperation");
add2HoursList.add(2 * 60 * 60 * 1_000);
DBObject add2Hours = new BasicDBObject("$add", add2HoursList);
// DBObject newDateOperation = new BasicDBObject("dateOperation", add2Hours);
project0Fields.put("dateOperation", add2Hours);
project0Fields.put("amount", "$amount");
project0Fields.put("assistantOneAmount", "$assistantOneAmount");
project0Fields.put("assistantTwoAmount", "$assistantTwoAmount");
project0Fields.put("paymentRecievedDate", "$paymentRecievedDate");
project0Fields.put("assistantsPaidDate", "$assistantsPaidDate");
DBObject projectFields = new BasicDBObject("_id", 0);
DBObject keyObject = new BasicDBObject();
keyObject.put("month", new BasicDBObject("$month", "$dateOperation"));
keyObject.put("year", new BasicDBObject("$year", "$dateOperation"));
projectFields.put("key", keyObject);
projectFields.put("amount", "$amount");
projectFields.put("assistantOneAmount", "$assistantOneAmount");
projectFields.put("assistantTwoAmount", "$assistantTwoAmount");
BasicDBList conditionAssistantsPaidList = new BasicDBList();
conditionAssistantsPaidList.add("$assistantsPaidDate");
conditionAssistantsPaidList.add(1);
conditionAssistantsPaidList.add(0);
projectFields.put(
"numberAssistantPaid", new BasicDBObject("$cond", conditionAssistantsPaidList));
BasicDBList assistantOneGtList = new BasicDBList();
assistantOneGtList.add("$assistantOneAmount");
assistantOneGtList.add(0);
BasicDBList assistantTwoGtList = new BasicDBList();
assistantTwoGtList.add("$assistantTwoAmount");
assistantTwoGtList.add(0);
BasicDBList orList = new BasicDBList();
orList.add(new BasicDBObject("$gt", assistantOneGtList));
orList.add(new BasicDBObject("$gt", assistantTwoGtList));
BasicDBObject orClause = new BasicDBObject();
orClause.put("$or", orList);
BasicDBList conditionNumberAssistedList = new BasicDBList();
conditionNumberAssistedList.add(orClause);
conditionNumberAssistedList.add(1);
conditionNumberAssistedList.add(0);
projectFields.put(
"numberAssistedOperations", new BasicDBObject("$cond", conditionNumberAssistedList));
BasicDBList conditionPaymentsRecievedList = new BasicDBList();
conditionPaymentsRecievedList.add("$paymentRecievedDate");
conditionPaymentsRecievedList.add(1);
conditionPaymentsRecievedList.add(0);
projectFields.put(
"numberPaymentsRecieved", new BasicDBObject("$cond", conditionPaymentsRecievedList));
DBObject project0 = new BasicDBObject();
project0.put("$project", project0Fields);
DBObject project = new BasicDBObject();
project.put("$project", projectFields);
DBObject groupFields = new BasicDBObject();
groupFields.put("_id", "$key");
groupFields.put("amount", new BasicDBObject("$sum", "$amount"));
groupFields.put("assistantOneAmount", new BasicDBObject("$sum", "$assistantOneAmount"));
groupFields.put("assistantTwoAmount", new BasicDBObject("$sum", "$assistantTwoAmount"));
groupFields.put("numberAssistantPaid", new BasicDBObject("$sum", "$numberAssistantPaid"));
groupFields.put(
"numberAssistedOperations", new BasicDBObject("$sum", "$numberAssistedOperations"));
groupFields.put("numberOperations", new BasicDBObject("$sum", 1));
groupFields.put(
"numberPaymentsRecieved", new BasicDBObject("$sum", "$numberPaymentsRecieved"));
DBObject group = new BasicDBObject();
group.put("$group", groupFields);
DBObject sortFields = new BasicDBObject();
sortFields.put("_id.year", Integer.valueOf(1));
sortFields.put("_id.month", Integer.valueOf(1));
DBObject sort = new BasicDBObject();
sort.put("$sort", sortFields);
List<DBObject> pipeline = Arrays.asList(project0, project, group, sort);
AggregationOutput aggOutput = collection.aggregate(pipeline);
for (DBObject result : aggOutput.results()) {
resultList.add(MonthOperationResumeConverter.getMonthOperationResume(result));
}
} finally {
if (client != null) {
client.close();
}
}
return resultList;
}