/**
* All headers except collection and database are non available for this operation.
*
* @param exchange
* @throws Exception
*/
protected void doAggregate(Exchange exchange) throws Exception {
DBCollection dbCol = calculateCollection(exchange);
DBObject query = exchange.getIn().getMandatoryBody(DBObject.class);
// Impossible with java driver to get the batch size and number to skip
Iterable<DBObject> dbIterator = null;
AggregationOutput aggregationResult = null;
// Allow body to be a pipeline
// @see http://docs.mongodb.org/manual/core/aggregation/
if (query instanceof BasicDBList) {
BasicDBList queryList = (BasicDBList) query;
aggregationResult =
dbCol.aggregate(
(DBObject) queryList.get(0),
queryList
.subList(1, queryList.size())
.toArray(new BasicDBObject[queryList.size() - 1]));
} else {
aggregationResult = dbCol.aggregate(query);
}
dbIterator = aggregationResult.results();
Message resultMessage = prepareResponseMessage(exchange, MongoDbOperation.aggregate);
resultMessage.setBody(dbIterator);
}
public void ExportMongoCollection(
String mongoCollectionName,
String xmlCollectionName,
String xmlOutputFileName,
HashSet fields)
throws Exception {
BasicDBList mongoCollection = GetDBList(mongoCollectionName, null);
try {
DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
// root elements
Document doc = docBuilder.newDocument();
Element rootElement = doc.createElement(xmlCollectionName);
doc.appendChild(rootElement);
for (int i = 0; i < mongoCollection.size(); i++) {
Element thisElement =
doc.createElement(
mongoCollectionName); // the xml nodes are named after the mongoColleciton
BasicDBObject record = (BasicDBObject) mongoCollection.get(i);
Object attributeArray[] = fields.toArray();
for (int j = 0; j < fields.size(); j++) {
String attribName = (String) attributeArray[j];
String attribValue = record.getString(attribName);
// Attr thisAttr = doc.createAttribute(attribName);
// thisAttr.setValue(attribValue);
thisElement.setAttribute(attribName, attribValue);
}
rootElement.appendChild(thisElement);
}
// write the content into xml file
TransformerFactory transformerFactory = TransformerFactory.newInstance();
Transformer transformer = transformerFactory.newTransformer();
DOMSource source = new DOMSource(doc);
String filePath = System.getProperty("user.home") + "/" + xmlOutputFileName;
File f = new File(filePath);
// StreamResult result = new StreamResult(new File("/" + xmlOutputFileName));
StreamResult result = new StreamResult(f.toURI().getPath());
// Output to console for testing
// StreamResult result = new StreamResult(System.out);
transformer.transform(source, result);
System.out.println("File saved!");
} catch (ParserConfigurationException pce) {
pce.printStackTrace();
} catch (TransformerException tfe) {
tfe.printStackTrace();
}
}
@Override
public Object getObject(String columnLabel) throws SQLException {
if (isSum) {
if (isGroupBy) {
Object ob = dblist.get(_row - 1);
if (ob instanceof DBObject) {
return ((DBObject) ob).get(columnLabel);
} else {
return "0";
}
} else {
return this._sum;
}
} else return this._cur.get(columnLabel);
}
public String getLOVId(String LovType, String GlobalVal) throws Exception {
String retVal = new ObjectId().toString();
QueryBuilder qm =
new QueryBuilder()
.and(
new QueryBuilder().start().put("LovType").is(LovType).get(),
new QueryBuilder().start().put("GlobalVal").is(GlobalVal).get());
BasicDBObject query = new BasicDBObject();
query.putAll(qm.get());
BasicDBList lovItem = (BasicDBList) GetDBList("rwLov", query);
if (lovItem != null && lovItem.size() > 0) {
BasicDBObject result = (BasicDBObject) lovItem.get(0);
retVal = result.get("_id").toString();
}
return retVal;
}
/** @see DATAMONGO-812 */
@Test
public void
updateMapperShouldRetainClassInformationForPushCorrectlyWhenCalledWithEachUsingCustomTypes() {
Update update = new Update().push("models").each(new ListModel("spring", "data", "mongodb"));
DBObject mappedObject =
mapper.getMappedObject(
update.getUpdateObject(), context.getPersistentEntity(ModelWrapper.class));
DBObject push = getAsDBObject(mappedObject, "$push");
DBObject model = getAsDBObject(push, "models");
BasicDBList each = getAsDBList(model, "$each");
assertThat(
((DBObject) each.get(0)).get("_class").toString(), equalTo(ListModel.class.getName()));
}
/** @see DATAMONGO-812 */
@SuppressWarnings({"unchecked", "rawtypes"})
@Test
public void updateMapperShouldConvertPushCorrectlyWhenCalledWithEachUsingCustomTypes() {
Update update = new Update().push("models").each(new ListModel("spring", "data", "mongodb"));
DBObject mappedObject =
mapper.getMappedObject(
update.getUpdateObject(), context.getPersistentEntity(ModelWrapper.class));
DBObject push = getAsDBObject(mappedObject, "$push");
DBObject model = getAsDBObject(push, "models");
BasicDBList each = getAsDBList(model, "$each");
BasicDBList values = getAsDBList((DBObject) each.get(0), "values");
assertThat(
values.toMap(), (Matcher) allOf(hasValue("spring"), hasValue("data"), hasValue("mongodb")));
}
public String getCurrentToken(String msisdn, DB userDB) {
try {
DBCollection collection = userDB.getCollection(HikeConstants.MONGO_USERS_COLLECTION);
BasicDBObject query = new BasicDBObject();
query.put(HikeConstants.MSISDN, msisdn);
DBObject result = collection.findOne(query);
if (result == null) {
return null;
}
BasicDBList deviceList = (BasicDBList) result.get("devices");
BasicDBObject device = (BasicDBObject) deviceList.get(0);
return device.get("token").toString();
} catch (Exception e) {
return null;
}
}
public MultiParameterPanel(
Core core,
DBObject data,
int minNb,
int maxNb,
Dimension minimumSize,
PanelDisplayer panelDisplayer,
boolean enableTest,
Class<T> clazz) {
this.core = core;
this.enableTest = enableTest;
this.minNbRows = maxNb == 1 ? 1 : Math.max(2, minimumSize.height / 40);
this.maxNb = maxNb;
this.minNb = minNb;
this.panelDisplayer = panelDisplayer;
this.clazz = clazz;
this.data = data;
this.panelElements = new ArrayList<PanelElementAbstract>();
listLayout = new GridLayout(0, 1, 5, 0);
this.listPanel = new JPanel(listLayout);
this.listPanel.setMinimumSize(minimumSize);
this.add = new JButton("Add");
if (maxNb > 1) {
Box addJP = Box.createHorizontalBox();
addJP.add(Box.createHorizontalStrut(5));
addJP.add(add);
addJP.add(Box.createHorizontalGlue());
listPanel.add(addJP);
add.addActionListener(this);
}
if (this.data != null && maxNb > 1) {
BasicDBList list = (BasicDBList) data;
for (int i = 0; i < list.size(); i++) {
addElement((BasicDBObject) list.get(i), i);
}
} else if (maxNb == 1) {
if (this.data != null) addElement((BasicDBObject) data, 0);
else addElement(null, 0);
} else if (minNb > 0) {
for (int i = 0; i < minNb; i++) {
addElement(null, i);
}
}
}
/**
* Reads the given {@link BasicDBList} into a collection of the given {@link TypeInformation}.
*
* @param targetType must not be {@literal null}.
* @param sourceValue must not be {@literal null}.
* @param path must not be {@literal null}.
* @return the converted {@link Collection} or array, will never be {@literal null}.
*/
private Object readCollectionOrArray(
TypeInformation<?> targetType, BasicDBList sourceValue, ObjectPath path) {
Assert.notNull(targetType, "Target type must not be null!");
Assert.notNull(path, "Object path must not be null!");
Class<?> collectionType = targetType.getType();
if (sourceValue.isEmpty()) {
return getPotentiallyConvertedSimpleRead(new HashSet<Object>(), collectionType);
}
TypeInformation<?> componentType = targetType.getComponentType();
Class<?> rawComponentType = componentType == null ? null : componentType.getType();
collectionType =
Collection.class.isAssignableFrom(collectionType) ? collectionType : List.class;
Collection<Object> items =
targetType.getType().isArray()
? new ArrayList<Object>()
: CollectionFactory.createCollection(
collectionType, rawComponentType, sourceValue.size());
for (int i = 0; i < sourceValue.size(); i++) {
Object dbObjItem = sourceValue.get(i);
if (dbObjItem instanceof DBRef) {
items.add(
DBRef.class.equals(rawComponentType)
? dbObjItem
: read(componentType, readRef((DBRef) dbObjItem), path));
} else if (dbObjItem instanceof DBObject) {
items.add(read(componentType, (DBObject) dbObjItem, path));
} else {
items.add(getPotentiallyConvertedSimpleRead(dbObjItem, rawComponentType));
}
}
return getPotentiallyConvertedSimpleRead(items, targetType.getType());
}
public void clearDangerMultiBind() {
DBCollection dc = MongoUtil.getInstance().getCollection("Bindings");
DBCursor c = dc.find();
JSONMessageBuilder bd = new JSONMessageBuilder();
bd.setContent(
"由于我们发现您绑定了多个不同姓名的账号，为保护个人隐私我们已经将您所有的绑定信息清空，如需继续使用请重新绑定自己的UIS账号。我们不推荐帮助其他人查询个人信息。");
while (c.hasNext()) {
DBObject obj = c.next();
try {
BasicDBList binds = (BasicDBList) obj.get("binds");
if (binds.size() > 1) {
Object name = null;
for (int i = 0; i < binds.size(); i++) {
DBObject u = (DBObject) binds.get(i);
if (i > 0) {
if (u.get("username") == null || !u.get("username").equals(name)) {
dc.remove(obj);
System.out.println(obj);
bd.set("touser", obj.get("openid"));
System.out.println(
CommonUtil.postWebRequest(
"https://api.weixin.qq.com/cgi-bin/message/custom/"
+ "send?access_token="
+ AccessTokenHelper.getInstance().getToken(AccessTokenHelper.WEIXIN),
WeixinMessageHelper.msg2jsonstr(bd.getMessage()).getBytes("utf-8"),
"application/json; charset=utf-8"));
}
break;
}
name = u.get("username");
}
}
} catch (Exception ex) {
}
}
}
public static void run(String term, DB db) throws IOException {
BasicDBList result = new BasicDBList();
DBCollection index = db.getCollection("index");
DBObject object = index.findOne(new BasicDBObject("word", term));
BasicDBList docList = (BasicDBList) object.get("document");
List<BasicDBObject> indexArr = new ArrayList<BasicDBObject>();
List<BasicDBObject> regArr = new ArrayList<BasicDBObject>();
for (int i = 0; i < docList.size(); i++) {
indexArr.add((BasicDBObject) docList.get(i));
}
DBCollection urlpages = db.getCollection("pagerank");
for (int i = 0; i < indexArr.size(); i++) {
regArr.add(
(BasicDBObject)
urlpages.findOne(new BasicDBObject("Hash", indexArr.get(i).get("docHash"))));
}
writeFile(terribleSort(indexArr, regArr, "wm"), "search_weighted_mean");
writeFile(terribleSort(indexArr, regArr, "hm"), "search_harmonic_mean");
}
@Override
public List<Post> crawl() throws Exception {
log.info("crawl " + seed + " starts...");
List<Post> posts = new ArrayList<Post>();
int page = 1;
boolean hasNextPage = true;
while (hasNextPage) {
log.info("current page " + page);
String buf = this.crawl(page);
// parse
BasicDBObject result = (BasicDBObject) JSON.parse(buf.toString());
BasicDBObject jobList = (BasicDBObject) result.get("jobList");
BasicDBList jobs = (BasicDBList) jobList.get("jobMes");
for (int i = 0; i < jobs.size(); i++) {
Post post = this.parseJob((BasicDBObject) jobs.get(i));
posts.add(post);
}
// next
if (!crawlAllPages) {
break;
}
String rows = jobList.getString("totalRows");
if (rows != null && !rows.isEmpty()) {
int total = (int) Math.ceil(Double.parseDouble(rows) / 10);
hasNextPage = page < total;
} else {
hasNextPage = false;
}
if (hasNextPage) {
page++;
Thread.sleep(Config.getInstance().getSpiderPolite());
}
}
log.info("crawl " + seed + " done");
return posts;
}
@Override
public Object decode(
final Class targetClass, final Object fromDBObject, final MappedField optionalExtraInfo) {
BasicDBList dbObject = (BasicDBList) fromDBObject;
return new ArrayFoo((String) dbObject.get(1), (String) dbObject.get(2));
}
public QueryResult getSimpleVariantsByRegion(
Region region, String sourceId, QueryOptions options) {
Long start, end, dbstart, dbend;
start = System.currentTimeMillis();
boolean includeStats;
boolean includeEffects;
if (!options.containsKey("stats") && !options.containsKey("effects")) {
includeStats = true;
includeEffects = true;
} else {
includeStats = options.containsKey("stats") && options.getBoolean("stats");
includeEffects = options.containsKey("effects") && options.getBoolean("effects");
}
QueryResult<Variant> queryResult =
new QueryResult<>(
String.format("%s:%d-%d", region.getChromosome(), region.getStart(), region.getEnd()));
List<Variant> results = new ArrayList<>();
String startRow = buildRowkey(region.getChromosome(), Long.toString(region.getStart()));
String stopRow = buildRowkey(region.getChromosome(), Long.toString(region.getEnd()));
BasicDBObject query =
new BasicDBObject("position", new BasicDBObject("$gte", startRow).append("$lte", stopRow))
.append("sources.sourceId", sourceId);
DBCollection collection = db.getCollection("variants");
dbstart = System.currentTimeMillis();
DBCursor variantInStudies = collection.find(query);
dbend = System.currentTimeMillis();
queryResult.setDbTime(dbend - dbstart);
for (DBObject result : variantInStudies) {
String[] rowkeyParts = result.get("position").toString().split("_");
String chromosome = rowkeyParts[0].replaceFirst("^0+(?!$)", "");
int position = Integer.parseInt(rowkeyParts[1]);
BasicDBList studies = (BasicDBList) result.get("sources");
BasicDBObject st = (BasicDBObject) studies.get(0);
String ref = (String) st.get("ref");
String alt = StringUtils.join((ArrayList<String>) st.get("alt"), ",");
// TODO Needs rework
Variant variant = new Variant(chromosome, position, position, ref, alt);
// Set stats informations
if (includeStats) {
VariantStats stats = new VariantStats();
BasicDBObject mongoStats = (BasicDBObject) st.get("stats");
stats.setMaf((float) (double) mongoStats.get("maf"));
stats.setMafAllele((String) mongoStats.get("alleleMaf"));
stats.setMissingGenotypes((int) mongoStats.get("missing"));
List<Genotype> genotypeCount = new ArrayList<>();
for (BasicDBObject s : (List<BasicDBObject>) mongoStats.get("genotypeCount")) {
for (Map.Entry<String, Object> entry : s.entrySet()) {
Genotype genotype = new Genotype(entry.getKey());
genotype.setCount((Integer) entry.getValue());
genotypeCount.add(genotype);
}
}
stats.setGenotypes(genotypeCount);
variant.setStats(stats);
}
// TODO Set consequence type names
if (includeEffects) {
BasicDBList mongoEffects = (BasicDBList) st.get("effects");
if (mongoEffects != null) {
for (Object e : mongoEffects) {
String effectObo = e.toString();
VariantEffect effect = new VariantEffect();
effect.setConsequenceTypeObo(effectObo);
variant.addEffect(effect);
}
}
}
results.add(variant);
}
queryResult.setResult(results);
queryResult.setNumResults(results.size());
end = System.currentTimeMillis();
queryResult.setTime(end - start);
return queryResult;
}
public Object[][] convertToKettleValue(BasicDBList mongoList, VariableSpace space)
throws KettleException {
if (mongoList == null) {
return nullResult();
}
if (m_tempParts.size() == 0) {
throw new KettleException(
BaseMessages.getString(
PKG, "MongoDbInput.ErrorMessage.MalformedPathArray")); // $NON-NLS-1$
}
String part = m_tempParts.remove(0);
if (!(part.charAt(0) == '[')) {
// we're expecting an array at this point - this document does not
// contain our field
return nullResult();
}
String index = part.substring(1, part.indexOf(']'));
if (part.indexOf(']') < part.length() - 1) {
// more dimensions to the array
part = part.substring(part.indexOf(']') + 1, part.length());
m_tempParts.add(0, part);
}
if (index.equals("*")) { // $NON-NLS-1$
// start the expansion - we delegate conversion to our subfields
Object[][] result =
new Object[mongoList.size()][m_outputRowMeta.size() + RowDataUtil.OVER_ALLOCATE_SIZE];
for (int i = 0; i < mongoList.size(); i++) {
Object element = mongoList.get(i);
for (int j = 0; j < m_subFields.size(); j++) {
MongoField sf = m_subFields.get(j);
sf.reset(space);
// what have we got?
if (element instanceof BasicDBObject) {
result[i][sf.m_outputIndex] = sf.convertToKettleValue((BasicDBObject) element);
} else if (element instanceof BasicDBList) {
result[i][sf.m_outputIndex] = sf.convertToKettleValue((BasicDBList) element);
} else {
// assume a primitive
result[i][sf.m_outputIndex] = sf.getKettleValue(element);
}
}
}
return result;
} else {
int arrayI = 0;
try {
arrayI = Integer.parseInt(index.trim());
} catch (NumberFormatException e) {
throw new KettleException(
BaseMessages.getString(
PKG, "MongoDbInput.ErrorMessage.UnableToParseArrayIndex", index)); // $NON-NLS-1$
}
if (arrayI >= mongoList.size() || arrayI < 0) {
// index is out of bounds
return nullResult();
}
Object element = mongoList.get(arrayI);
if (element == null) {
return nullResult();
}
if (element instanceof BasicDBObject) {
return convertToKettleValue(((BasicDBObject) element), space);
}
if (element instanceof BasicDBList) {
return convertToKettleValue(((BasicDBList) element), space);
}
// must mean we have a primitive here, but we're expecting to process
// more
// path so this doesn't match us - return null
return nullResult();
}
}
public RoomData(Message json, boolean isDGR) {
// TODO add in sticker placements
users = new ArrayList<User>();
if (!isDGR) {
if (json.getBoolean("success")) {
Message to;
for (Message user : json.getMessageList("users")) {
users.add(new User(user));
}
to = new Message(json.get("room"));
name = to.getString("name");
name_lower = to.getString("name_lower");
shortcut = to.getString("shortcut");
roomid = to.getString("roomid");
created = to.getDouble("created");
to = to.getSubObject("metadata");
creator = new User(to.getSubObject("creator"));
try {
song = new Song(to.getSubObject("current_song"), false);
} catch (Exception ex) {
song = null;
}
djFull = to.getBoolean("dj_full");
downvotes = to.getInt("downvotes");
privacy = to.getString("privacy");
upvotes = to.getInt("upvotes");
if (to.has("djthreshold")) pointLimit = to.getInt("djthreshold");
mods = to.getStringList("moderator_id");
djs = to.getStringList("djs");
maxDjs = to.getInt("max_djs");
currentDj = to.getString("current_dj");
listeners = to.getInt("listeners");
djCount = to.getInt("djcount");
maxUsers = to.getInt("max_size");
// votelog = temp.get("votelog");
// stickerPlacements = temp.get("sticker_placements");
// songlog = temp.get("songlog");
}
} else {
Message to = new Message(json.get("room"));
BasicDBList tl = to.getList("chatserver");
chatServer = tl.get(0).toString();
chatPort = Integer.parseInt(tl.get(1).toString());
name = to.getString("name");
name_lower = to.getString("name_lower");
shortcut = to.getString("shortcut");
roomid = to.getString("roomid");
created = to.getDouble("created");
to = to.getSubObject("metadata");
try {
song = new Song(to.getSubObject("current_song"), false);
} catch (Exception ex) {
song = null;
}
djFull = to.getBoolean("dj_full");
downvotes = to.getInt("downvotes");
privacy = to.getString("privacy");
upvotes = to.getInt("upvotes");
if (to.has("djthreshold")) pointLimit = to.getInt("djthreshold");
mods = to.getStringList("moderator_id");
djs = to.getStringList("djs");
maxDjs = to.getInt("max_djs");
currentDj = to.getString("current_dj");
listeners = to.getInt("listeners");
djCount = to.getInt("djcount");
maxUsers = to.getInt("max_size");
// votelog = temp.get("votelog");
// stickerPlacements = temp.get("sticker_placements");
// songlog = temp.get("songlog");
for (Message user : json.getMessageList("users")) {
User u = new User(user);
users.add(u);
}
}
}
