/**
* 条件检索</br> 日期:2014-3-11 下午02:16:29
*
* @throws IOException
*/
@Test
public void query1() throws IOException {
long startTiem = System.currentTimeMillis();
BasicDBObject query = new BasicDBObject();
query.put("secretStatus", 2);
// query.put("secretLevel", 2);
// query.put("timelimitType", 2); //升序asc 1 降序desc -1
DBCursor cursor = bakFiles.find(query).sort(new BasicDBObject().append("secretStatus", 1));
System.out.println(cursor.size());
// while (cursor.hasNext()) {
// System.out.println(cursor.next());
// }
long endTime = System.currentTimeMillis();
System.out.println(
"查询 secretStatus，secretLevel，timelimitType = 2 的数据 总共有 "
+ cursor.size()
+ " 数据 查询完毕"
+ "本次操作消费 "
+ (endTime - startTiem)
+ " 毫秒");
// ReadWriteFile.creatTxtFile("记录");
// ReadWriteFile.writeTxtFile("查询 secretStatus，secretLevel，timelimitType = 2 的数据 总共有
// "+cursor.size()+" 数据 查询完毕"
// +"本次操作消费 "+(endTime-startTiem)+" 毫秒");
}
@Override
public String Json(String nombreDB, int clienteId) throws UnknownHostException, JSONException {
// TODO Auto-generated method stub
MongoClient mongoClient = new MongoClient("localhost", 27017);
DB base = mongoClient.getDB(nombreDB);
DBCollection collection = base.getCollection("Json");
BasicDBObject query = new BasicDBObject();
query.put("id", clienteId);
DBCursor cursor = collection.find(query);
if (cursor.size() == 0) {
// System.out.println("********************\n");
// System.out.println("No existe el cliente, no se puede ingresar json");
// System.out.println("********************\n");
return "No existe el cliente, no se puede ingresar json";
}
// Existe el cliente
DBObject objeto = (DBObject) cursor.next();
DBObject json = (DBObject) objeto.get("json");
// DBObject j = JSON.parse(json.toString());
JSONObject aj = new JSONObject(json.toString());
return aj.toString();
}
/**
* 获取数据库总条数
*
* @author felixsion *
*/
@Override
public int getDataBaseCollectionToyalNum() {
DBCursor cur = collection.find();
int num = cur.size();
cur.close();
return num;
}
private void awaitUntilMessageIsPersisted(DBObject query) throws Exception {
await()
.atMost(ONE_SECOND)
.until(
() -> {
DBCursor cursor = database.getCollection(COLLECTION_PUBLISHED_NAME).find(query);
return cursor.size() == 1;
});
}
public boolean ExisteCliente(String nombreDB, int clienteId) throws UnknownHostException {
MongoClient mongoClient = new MongoClient("localhost", 27017);
DB base = mongoClient.getDB(nombreDB);
DBCollection collection = base.getCollection("Json");
BasicDBObject query = new BasicDBObject();
query.put("id", clienteId);
DBCursor cursor = collection.find(query);
return cursor.size() > 0;
}
protected void doFindAll(Exchange exchange) throws Exception {
DBCollection dbCol = calculateCollection(exchange);
// do not use getMandatoryBody, because if the body is empty we want to retrieve all objects in
// the collection
DBObject query = null;
// do not run around looking for a type converter unless there is a need for it
if (exchange.getIn().getBody() != null) {
query = exchange.getIn().getBody(DBObject.class);
}
DBObject fieldFilter =
exchange.getIn().getHeader(MongoDbConstants.FIELDS_FILTER, DBObject.class);
// get the batch size and number to skip
Integer batchSize = exchange.getIn().getHeader(MongoDbConstants.BATCH_SIZE, Integer.class);
Integer numToSkip = exchange.getIn().getHeader(MongoDbConstants.NUM_TO_SKIP, Integer.class);
Integer limit = exchange.getIn().getHeader(MongoDbConstants.LIMIT, Integer.class);
DBObject sortBy = exchange.getIn().getHeader(MongoDbConstants.SORT_BY, DBObject.class);
DBCursor ret = null;
try {
if (query == null && fieldFilter == null) {
ret = dbCol.find(new BasicDBObject());
} else if (fieldFilter == null) {
ret = dbCol.find(query);
} else {
ret = dbCol.find(query, fieldFilter);
}
if (sortBy != null) {
ret.sort(sortBy);
}
if (batchSize != null) {
ret.batchSize(batchSize.intValue());
}
if (numToSkip != null) {
ret.skip(numToSkip.intValue());
}
if (limit != null) {
ret.limit(limit.intValue());
}
Message resultMessage = prepareResponseMessage(exchange, MongoDbOperation.findAll);
resultMessage.setBody(ret.toArray());
resultMessage.setHeader(MongoDbConstants.RESULT_TOTAL_SIZE, ret.count());
resultMessage.setHeader(MongoDbConstants.RESULT_PAGE_SIZE, ret.size());
} finally {
// make sure the cursor is closed
if (ret != null) {
ret.close();
}
}
}
/**
* Return true if the key exists in cache
*
* @param key
* @return
*/
public boolean exists(String key) {
// clean
clearExpired();
// get the item
DBCursor obj =
_coll.find(new BasicDBObject("key", key.toLowerCase()), new BasicDBObject("key", 1));
if (obj.size() == 0) {
return false;
}
return true;
}
/** 查询指定列，分页，排序</br> 日期:2014-3-11 下午02:18:00 */
@Test
public void query2() {
DBObject query = new BasicDBObject();
query.put("secretLevel", 4);
DBObject files = new BasicDBObject();
files.put("secretStatus", true);
files.put("secretLevel", true);
files.put("username", true);
DBObject orderBy = new BasicDBObject();
orderBy.put("secretStatus", 1);
DBCursor cursor =
bakFiles.find(query, files).sort(orderBy).limit(10).skip(2); // limit 页数 skip 页码
System.out.println(cursor.size());
System.out.println(JSON.serialize(cursor));
}
/**
* 查询特定条件文件函数 查询根据 Tags（标签） Tags支持复合条件查询
*
* @author felixerio *
*/
@Override
public List<YandeCG> querySpecifiedConditionsForTags(List<String> tags) {
String query[] = tags.toArray(new String[tags.size()]);
Pattern pattern = Pattern.compile("^.*" + " chikotam" + ".*$", Pattern.CASE_INSENSITIVE);
Pattern pattern2 = Pattern.compile("^.*" + " minori" + ".*$", Pattern.CASE_INSENSITIVE);
Pattern pattern3 = Pattern.compile("^.*" + "chikotam " + ".*$", Pattern.CASE_INSENSITIVE);
Pattern pattern4 = Pattern.compile("^.*" + "minori " + ".*$", Pattern.CASE_INSENSITIVE);
DBObject queryCondition = new BasicDBObject();
BasicDBList condList = new BasicDBList();
queryCondition.put("tags", pattern);
condList.add(queryCondition);
DBObject queryCondition2 = new BasicDBObject();
BasicDBList condList2 = new BasicDBList();
queryCondition2.put("tags", pattern2);
condList.add(queryCondition2);
BasicDBObject searchCond = new BasicDBObject();
searchCond.put("$and", condList);
DBCursor cur = collection.find(searchCond);
logger.info("批量获取指定Tags数据数据任务 查询到 " + cur.size() + " 条数据");
int num = 1;
List<YandeCG> yandeCGList = new ArrayList<YandeCG>();
while (cur.hasNext()) {
YandeCG yanedCG = new YandeCG();
DBObject dBObject = cur.next();
yanedCG.setId(Integer.parseInt(dBObject.get("id").toString()));
yanedCG.setTags(dBObject.get("tags").toString());
yanedCG.setMd5(dBObject.get("md5").toString());
yanedCG.setDownComplete(Boolean.parseBoolean(dBObject.get("downComplete").toString()));
yanedCG.setSource(dBObject.get("source").toString());
yanedCG.setFile_url(dBObject.get("file_url").toString());
yanedCG.setFile_size(dBObject.get("file_size").toString());
yandeCGList.add(yanedCG);
logger.info("批量获取指定Tags数据数据任务 完成处理第 " + num + " 个任务");
num++;
}
cur.close();
return yandeCGList;
}
@Test
public void testUpdate() throws Exception {
BasicDBObject obj1 = new BasicDBObject().append("f1", "a").append("f2", "value1");
BasicDBObject obj2 = new BasicDBObject().append("f1", "b").append("f2", "value2");
insertData("testUpdate", obj1, obj2);
String[] input = {"a\tnewValue1\t1", "b\tnewValue2\t2"};
Util.createLocalInputFile("simple_input", input);
pigServerLocal = new PigServer(ExecType.LOCAL);
pigServerLocal.registerQuery(
"A = LOAD 'simple_input' as (f1:chararray, f2:chararray, f3:int);");
pigServerLocal.registerQuery(
String.format(
"STORE A INTO 'mongodb://localhost:27017/%s.%s' USING com.mongodb.hadoop.pig.MongoUpdateStorage("
+ " '{f1:\"\\\\$f1\"}',"
+ " '{\\\\$set:{f2:\"\\\\$f2\", f3:\"\\\\$f3\"}}',"
+ " 'f1:chararray, f2:chararray, f3:int'"
+ ");",
dbName, "update_simple"));
pigServerLocal.setBatchOn();
pigServerLocal.executeBatch();
MongoClient mc = new MongoClient();
DBCollection col = mc.getDB(dbName).getCollection("update_simple");
DBCursor cursor = col.find();
assertEquals(2, cursor.size());
DBObject result1 = cursor.next();
assertEquals("a", result1.get("f1"));
assertEquals("newValue1", result1.get("f2"));
assertEquals(1, result1.get("f3"));
DBObject result2 = cursor.next();
assertEquals("b", result2.get("f1"));
assertEquals("newValue2", result2.get("f2"));
assertEquals(2, result2.get("f3"));
}
/** 获取没有下载的文件总素* */
public int noDownloadCompleteFileNum() {
DBCursor cur = collection.find(new BasicDBObject("downComplete", Boolean.FALSE));
return cur.size();
}
public void InitializeIndex(
boolean bDeleteDocs,
boolean bDeleteEntityFeature,
boolean bDeleteEventFeature,
boolean bRebuildDocsIndex) {
try { // create elasticsearch indexes
PropertiesManager pm = new PropertiesManager();
if (!pm.getAggregationDisabled()) {
Builder localSettingsEvent = ImmutableSettings.settingsBuilder();
localSettingsEvent.put("number_of_shards", 1).put("number_of_replicas", 0);
localSettingsEvent.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
localSettingsEvent.putArray(
"index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");
localSettingsEvent.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
localSettingsEvent.putArray(
"index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");
Builder localSettingsGaz = ImmutableSettings.settingsBuilder();
localSettingsGaz.put("number_of_shards", 1).put("number_of_replicas", 0);
localSettingsGaz.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
localSettingsGaz.putArray(
"index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");
// event feature
String eventGazMapping =
new Gson()
.toJson(
new AssociationFeaturePojoIndexMap.Mapping(),
AssociationFeaturePojoIndexMap.Mapping.class);
ElasticSearchManager eventIndex =
IndexManager.createIndex(
AssociationFeaturePojoIndexMap.indexName_,
null,
false,
null,
eventGazMapping,
localSettingsEvent);
if (bDeleteEventFeature) {
eventIndex.deleteMe();
eventIndex =
IndexManager.createIndex(
AssociationFeaturePojoIndexMap.indexName_,
null,
false,
null,
eventGazMapping,
localSettingsEvent);
}
// entity feature
String gazMapping =
new Gson()
.toJson(
new EntityFeaturePojoIndexMap.Mapping(),
EntityFeaturePojoIndexMap.Mapping.class);
ElasticSearchManager entityIndex =
IndexManager.createIndex(
EntityFeaturePojoIndexMap.indexName_,
null,
false,
null,
gazMapping,
localSettingsGaz);
if (bDeleteEntityFeature) {
entityIndex.deleteMe();
entityIndex =
IndexManager.createIndex(
EntityFeaturePojoIndexMap.indexName_,
null,
false,
null,
gazMapping,
localSettingsGaz);
}
}
// DOCS - much more complicated than anything else
boolean bPingMainIndexFailed =
!ElasticSearchManager.pingIndex(DocumentPojoIndexMap.globalDocumentIndex_);
// (ie if main doc index doesn't exist then always rebuild all indexes)
if (bPingMainIndexFailed) { // extra level of robustness... sleep for a minute then double
// check the index is really missing...
try {
Thread.sleep(60000);
} catch (Exception e) {
}
bPingMainIndexFailed =
!ElasticSearchManager.pingIndex(DocumentPojoIndexMap.globalDocumentIndex_);
}
bRebuildDocsIndex |= bPingMainIndexFailed;
createCommunityDocIndex(
DocumentPojoIndexMap.globalDocumentIndex_, null, false, true, bDeleteDocs);
createCommunityDocIndex(
DocumentPojoIndexMap.manyGeoDocumentIndex_, null, false, false, bDeleteDocs);
// Some hardwired dummy communities
createCommunityDocIndex(
"4e3706c48d26852237078005", null, true, false, bDeleteDocs); // (admin)
createCommunityDocIndex(
"4e3706c48d26852237079004", null, true, false, bDeleteDocs); // (test user)
// (create dummy index used to keep personal group aliases)
// OK, going to have different shards for different communities:
// Get a list of all the communities:
BasicDBObject query = new BasicDBObject();
BasicDBObject fieldsToDrop = new BasicDBObject("members", 0);
fieldsToDrop.put("communityAttributes", 0);
fieldsToDrop.put("userAttributes", 0);
DBCursor dbc = DbManager.getSocial().getCommunity().find(query, fieldsToDrop);
if (bRebuildDocsIndex || bDeleteDocs) {
List<DBObject> tmparray =
dbc.toArray(); // (brings the entire thing into memory so don't get cursor timeouts)
int i = 0;
System.out.println("Initializing " + dbc.size() + " indexes:");
for (int j = 0; j < 2; ++j) {
for (DBObject dbotmp : tmparray) {
if ((++i % 100) == 0) {
System.out.println("Initialized " + i + " indexes.");
}
BasicDBObject dbo = (BasicDBObject) dbotmp;
// OK, going to see if there are any sources with this group id, create a new index if
// so:
// (Don't use CommunityPojo data model here for performance reasons....
// (Also, haven't gotten round to porting CommunityPojo field access to using static
// fields))
ObjectId communityId = (ObjectId) dbo.get("_id");
boolean bPersonalGroup = dbo.getBoolean("isPersonalCommunity", false);
boolean bSystemGroup = dbo.getBoolean("isSystemCommunity", false);
ObjectId parentCommunityId = (ObjectId) dbo.get("parentId");
createCommunityDocIndex(
communityId.toString(),
parentCommunityId,
bPersonalGroup,
bSystemGroup,
bDeleteDocs,
j == 0);
} // end loop over communities
} // end loop over communities - first time parents only
} // (end if need to do big loop over all sources)
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
} // TESTED (not changed since by-eye test in Beta - retested after moving code into
@Command(
aliases = {"data"},
usage = "<player> <options>",
desc = "Displays a summary of data applicable to a player",
min = 1,
max = 2)
@CommandPermissions({"oresomedata.view"})
public void transact(CommandContext args, CommandSender sender) {
DB database = MongoDatabaseManager.mongo.getDB(MongoDatabaseManager.mongodb_db);
DBCollection playerTable = database.getCollection("players");
BasicDBObject searchQuery = new BasicDBObject();
searchQuery.put("name", args.getString(0));
DBCursor cursor = playerTable.find(searchQuery);
if (cursor.size() > 0) {
if (cursor.size() == 1) {
BasicDBObject playerDocument = (BasicDBObject) cursor.curr();
StringBuilder overview = new StringBuilder();
StringBuilder statisticString = new StringBuilder();
StringBuilder raidHouseString = new StringBuilder();
overview.append(
ChatColor.GREEN
+ "++++++++ "
+ ChatColor.AQUA
+ args.getString(0)
+ ChatColor.GREEN
+ " ++++++++\n");
overview.append(ChatColor.GOLD + "UUID: " + playerDocument.getString("uuid") + "\n");
overview.append(ChatColor.GOLD + "Coins: " + playerDocument.getString("coins") + "\n");
if (playerDocument.get("statistics") != null) {
BasicDBObject statistics = (BasicDBObject) playerDocument.get("statistics");
statisticString.append(ChatColor.GREEN + "--------- Stats --------\n");
statisticString.append(ChatColor.GOLD + "Kills: " + statistics.getString("kills") + "\n");
statisticString.append(
ChatColor.GOLD + "Deaths: " + statistics.getString("deaths") + "\n");
statisticString.append(
ChatColor.GOLD + "K/D Ratio: " + statistics.getString("ks") + "\n");
statisticString.append(
ChatColor.GOLD + "FFA Wins: " + statistics.getString("ffawins") + "\n");
statisticString.append(
ChatColor.GOLD + "Infection Wins: " + statistics.getString("infectionwins") + "\n");
statisticString.append(
ChatColor.GOLD
+ "Highest Streak: "
+ statistics.getString("highestkillstreak")
+ "\n");
// statisticString.append(ChatColor.GREEN + "-------- -------- --------\n");
}
if (playerDocument.get("raidhouse") != null) {
BasicDBObject raidHouse = (BasicDBObject) playerDocument.get("raidhouse");
raidHouseString.append(ChatColor.GREEN + "-------- RaidHouse --------\n");
raidHouseString.append(
ChatColor.GOLD + "Recognition: " + raidHouse.getString("recognition") + "\n");
raidHouseString.append(ChatColor.GOLD + "Tier: " + raidHouse.getString("tier") + "\n");
raidHouseString.append(
ChatColor.GOLD + "Total Raids: " + raidHouse.getString("totalraids") + "\n");
raidHouseString.append(
ChatColor.GOLD
+ "Successful Raids: "
+ raidHouse.getString("successfulraids")
+ "\n");
raidHouseString.append(
ChatColor.GOLD + "Failed Raids: " + raidHouse.getString("failedraids") + "\n");
raidHouseString.append(
ChatColor.GOLD + "Reroll Amount: " + raidHouse.getString("rerollamount") + "\n");
// raidHouseString.append(ChatColor.GREEN + "-------- -------- --------\n");
}
// sender.sendMessage(ChatColor.GREEN + "++++++++ ++++++++ ++++++++");
SortedMap<Integer, String> map = new TreeMap<Integer, String>();
if (overview.toString().length() > 0) {
map.put(1, overview.toString());
}
if (statisticString.toString().length() > 0) {
map.put(2, statisticString.toString());
}
if (raidHouseString.length() > 0) {
map.put(3, raidHouseString.toString());
}
if (args.getString(1) != null) {
paginate(sender, map, Integer.parseInt(args.getString(1)), 7);
} else {
paginate(sender, map, 1, 7);
}
} else {
sender.sendMessage(
ChatColor.RED
+ "More than one result was found! Are you sure you spelt the player's name correctly?");
}
} else {
}
}
@Override
public QueryResult getVariantsHistogramByRegion(
Region region, String sourceId, boolean histogramLogarithm, int histogramMax) {
QueryResult<ObjectMap> queryResult =
new QueryResult<>(
String.format("%s:%d-%d", region.getChromosome(), region.getStart(), region.getEnd()));
List<ObjectMap> data = new ArrayList<>();
String startRow = buildRowkey(region.getChromosome(), Long.toString(region.getStart()));
String stopRow = buildRowkey(region.getChromosome(), Long.toString(region.getEnd()));
long startTime = System.currentTimeMillis();
long startDbTime = System.currentTimeMillis();
BasicDBObject query =
new BasicDBObject("position", new BasicDBObject("$gte", startRow).append("$lte", stopRow))
.append("studies.studyId", sourceId);
DBCollection collection = db.getCollection("variants");
DBCursor queryResults = collection.find(query);
queryResult.setDbTime(System.currentTimeMillis() - startDbTime);
int resultSize = queryResults.size();
if (resultSize > histogramMax) { // Need to group results to fit maximum size of the histogram
int sumChunkSize = resultSize / histogramMax;
int i = 0, j = 0;
int featuresCount = 0;
ObjectMap item = null;
for (DBObject result : queryResults) {
// featuresCount += result.getInt("features_count");
// if (i == 0) {
// item = new ObjectMap("chromosome", result.getString("chromosome"));
// item.put("chunkId", result.getInt("chunk_id"));
// item.put("start", result.getInt("start"));
// } else if (i == sumChunkSize - 1 || j == resultSize - 1) {
// if (histogramLogarithm) {
// item.put("featuresCount", (featuresCount > 0) ?
// Math.log(featuresCount) : 0);
// } else {
// item.put("featuresCount", featuresCount);
// }
// item.put("end", result.getInt("end"));
// data.add(item);
// i = -1;
// featuresCount = 0;
// }
// j++;
// i++;
}
} else {
for (DBObject result : queryResults) {
// ObjectMap item = new ObjectMap("chromosome",
// result.getString("chromosome"));
// item.put("chunkId", result.getInt("chunk_id"));
// item.put("start", result.getInt("start"));
// if (histogramLogarithm) {
// int features_count = result.getInt("features_count");
// result.put("featuresCount", (features_count > 0) ?
// Math.log(features_count) : 0);
// } else {
// item.put("featuresCount", result.getInt("features_count"));
// }
// item.put("end", result.getInt("end"));
// data.add(item);
}
}
queryResult.setResult(data);
queryResult.setNumResults(data.size());
queryResult.setTime(System.currentTimeMillis() - startTime);
return queryResult;
}
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String email = request.getParameter("email");
String password = request.getParameter("password");
String verifypassword = request.getParameter("verifypassword");
Map<String, String> myResponse = new HashMap<String, String>();
PrintWriter out = response.getWriter();
if (email.matches(
"^[_A-Za-z0-9-\\+]+(\\.[_A-Za-z0-9-]+)*@"
+ "[A-Za-z0-9-]+(\\.[A-Za-z0-9]+)*(\\.[A-Za-z]{2,})$")) // make sure email is properly
// formatted
{
try {
MongoURI mongoURI = new MongoURI(System.getenv("MONGOHQ_URL"));
DB db = mongoURI.connectDB(); // instance of databse
db.authenticate(mongoURI.getUsername(), mongoURI.getPassword()); // authenticates d
// Set<string> accounts = db.getCollectionName("accounts");
// Mongo mongo = new Mongo("localhost", 27017); //creates new instance of mongo
// DB db = mongo.getDB("fourup"); //gets fourup database
DBCollection accounts = db.getCollection("accounts"); // creates collection for accounts
BasicDBObject query = new BasicDBObject(); // creates a basic object named query
query.put("email", email); // sets email to email
DBCursor cursor = accounts.find(query);
if (cursor.size() > 0) // check if email has already been registered
{
myResponse.put("Status", "Error");
myResponse.put("Error", "Account already exists using this email address.");
} else // since email doesn't currently exist in DB, go ahead and register user
{
if (password.equals(
verifypassword)) // check that both of the passwords entered match each other
{
BasicDBObject document = new BasicDBObject();
int salt = getSalt();
String hpass = passwrdHash(password, salt);
document.put("email", email);
document.put("salt", salt);
document.put("password", hpass); // this is where we need to hash the password
accounts.insert(document);
myResponse.put("Status", "Sucess");
myResponse.put("Sucess", "Account has been Created");
AccountObject user = new AccountObject(email, hpass);
// set session
HttpSession session = request.getSession();
session.setAttribute("currentUser", email);
// return cookie
Cookie cookie = new Cookie("fourupCookie", email); // add the login information here
response.addCookie(cookie);
// redirect to homepage
String message = "this is a test";
myResponse.put("html", "<html></html>");
response.setContentType("application/json");
response.setStatus(HttpServletResponse.SC_OK);
// response.sendRedirect("index.html"); //should add check to index page for cookie with
// login information
} else {
myResponse.put("Status", "Failed");
myResponse.put("Failed", "Passwords do not match.");
}
}
} catch (MongoException e) {
out.write(e.getMessage());
}
} else {
myResponse.put("Status", "Invalid");
myResponse.put(
"Invalid", "The email address has not been entered correctly."); // should output error
}
String strResponse = new Gson().toJson(myResponse);
response.getWriter().write(strResponse);
response.getWriter().close();
}
