// ----This function gets as a parameter a list of terms----
// ---------the function returns a list of tweet ids from collection search_results-----------
// ----------------------------------------------------------------------------------------
public LinkedList<String> get_tweets(LinkedList<String> search_terms) {
log4j.info("starting function get_tweets");
LinkedList<String> result = new LinkedList<String>();
Iterator<String> terms = search_terms.iterator();
long curr_time = System.currentTimeMillis();
long min_time = curr_time - this.frame_time; // time below min_time will be ignored
int count_all = 0; // tweets counter
while (terms.hasNext()) {
int count = 0;
String term = terms.next();
DBObject st = new BasicDBObject();
try {
st.put("searchword", term);
DBObject obj = this.collsr.findOne(st); // look for the relevant document
String[] tweets_plus_time =
obj.get("tweets")
.toString()
.split(","); // make an array, even indexes are tweet_id's and odd indexes are their
// time
String new_string =
""; // the string to replace eventually the current field 'tweets' in the document
for (int i = 0;
i < tweets_plus_time.length - 1;
i += 2) // go over the tweet ids from the document
{
if (Long.parseLong(tweets_plus_time[i + 1])
>= min_time) // tweet time is within the time frame
{
result.add(tweets_plus_time[i]); // add tweet id to result
count++;
if (new_string == "") // add tweet information without leading comma
{
new_string += tweets_plus_time[i] + "," + tweets_plus_time[i + 1];
// count++;
} else // add tweet information with leading comma
{
new_string += "," + tweets_plus_time[i] + "," + tweets_plus_time[i + 1];
}
}
}
count_all += count;
log4j.info(count + " tweets for term: " + term);
obj.put("tweets", new_string); // replace 'tweets' field
obj.put("last_update", System.currentTimeMillis()); // update time of update
collsr.save(obj);
} catch (NullPointerException e) {
log4j.info("search_term: " + term + ", is not in collection search_results");
}
}
log4j.info("over_all there are " + count_all + " tweets to compare!!!");
log4j.info("ending function get_tweets");
return result;
}
private OperationResult optimisticCheckEtag(
DBCollection coll,
DBObject oldDocument,
ObjectId newEtag,
ObjectId requestEtag,
int httpStatusIfOk) {
Object oldEtag = oldDocument.get("_etag");
if (oldEtag == null) { // well we don't had an etag there so fine
return new OperationResult(httpStatusIfOk);
}
if (!(oldEtag instanceof ObjectId)) { // well the _etag is not an ObjectId. no check is possible
return new OperationResult(httpStatusIfOk);
}
if (requestEtag == null) {
coll.save(oldDocument);
return new OperationResult(HttpStatus.SC_CONFLICT, oldEtag);
}
if (oldEtag.equals(requestEtag)) {
return new OperationResult(httpStatusIfOk, newEtag);
} else {
// oopps, we need to restore old document
// they call it optimistic lock strategy
coll.save(oldDocument);
return new OperationResult(HttpStatus.SC_PRECONDITION_FAILED, oldEtag);
}
}
protected void doSave(Exchange exchange) throws Exception {
DBCollection dbCol = calculateCollection(exchange);
DBObject saveObj = exchange.getIn().getMandatoryBody(DBObject.class);
WriteConcern wc = extractWriteConcern(exchange);
WriteResult result = wc == null ? dbCol.save(saveObj) : dbCol.save(saveObj, wc);
prepareResponseMessage(exchange, MongoDbOperation.save);
// we always return the WriteResult, because whether the getLastError was called or not, the
// user will have the means to call it or
// obtain the cached CommandResult
processAndTransferWriteResult(result, exchange);
}
@Test
public void testSave() {
DBCollection collection = newCollection();
BasicDBObject inserted = new BasicDBObject("_id", 1);
collection.insert(inserted);
collection.save(inserted);
}
public void test() {
MongoConn conn = new MongoConn();
conn.init();
DB db = conn.getDB();
DBCollection coll = db.getCollection("testKHT");
BasicDBObject ob = new BasicDBObject();
ob.append("_id", 1);
ob.append("id", 12);
coll.save(ob);
// ----
// DBRef addressRef = new DBRef(db, "foo.bar", "202.102.40.43");
// DBObject address = addressRef.fetch();
//
// DBObject person = BasicDBObjectBuilder.start()
// .add("name", "Fred")
// .add("address", addressRef)
// .get();
//
// coll.save(person);
//
// DBObject fred = coll.findOne();
// DBRef addressObj = (DBRef)fred.get("address");
// addressObj.fetch();
}
private void addNewAdminRolesToRealm(BasicDBObject currentRealm) {
DBCollection applications = db.getCollection("applications");
DBCollection roles = db.getCollection("roles");
BasicDBObject adminApp =
(BasicDBObject)
applications.findOne(
new BasicDBObject()
.append("realmId", currentRealm.get("_id"))
.append("name", "realm-management"));
String viewIdProvidersRoleId =
insertApplicationRole(roles, AdminRoles.VIEW_IDENTITY_PROVIDERS, adminApp.getString("_id"));
String manageIdProvidersRoleId =
insertApplicationRole(
roles, AdminRoles.MANAGE_IDENTITY_PROVIDERS, adminApp.getString("_id"));
BasicDBObject adminRole =
(BasicDBObject)
roles.findOne(
new BasicDBObject()
.append("applicationId", adminApp.get("_id"))
.append("name", AdminRoles.REALM_ADMIN));
BasicDBList adminCompositeRoles = (BasicDBList) adminRole.get("compositeRoleIds");
adminCompositeRoles.add(viewIdProvidersRoleId);
adminCompositeRoles.add(manageIdProvidersRoleId);
roles.save(adminRole);
log.debugv(
"Added roles {0} and {1} to application realm-management of realm {2}",
AdminRoles.VIEW_IDENTITY_PROVIDERS,
AdminRoles.MANAGE_IDENTITY_PROVIDERS,
currentRealm.get("name"));
}
/*
* (non-Javadoc)
*
* @see org.ow2.play.service.registry.api.Registry#put(java.lang.String,
* java.lang.String)
*/
@Override
@WebMethod
public void put(String name, String url) throws RegistryException {
if (logger.isLoggable(Level.FINE)) {
logger.fine(String.format("Put url %s for name %s", url, name));
}
checkInitialized();
if (name == null || url == null) {
throw new RegistryException("Can not put null values name = %s, url = %s", name, url);
}
// update the entry if it already exists
DBObject filter = new BasicDBObject();
filter.put(NAME_KEY, name);
DBObject filtered = collection.findOne(filter);
if (filtered != null) {
filtered.put(URL_KEY, url);
collection.save(filtered);
} else {
DBObject o = new BasicDBObject();
o.put(NAME_KEY, name);
o.put(URL_KEY, url);
collection.insert(o);
}
}
public void incrementIngestedLogLines(String id, long increment) {
MongoIngestedLog log = (MongoIngestedLog) getLog(id);
if (log != null) {
log.setIngestedLines(log.getIngestedLines() + increment);
collection.save(log.getBackingDBO());
}
}
public void introduceType(SpaceTypeDescriptor typeDescriptor) {
DBCollection m = getConnection().getCollection(METADATA_COLLECTION_NAME);
BasicDBObjectBuilder builder =
BasicDBObjectBuilder.start().add(Constants.ID_PROPERTY, typeDescriptor.getTypeName());
try {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ObjectOutputStream out = new ObjectOutputStream(bos);
IOUtils.writeObject(
out, SpaceTypeDescriptorVersionedSerializationUtils.toSerializableForm(typeDescriptor));
builder.add(TYPE_DESCRIPTOR_FIELD_NAME, bos.toByteArray());
WriteResult wr = m.save(builder.get());
if (logger.isTraceEnabled()) logger.trace(wr);
indexBuilder.ensureIndexes(typeDescriptor);
} catch (IOException e) {
logger.error(e);
throw new SpaceMongoException(
"error occurs while serialize and save type descriptor: " + typeDescriptor, e);
}
}
@Override
public List<Favorite> getFavorites(String email) throws UserNotFound {
if (email == null) {
throw new IllegalArgumentException("Email cannot be null");
}
DBCollection col = db.getCollection(MongoDBConstants.COLLECTION_USERS);
// Search user by email
DBObject userDb = col.findOne(new BasicDBObject(MongoDBConstants.USER_PROP_EMAIL, email));
if (userDb == null) {
throw new UserNotFound("Unable to find user with email '" + email + "'");
}
DBObject favoritesDb = (DBObject) userDb.get(MongoDBConstants.USER_PROP_FAVORITES);
if (favoritesDb == null) {
favoritesDb = new BasicDBObject();
userDb.put(MongoDBConstants.USER_PROP_FAVORITES, favoritesDb);
col.save(userDb);
}
List<Favorite> returnValue = new ArrayList<Favorite>(favoritesDb.keySet().size());
for (String stationId : favoritesDb.keySet()) {
Favorite favorite = new Favorite();
favorite.setStationId(stationId);
DBObject favoriteItemsDb = (DBObject) favoritesDb.get(stationId);
favorite.setLastMessageId(
(Long) favoriteItemsDb.get(MongoDBConstants.USER_PROP_FAVORITE_LASTMESSAGEID));
returnValue.add(favorite);
}
return returnValue;
}
public static void testStringUserId(int max, DB db) {
String collName = "teststringid";
DBCollection coll = db.getCollection(collName);
// Setup a sharded collection
BasicDBObject command = new BasicDBObject();
command.put("shardcollection", collName);
DBObject key = new BasicDBObject();
key.put("_id", 1);
command.put("key", key);
command.put("unique", true);
db.command(command);
long startM = System.currentTimeMillis();
BasicDBObject obj = new BasicDBObject();
for (int i = 0; i < max; i++) {
obj.put("_id", "username" + i);
obj.put("test", "value-" + i);
coll.save(obj);
}
long endM = System.currentTimeMillis();
System.out.println("Insert " + max + " my objectid. time: " + (endM - startM) + " benchmark()");
CommandResult result = db.getStats();
System.out.println(result);
}
public void categorizarFotografiaUsuario(ObjectId id, String usuario, String emocion) {
DBObject object = new BasicDBObject();
object.put("_id", id);
object.put("usuario", usuario);
object.put("emocion", emocion);
bitacoraFotografiasUsuario.save(object);
}
@Override
public void setMonitoringEnabled(String id, boolean monitoringEnabled) {
MongoIngestedLog log = (MongoIngestedLog) getLog(id);
if (log != null) {
log.setIsMonitored(monitoringEnabled);
collection.save(log.getBackingDBO());
}
}
public static void main(String[] args) throws Exception {
DB db = new MongoClient().getDB("driver_test_framework");
DBCollection c = db.getCollection("test");
DBObject foo = new BasicDBObject();
foo.put("a", 2);
c.save(foo);
}
public static void saveJsonData(String dbName, String space, String data, DBObject prev) {
DB db = mongo.getDB(dbName);
DBCollection q = db.getCollection(space);
DBObject obj = (BasicDBObject) JSON.parse(data);
if (prev != null) {
obj.put("_id", prev.get("_id"));
}
q.save(obj);
}
public static void insert() throws UnknownHostException {
Mongo mongoClient = new Mongo("localhost", 27017);
DB db = mongoClient.getDB("developers");
DBCollection coll = db.getCollection("developers");
Developer developer = new Developer("Oscar", "Wilde", 50);
coll.save(developer);
}
public void create(Receipt receipt) {
try {
BasicDBObject basicDBObject = toBasicDBObject(receipt);
DB db = MongoDBCreater.getDB("MyDB");
DBCollection collection = db.getCollection("MyCollection");
collection.save(basicDBObject);
} catch (Exception e) {
e.printStackTrace();
}
}
/** {@inheritDoc} */
@Override
public void create(Feature fp) {
if (fp == null) {
throw new IllegalArgumentException("Feature cannot be null nor empty");
}
if (exist(fp.getUid())) {
throw new FeatureAlreadyExistException(fp.getUid());
}
collection.save(MAPPER.toDBObject(fp));
}
public void scrape(Calendar startDate, Calendar endDate, String symbol) {
URL url = null;
HttpURLConnection urlConn = null;
InputStreamReader inStream = null;
try {
if (logger.isTraceEnabled()) {
logger.trace(String.format("scraping %s", symbol));
}
url = buildUrl(startDate, endDate, symbol);
urlConn = (HttpURLConnection) url.openConnection();
int code = urlConn.getResponseCode();
if (code != 200) {
throw new IOException("Response not 200 OK, code=" + code + ", url=" + url);
}
inStream = new InputStreamReader(urlConn.getInputStream());
BufferedReader buff = new BufferedReader(inStream);
String stringLine;
buff.readLine(); // Read the firstLine. This is the header.
while ((stringLine = buff.readLine()) != null) {
String[] dohlcav = stringLine.split("\\,");
String date = dohlcav[0];
double open = Double.parseDouble(dohlcav[1]);
double high = Double.parseDouble(dohlcav[2]);
double low = Double.parseDouble(dohlcav[3]);
double close = Double.parseDouble(dohlcav[4]);
long volume = Long.parseLong(dohlcav[5]);
double adjClose = Double.parseDouble(dohlcav[6]);
DBObject stockDaily = new BasicDBObject();
DBObject id = new BasicDBObject();
id.put("sym", symbol);
id.put("dt", date);
stockDaily.put("_id", id);
stockDaily.put("open", open);
stockDaily.put("high", high);
stockDaily.put("low", low);
stockDaily.put("close", close);
stockDaily.put("vol", volume);
stockDaily.put("adjClose", adjClose);
collection.save(stockDaily);
}
} catch (MalformedURLException e) {
logger.error("url error", e);
} catch (IOException e) {
logger.error("io error", e);
}
}
protected <T> Key<T> save(DBCollection dbColl, T entity, WriteConcern wc) {
MappedClass mc = mapr.getMappedClass(entity);
WriteResult wr = null;
// involvedObjects is used not only as a cache but also as a list of what needs to be called for
// life-cycle methods at the end.
LinkedHashMap<Object, DBObject> involvedObjects = new LinkedHashMap<Object, DBObject>();
DBObject dbObj = entityToDBObj(entity, involvedObjects);
// try to do an update if there is a @Version field
wr = tryVersionedUpdate(dbColl, entity, dbObj, wc, db, mc);
if (wr == null)
if (wc == null) wr = dbColl.save(dbObj);
else wr = dbColl.save(dbObj, wc);
throwOnError(wc, wr);
return postSaveGetKey(entity, dbObj, dbColl, involvedObjects);
}
public void updateValidity(String user, String token) {
DBCollection coll = db().getCollection(M_TOKENS);
BasicDBObject query = new BasicDBObject();
query.put("user", user);
query.put("token", token);
DBCursor cursor = coll.find(query);
while (cursor.hasNext()) {
DBObject doc = cursor.next();
doc.put("validity", System.currentTimeMillis());
coll.save(doc, WriteConcern.SAFE);
}
}
protected <T> WriteResult tryVersionedUpdate(
DBCollection dbColl, T entity, DBObject dbObj, WriteConcern wc, DB db, MappedClass mc) {
WriteResult wr = null;
if (mc.getFieldsAnnotatedWith(Version.class).isEmpty()) return wr;
MappedField mfVersion = mc.getFieldsAnnotatedWith(Version.class).get(0);
String versionKeyName = mfVersion.getNameToStore();
Long oldVersion = (Long) mfVersion.getFieldValue(entity);
long newVersion = VersionHelper.nextValue(oldVersion);
dbObj.put(versionKeyName, newVersion);
if (oldVersion != null && oldVersion > 0) {
Object idValue = dbObj.get(Mapper.ID_KEY);
UpdateResults<T> res =
update(
find((Class<T>) entity.getClass(), Mapper.ID_KEY, idValue)
.filter(versionKeyName, oldVersion),
dbObj,
false,
false,
wc);
wr = res.getWriteResult();
if (res.getUpdatedCount() != 1)
throw new ConcurrentModificationException(
"Entity of class "
+ entity.getClass().getName()
+ " (id='"
+ idValue
+ "',version='"
+ oldVersion
+ "') was concurrently updated.");
} else if (wc == null) wr = dbColl.save(dbObj);
else wr = dbColl.save(dbObj, wc);
// update the version.
mfVersion.setFieldValue(entity, newVersion);
return wr;
}
// #### ... with some more help
public void testMongodForTests() throws IOException {
MongodForTestsFactory factory = null;
try {
factory = MongodForTestsFactory.with(Version.Main.V2_0);
Mongo mongo = factory.newMongo();
DB db = mongo.getDB("test-" + UUID.randomUUID());
DBCollection col = db.createCollection("testCol", new BasicDBObject());
col.save(new BasicDBObject("testDoc", new Date()));
} finally {
if (factory != null) factory.shutdown();
}
}
private void addAccessCodeLoginTimeout() {
DBCollection realms = db.getCollection("realms");
DBCursor realmsCursor = realms.find();
try {
while (realmsCursor.hasNext()) {
BasicDBObject realm = (BasicDBObject) realmsCursor.next();
realm.put("accessCodeLifespanLogin", 1800);
realms.save(realm);
}
} finally {
realmsCursor.close();
}
}
/**
* Test that ObjectId is getting generated even if _id is present in DBObject but it's value is
* null
*
* @throws Exception
*/
@Test
public void testIdGenerated() throws Exception {
DBObject toSave = new BasicDBObject();
toSave.put("_id", null);
toSave.put("name", "test");
Fongo fongo = newFongo();
DB fongoDB = fongo.getDB("testDB");
DBCollection collection = fongoDB.getCollection("testCollection");
collection.save(toSave);
DBObject result = collection.findOne(new BasicDBObject("name", "test"));
// default index in mongoDB
final String ID_KEY = "_id";
assertNotNull("Expected _id to be generated" + result.get(ID_KEY));
}
/**
* submit a command to the database
*
* @param database - databse name
* @param collection - collection name
* @param query - query for doc search
* @param operation - operation to be done in DB (insert, remove or update)
* @param command - command
* @param all - if true, all docs will be affected by the command
* @return
*/
public String submitDBCommand(
String database,
String collection,
String query,
String operation,
String command,
Boolean all) {
WriteResult result = null;
try {
DBCollection DBCollection = getCollection(database, collection);
if (operation.equals("insert")) {
JSONObject teste = new JSONObject(command);
Object teste1 = JSON.parse(teste.toString());
DBObject doc = (DBObject) teste1;
result = DBCollection.save(doc);
} else if (operation.equals("update")) {
DBObject queryDB = (DBObject) JSON.parse(query);
DBObject doc = (DBObject) JSON.parse(command);
if (all) {
result = DBCollection.update(queryDB, doc, false, true);
} else {
result = DBCollection.update(queryDB, doc);
}
} else if (operation.equals("remove")) {
DBObject queryDB = (DBObject) JSON.parse(query);
DBObject doc = (DBObject) JSON.parse(command);
if (all) {
result = DBCollection.update(queryDB, doc, false, true);
} else {
result = DBCollection.update(queryDB, doc);
}
}
return "Result: " + result.toString();
} catch (Exception e) {
logger.logp(
Level.SEVERE,
TAG,
"executeDBCommand",
e.getMessage()
+ " database:"
+ database
+ " collection: "
+ collection
+ ", result: "
+ result.toString());
e.printStackTrace();
}
return null;
} // database, collection, query, operation, command, all
@Test
public void saveThisTweet() throws Exception {
Tweet tweet = new Tweet("100000", "XebiaFR", new Date(), "Discovering #mongodb");
DBObject dbObject = new BasicDBObject();
dbObject.put("from_user_id", tweet.getUserId());
dbObject.put("from_user_name", tweet.getUserName());
dbObject.put("created_at", tweet.getCreateAt());
dbObject.put("text", tweet.getText());
collection.save(dbObject, WriteConcern.SAFE);
assertThat(collection.count()).isGreaterThan(0);
}
public static void main(String[] args) {
// TODO Auto-generated method stub
String[] repo = {
"c9s/App-gh", "joshsh/sesametools", "jbr/sibilant", "r1k0/kigen"
}; // "pouchdb/pouchdb"
Mongo mongo = new Mongo(MongoInfo.getMongoServerIp(), 27017);
DB db = mongo.getDB("ghcrawlerV3");
DBCollection pullcache = db.getCollection("pullcacheB");
DBCollection issuecache = db.getCollection("issuecacheB");
DBCollection pulls = db.getCollection("pullscp");
DBCollection issues = db.getCollection("issuescp");
for (int i = 0; i < repo.length; i++) {
pullcache.drop();
issuecache.drop();
PullCrawlerB pullCrawler = new PullCrawlerB();
IssueCrawlerB issueCrawler = new IssueCrawlerB();
issueCrawler.crawlIssues(repo[i]);
pullCrawler.crawlPulls(repo[i]);
DBCursor issuecursor = issuecache.find();
issuecursor.addOption(com.mongodb.Bytes.QUERYOPTION_NOTIMEOUT);
while (issuecursor.hasNext()) {
issues.save(issuecursor.next());
}
issuecursor.close();
DBCursor pullcursor = pullcache.find();
pullcursor.addOption(com.mongodb.Bytes.QUERYOPTION_NOTIMEOUT);
while (pullcursor.hasNext()) {
pulls.save(pullcursor.next());
}
pullcursor.close();
}
}
@Override
public void clearFavorites(String email) throws UserNotFound {
if (email == null) {
throw new IllegalArgumentException("Email cannot be null");
}
DBCollection col = db.getCollection(MongoDBConstants.COLLECTION_USERS);
// Search user by email
DBObject userDb = col.findOne(new BasicDBObject(MongoDBConstants.USER_PROP_EMAIL, email));
if (userDb == null) {
throw new UserNotFound("Unable to find user with email '" + email + "'");
}
userDb.removeField(MongoDBConstants.USER_PROP_FAVORITES);
col.save(userDb);
}
/** {@inheritDoc} */
@Override
public void update(Feature fp) {
if (fp == null) {
throw new IllegalArgumentException("Feature cannot be null nor empty");
}
Feature fpExist = read(fp.getUid());
collection.save(MAPPER.toDBObject(fp));
// enable/disable
if (fp.isEnable() != fpExist.isEnable()) {
if (fp.isEnable()) {
enable(fp.getUid());
} else {
disable(fp.getUid());
}
}
}
