public void introduceType(SpaceTypeDescriptor typeDescriptor) { DBCollection m = getConnection().getCollection(METADATA_COLLECTION_NAME); BasicDBObjectBuilder builder = BasicDBObjectBuilder.start().add(Constants.ID_PROPERTY, typeDescriptor.getTypeName()); try { ByteArrayOutputStream bos = new ByteArrayOutputStream(); ObjectOutputStream out = new ObjectOutputStream(bos); IOUtils.writeObject( out, SpaceTypeDescriptorVersionedSerializationUtils.toSerializableForm(typeDescriptor)); builder.add(TYPE_DESCRIPTOR_FIELD_NAME, bos.toByteArray()); WriteResult wr = m.save(builder.get()); if (logger.isTraceEnabled()) logger.trace(wr); indexBuilder.ensureIndexes(typeDescriptor); } catch (IOException e) { logger.error(e); throw new SpaceMongoException( "error occurs while serialize and save type descriptor: " + typeDescriptor, e); } }
public DBObject asDBObject() { final BasicDBObjectBuilder entryBuilder = BasicDBObjectBuilder.start(); return entryBuilder .add(SERIALIZED_PAYLOAD_PROPERTY, serializedPayload) .add(PAYLOAD_TYPE_PROPERTY, payloadType) .add(PAYLOAD_REVISION_PROPERTY, payloadRevision) .add(EVENT_TIMESTAMP_PROPERTY, timestamp) .add(EVENT_SEQUENCE_NUMBER_PROPERTY, sequenceNumber) .add(META_DATA_PROPERTY, serializedMetaData) .add(EVENT_IDENTIFIER_PROPERTY, eventIdentifier) .get(); }
@Test public void testExplain() { DBCollection c = _db.getCollection("explain1"); c.drop(); for (int i = 0; i < 100; i++) c.save(new BasicDBObject("x", i)); DBObject q = BasicDBObjectBuilder.start().push("x").add("$gt", 50).get(); assertEquals(49, c.find(q).count()); assertEquals(49, c.find(q).itcount()); assertEquals(49, c.find(q).toArray().size()); assertEquals(49, c.find(q).itcount()); assertEquals(20, c.find(q).limit(20).itcount()); assertEquals(20, c.find(q).limit(-20).itcount()); c.ensureIndex(new BasicDBObject("x", 1)); assertEquals(49, c.find(q).count()); assertEquals(49, c.find(q).toArray().size()); assertEquals(49, c.find(q).itcount()); assertEquals(20, c.find(q).limit(20).itcount()); assertEquals(20, c.find(q).limit(-20).itcount()); assertEquals(49, c.find(q).explain().get("n")); assertEquals(20, c.find(q).limit(20).explain().get("n")); assertEquals(20, c.find(q).limit(-20).explain().get("n")); }
/** * Returns the current CommitEntry as a mongo DBObject. * * @return DBObject representing the CommitEntry */ public DBObject asDBObject() { final BasicDBList events = new BasicDBList(); BasicDBObjectBuilder commitBuilder = BasicDBObjectBuilder.start() .add(AGGREGATE_IDENTIFIER_PROPERTY, aggregateIdentifier) .add(SEQUENCE_NUMBER_PROPERTY, firstSequenceNumber) .add(LAST_SEQUENCE_NUMBER_PROPERTY, lastSequenceNumber) .add(FIRST_SEQUENCE_NUMBER_PROPERTY, firstSequenceNumber) .add(TIME_STAMP_PROPERTY, firstTimestamp) .add(FIRST_TIME_STAMP_PROPERTY, firstTimestamp) .add(LAST_TIME_STAMP_PROPERTY, lastTimestamp) .add(EVENTS_PROPERTY, events); for (EventEntry eventEntry : eventEntries) { events.add(eventEntry.asDBObject()); } return commitBuilder.get(); }
@Override public DBCursor findEvents(DBCollection collection, MongoCriteria criteria) { DBObject filter = criteria == null ? null : criteria.asMongoObject(); DBObject sort = BasicDBObjectBuilder.start() .add(CommitEntry.TIME_STAMP_PROPERTY, ORDER_ASC) .add(CommitEntry.SEQUENCE_NUMBER_PROPERTY, ORDER_ASC) .get(); return collection.find(filter).sort(sort); }
@Override public DBCursor findLastSnapshot(DBCollection collection, String aggregateIdentifier) { DBObject mongoEntry = BasicDBObjectBuilder.start() .add(CommitEntry.AGGREGATE_IDENTIFIER_PROPERTY, aggregateIdentifier) .get(); return collection .find(mongoEntry) .sort(new BasicDBObject(CommitEntry.SEQUENCE_NUMBER_PROPERTY, ORDER_DESC)) .limit(1); }
private static DBObject normalize(DBObject obj) { BasicDBObjectBuilder builder = BasicDBObjectBuilder.start(); Iterator<String> iterator = obj.keySet().iterator(); builder.push("$set"); while (iterator.hasNext()) { String key = iterator.next(); if (Constants.ID_PROPERTY.equals(key)) continue; Object value = obj.get(key); if (value == null) continue; builder.add(key, value); } return builder.get(); }
public void performBatch(List<BatchUnit> rows) { if (logger.isTraceEnabled()) { logger.trace("MongoClientWrapper.performBatch(" + rows + ")"); logger.trace("Batch size to be performed is " + rows.size()); } // List<Future<? extends Number>> pending = new ArrayList<Future<? extends Number>>(); for (BatchUnit row : rows) { SpaceDocument spaceDoc = row.getSpaceDocument(); SpaceTypeDescriptor typeDescriptor = types.get(row.getTypeName()).getTypeDescriptor(); SpaceDocumentMapper<DBObject> mapper = getMapper(typeDescriptor); DBObject obj = mapper.toDBObject(spaceDoc); DBCollection col = getCollection(row.getTypeName()); switch (row.getDataSyncOperationType()) { case WRITE: case UPDATE: col.save(obj); break; case PARTIAL_UPDATE: DBObject query = BasicDBObjectBuilder.start() .add(Constants.ID_PROPERTY, obj.get(Constants.ID_PROPERTY)) .get(); DBObject update = normalize(obj); col.update(query, update); break; // case REMOVE_BY_UID: // Not supported by this implementation case REMOVE: col.remove(obj); break; default: throw new IllegalStateException( "Unsupported data sync operation type: " + row.getDataSyncOperationType()); } } /*long totalCount = waitFor(pending); if (logger.isTraceEnabled()) { logger.trace("total accepted replies is: " + totalCount); }*/ }
@Test public void testBig() { DBCollection c = _db.getCollection("big1"); c.drop(); String bigString; { StringBuilder buf = new StringBuilder(16000); for (int i = 0; i < 16000; i++) buf.append("x"); bigString = buf.toString(); } int numToInsert = (15 * 1024 * 1024) / bigString.length(); for (int i = 0; i < numToInsert; i++) c.save(BasicDBObjectBuilder.start().add("x", i).add("s", bigString).get()); assert (800 < numToInsert); assertEquals(numToInsert, c.find().count()); assertEquals(numToInsert, c.find().toArray().size()); assertEquals(numToInsert, c.find().limit(800).count()); assertEquals(800, c.find().limit(800).toArray().size()); // negative limit works like negative batchsize, for legacy reason int x = c.find().limit(-800).toArray().size(); assertLess(x, 800); DBCursor a = c.find(); assertEquals(numToInsert, a.itcount()); DBCursor b = c.find().batchSize(10); assertEquals(numToInsert, b.itcount()); assertEquals(10, b.getSizes().get(0).intValue()); assertLess(a.numGetMores(), b.numGetMores()); assertEquals(numToInsert, c.find().batchSize(2).itcount()); assertEquals(numToInsert, c.find().batchSize(1).itcount()); assertEquals(numToInsert, _count(c.find(null, null).skip(0).batchSize(5))); assertEquals(5, _count(c.find(null, null).skip(0).batchSize(-5))); }
@Test public void testLimitAndBatchSize() { DBCollection c = _db.getCollection("LimitAndBatchSize"); c.drop(); for (int i = 0; i < 1000; i++) c.save(new BasicDBObject("x", i)); DBObject q = BasicDBObjectBuilder.start().push("x").add("$lt", 200).get(); DBCursor cur = c.find(q); assertEquals(0, cur.getCursorId()); assertEquals(200, cur.itcount()); cur = c.find(q).limit(50); assertEquals(0, cur.getCursorId()); assertEquals(50, cur.itcount()); cur = c.find(q).batchSize(50); assertEquals(0, cur.getCursorId()); assertEquals(200, cur.itcount()); cur = c.find(q).batchSize(100).limit(50); assertEquals(0, cur.getCursorId()); assertEquals(50, cur.itcount()); cur = c.find(q).batchSize(-40); assertEquals(0, cur.getCursorId()); assertEquals(40, cur.itcount()); cur = c.find(q).limit(-100); assertEquals(0, cur.getCursorId()); assertEquals(100, cur.itcount()); cur = c.find(q).batchSize(-40).limit(20); assertEquals(0, cur.getCursorId()); assertEquals(20, cur.itcount()); cur = c.find(q).batchSize(-20).limit(100); assertEquals(0, cur.getCursorId()); assertEquals(20, cur.itcount()); }
public void indexTweet(Tweet tweet) { this.connect(); DBObject dbObject = BasicDBObjectBuilder.start() // .add("_id", status.getId()) .add("id", tweet.getId()) .add("text", tweet.getText()) .add("created_at", tweet.getCreatedAt()) .add("hashtags", tweet.getHashTags()) .add("userId", tweet.getUserId()) .add("userName", tweet.getUserName()) .add("userScreenName", tweet.getUserScreenName()) .add("coordinates", tweet.getCoordinates()) .get(); // WriteConcern wc = WriteConcern.UNACKNOWLEDGED; WriteConcern wc = WriteConcern.ACKNOWLEDGED; wc = wc.continueOnError(true); WriteResult writeResult = this.dbCollection.insert(dbObject, wc); }
private static BasicDBObjectBuilder baseNotificationBuilder(String login, String messageType) { return BasicDBObjectBuilder.start("date", new Date()) .add("login", login) .add("type", messageType); }
/** * Returns the mongo DBObject used to query mongo for events for specified aggregate identifier. * * @param aggregateIdentifier Identifier of the aggregate to obtain the mongo DBObject for * @param firstSequenceNumber number representing the first event to obtain * @return Created DBObject based on the provided parameters to be used for a query */ public static DBObject forAggregate(String aggregateIdentifier, long firstSequenceNumber) { return BasicDBObjectBuilder.start() .add(CommitEntry.AGGREGATE_IDENTIFIER_PROPERTY, aggregateIdentifier) .add(CommitEntry.SEQUENCE_NUMBER_PROPERTY, new BasicDBObject("$gte", firstSequenceNumber)) .get(); }
/** * Evaluates JavaScript functions on the database server. This is useful if you need to touch a * lot of data lightly, in which case network transfer could be a bottleneck. * * @param code @{code String} representation of JavaScript function * @param args arguments to pass to the JavaScript function * @return result of the command execution * @throws MongoException */ public CommandResult doEval(String code, Object... args) { return command(BasicDBObjectBuilder.start().add("$eval", code).add("args", args).get()); }
class DBTCPConnector implements DBConnector { static Logger _logger = Logger.getLogger(Bytes.LOGGER.getName() + ".tcp"); static Logger _createLogger = Logger.getLogger(_logger.getName() + ".connect"); public DBTCPConnector(Mongo m, ServerAddress addr) throws MongoException { _mongo = m; _portHolder = new DBPortPool.Holder(m._options); _checkAddress(addr); _createLogger.info(addr.toString()); if (addr.isPaired()) { _allHosts = new ArrayList<ServerAddress>(addr.explode()); _createLogger.info("switch to paired mode : " + _allHosts + " -> " + _curAddress); } else { _set(addr); _allHosts = null; } } public DBTCPConnector(Mongo m, ServerAddress... all) throws MongoException { this(m, Arrays.asList(all)); } public DBTCPConnector(Mongo m, List<ServerAddress> all) throws MongoException { _mongo = m; _portHolder = new DBPortPool.Holder(m._options); _checkAddress(all); _allHosts = new ArrayList<ServerAddress>(all); // make a copy so it can't be modified _createLogger.info(all + " -> " + _curAddress); } private static ServerAddress _checkAddress(ServerAddress addr) { if (addr == null) throw new NullPointerException("address can't be null"); return addr; } private static ServerAddress _checkAddress(List<ServerAddress> addrs) { if (addrs == null) throw new NullPointerException("addresses can't be null"); if (addrs.size() == 0) throw new IllegalArgumentException("need to specify at least 1 address"); return addrs.get(0); } /** * Start a "request". * * <p>A "request" is a group of operations in which order matters. Examples include inserting a * document and then performing a query which expects that document to have been inserted, or * performing an operation and then using com.mongodb.Mongo.getLastError to perform error-checking * on that operation. When a thread performs operations in a "request", all operations will be * performed on the same socket, so they will be correctly ordered. */ public void requestStart() { _threadPort.get().requestStart(); } /** * End the current "request", if this thread is in one. * * <p>By ending a request when it is safe to do so the built-in connection- pool is allowed to * reassign requests to different sockets in order to more effectively balance load. See * requestStart for more information. */ public void requestDone() { _threadPort.get().requestDone(); } public void requestEnsureConnection() { _threadPort.get().requestEnsureConnection(); } WriteResult _checkWriteError(MyPort mp, DBPort port) throws MongoException { CommandResult e = _mongo.getDB("admin").getLastError(); mp.done(port); Object foo = e.get("err"); if (foo == null) return new WriteResult(e); int code = -1; if (e.get("code") instanceof Number) code = ((Number) e.get("code")).intValue(); String s = foo.toString(); if (code == 11000 || code == 11001 || s.startsWith("E11000") || s.startsWith("E11001")) throw new MongoException.DuplicateKey(code, s); throw new MongoException(code, s); } public WriteResult say(DB db, OutMessage m, DB.WriteConcern concern) throws MongoException { MyPort mp = _threadPort.get(); DBPort port = mp.get(true); port.checkAuth(db); try { port.say(m); if (concern == DB.WriteConcern.STRICT) { return _checkWriteError(mp, port); } else { mp.done(port); return new WriteResult(db, port); } } catch (IOException ioe) { mp.error(ioe); _error(ioe); if (concern == DB.WriteConcern.NONE) { CommandResult res = new CommandResult(); res.put("ok", false); res.put("$err", "NETWORK ERROR"); return new WriteResult(res); } throw new MongoException.Network("can't say something", ioe); } catch (MongoException me) { throw me; } catch (RuntimeException re) { mp.error(re); throw re; } } public Response call(DB db, DBCollection coll, OutMessage m) throws MongoException { return call(db, coll, m, 2); } public Response call(DB db, DBCollection coll, OutMessage m, int retries) throws MongoException { final MyPort mp = _threadPort.get(); final DBPort port = mp.get(false); port.checkAuth(db); Response res = null; try { res = port.call(m, coll); mp.done(port); } catch (IOException ioe) { mp.error(ioe); if (_error(ioe) && retries > 0) { return call(db, coll, m, retries - 1); } throw new MongoException.Network("can't call something", ioe); } catch (RuntimeException re) { mp.error(re); throw re; } ServerError err = res.getError(); if (err != null && err.isNotMasterError()) { _pickCurrent(); if (retries <= 0) { throw new MongoException("not talking to master and retries used up"); } return call(db, coll, m, retries - 1); } return res; } public ServerAddress getAddress() { return _curAddress; } public List<ServerAddress> getAllAddress() { return _allHosts; } public String getConnectPoint() { return _curAddress.toString(); } boolean _error(Throwable t) throws MongoException { if (_allHosts != null) { System.out.println("paired mode, switching master b/c of: " + t); t.printStackTrace(); _pickCurrent(); } return true; } class MyPort { DBPort get(boolean keep) { _internalStack++; if (_internalStack > 1) { if (_last == null) { System.err.println("_internalStack > 1 and _last is null!"); } else { return _last; } } if (_port != null) return _port; try { DBPort p = _curPortPool.get(); if (keep && _inRequest) _port = p; _last = p; return p; } catch (DBPortPool.NoMoreConnection nmc) { _internalStack = 0; throw nmc; } } void done(DBPort p) { if (_internalStack <= 0) { int prev = _internalStack; _reset(); throw new IllegalStateException("done called and _internalStack was: " + _internalStack); } _internalStack--; if (p != _port && _internalStack == 0) _curPortPool.done(p); if (_internalStack < 0) { System.err.println("_internalStack < 0 : " + _internalStack); _internalStack = 0; } } void error(Exception e) { _curPortPool.remove(_port); _curPortPool.gotError(e); _internalStack = 0; _last = null; } void requestEnsureConnection() { if (!_inRequest) return; if (_port != null) return; _port = _curPortPool.get(); } void requestStart() { _inRequest = true; if (_port != null) { _port = null; System.err.println("ERROR. somehow _port was not null at requestStart"); } } void requestDone() { if (_port != null) _curPortPool.done(_port); _port = null; _inRequest = false; if (_internalStack > 0) { System.err.println("_internalStack in requestDone should be 0 is: " + _internalStack); _internalStack = 0; } } void _reset() { _internalStack = 0; _port = null; _last = null; } int _internalStack = 0; DBPort _port; DBPort _last; boolean _inRequest; } void _pickInitial() throws MongoException { if (_curAddress != null) return; // we need to just get a server to query for ismaster _pickCurrent(); try { _logger.info("current address beginning of _pickInitial: " + _curAddress); DBObject im = isMasterCmd(); if (_isMaster(im)) return; synchronized (_allHosts) { Collections.shuffle(_allHosts); for (ServerAddress a : _allHosts) { if (_curAddress == a) continue; _logger.info("remote [" + _curAddress + "] -> [" + a + "]"); _set(a); im = isMasterCmd(); if (_isMaster(im)) return; _logger.severe("switched to: " + a + " but isn't master"); } throw new MongoException("can't find master"); } } catch (Exception e) { _logger.log(Level.SEVERE, "can't pick initial master, using random one", e); } } private void _pickCurrent() throws MongoException { if (_allHosts == null) throw new MongoException( "got master/slave issue but not in master/slave mode on the client side"); synchronized (_allHosts) { Collections.shuffle(_allHosts); for (int i = 0; i < _allHosts.size(); i++) { ServerAddress a = _allHosts.get(i); if (a == _curAddress) continue; if (_curAddress != null) { _logger.info("switching from [" + _curAddress + "] to [" + a + "]"); } _set(a); return; } } throw new MongoException("couldn't find a new host to swtich too"); } private boolean _set(ServerAddress addr) { if (_curAddress == addr) return false; _curAddress = addr; _curPortPool = _portHolder.get(addr.getSocketAddress()); return true; } public String debugString() { StringBuilder buf = new StringBuilder("DBTCPConnector: "); if (_allHosts != null) buf.append("paired : ").append(_allHosts); else buf.append(_curAddress).append(" ").append(_curAddress._addr); return buf.toString(); } DBObject isMasterCmd() { DBCollection collection = _mongo.getDB("admin").getCollection("$cmd"); Iterator<DBObject> i = collection.__find(_isMaster, null, 0, 1, 0); if (i == null || !i.hasNext()) throw new MongoException("no result for ismaster query?"); DBObject res = i.next(); if (i.hasNext()) throw new MongoException("what's going on"); return res; } boolean _isMaster(DBObject res) { Object x = res.get("ismaster"); if (x == null) throw new IllegalStateException("ismaster shouldn't be null: " + res); if (x instanceof Boolean) return (Boolean) x; if (x instanceof Number) return ((Number) x).intValue() == 1; throw new IllegalArgumentException("invalid ismaster [" + x + "] : " + x.getClass().getName()); } public void close() { _portHolder.close(); } final Mongo _mongo; private ServerAddress _curAddress; private DBPortPool _curPortPool; private DBPortPool.Holder _portHolder; private final List<ServerAddress> _allHosts; private final ThreadLocal<MyPort> _threadPort = new ThreadLocal<MyPort>() { protected MyPort initialValue() { return new MyPort(); } }; private static final DBObject _isMaster = BasicDBObjectBuilder.start().add("ismaster", 1).get(); }