public void introduceType(SpaceTypeDescriptor typeDescriptor) {
DBCollection m = getConnection().getCollection(METADATA_COLLECTION_NAME);
BasicDBObjectBuilder builder =
BasicDBObjectBuilder.start().add(Constants.ID_PROPERTY, typeDescriptor.getTypeName());
try {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ObjectOutputStream out = new ObjectOutputStream(bos);
IOUtils.writeObject(
out, SpaceTypeDescriptorVersionedSerializationUtils.toSerializableForm(typeDescriptor));
builder.add(TYPE_DESCRIPTOR_FIELD_NAME, bos.toByteArray());
WriteResult wr = m.save(builder.get());
if (logger.isTraceEnabled()) logger.trace(wr);
indexBuilder.ensureIndexes(typeDescriptor);
} catch (IOException e) {
logger.error(e);
throw new SpaceMongoException(
"error occurs while serialize and save type descriptor: " + typeDescriptor, e);
}
}
public DBObject asDBObject() {
final BasicDBObjectBuilder entryBuilder = BasicDBObjectBuilder.start();
return entryBuilder
.add(SERIALIZED_PAYLOAD_PROPERTY, serializedPayload)
.add(PAYLOAD_TYPE_PROPERTY, payloadType)
.add(PAYLOAD_REVISION_PROPERTY, payloadRevision)
.add(EVENT_TIMESTAMP_PROPERTY, timestamp)
.add(EVENT_SEQUENCE_NUMBER_PROPERTY, sequenceNumber)
.add(META_DATA_PROPERTY, serializedMetaData)
.add(EVENT_IDENTIFIER_PROPERTY, eventIdentifier)
.get();
}
@Test
public void testExplain() {
DBCollection c = _db.getCollection("explain1");
c.drop();
for (int i = 0; i < 100; i++) c.save(new BasicDBObject("x", i));
DBObject q = BasicDBObjectBuilder.start().push("x").add("$gt", 50).get();
assertEquals(49, c.find(q).count());
assertEquals(49, c.find(q).itcount());
assertEquals(49, c.find(q).toArray().size());
assertEquals(49, c.find(q).itcount());
assertEquals(20, c.find(q).limit(20).itcount());
assertEquals(20, c.find(q).limit(-20).itcount());
c.ensureIndex(new BasicDBObject("x", 1));
assertEquals(49, c.find(q).count());
assertEquals(49, c.find(q).toArray().size());
assertEquals(49, c.find(q).itcount());
assertEquals(20, c.find(q).limit(20).itcount());
assertEquals(20, c.find(q).limit(-20).itcount());
assertEquals(49, c.find(q).explain().get("n"));
assertEquals(20, c.find(q).limit(20).explain().get("n"));
assertEquals(20, c.find(q).limit(-20).explain().get("n"));
}
/**
* Returns the current CommitEntry as a mongo DBObject.
*
* @return DBObject representing the CommitEntry
*/
public DBObject asDBObject() {
final BasicDBList events = new BasicDBList();
BasicDBObjectBuilder commitBuilder =
BasicDBObjectBuilder.start()
.add(AGGREGATE_IDENTIFIER_PROPERTY, aggregateIdentifier)
.add(SEQUENCE_NUMBER_PROPERTY, firstSequenceNumber)
.add(LAST_SEQUENCE_NUMBER_PROPERTY, lastSequenceNumber)
.add(FIRST_SEQUENCE_NUMBER_PROPERTY, firstSequenceNumber)
.add(TIME_STAMP_PROPERTY, firstTimestamp)
.add(FIRST_TIME_STAMP_PROPERTY, firstTimestamp)
.add(LAST_TIME_STAMP_PROPERTY, lastTimestamp)
.add(EVENTS_PROPERTY, events);
for (EventEntry eventEntry : eventEntries) {
events.add(eventEntry.asDBObject());
}
return commitBuilder.get();
}
@Override
public DBCursor findEvents(DBCollection collection, MongoCriteria criteria) {
DBObject filter = criteria == null ? null : criteria.asMongoObject();
DBObject sort =
BasicDBObjectBuilder.start()
.add(CommitEntry.TIME_STAMP_PROPERTY, ORDER_ASC)
.add(CommitEntry.SEQUENCE_NUMBER_PROPERTY, ORDER_ASC)
.get();
return collection.find(filter).sort(sort);
}
@Override
public DBCursor findLastSnapshot(DBCollection collection, String aggregateIdentifier) {
DBObject mongoEntry =
BasicDBObjectBuilder.start()
.add(CommitEntry.AGGREGATE_IDENTIFIER_PROPERTY, aggregateIdentifier)
.get();
return collection
.find(mongoEntry)
.sort(new BasicDBObject(CommitEntry.SEQUENCE_NUMBER_PROPERTY, ORDER_DESC))
.limit(1);
}
private static DBObject normalize(DBObject obj) {
BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
Iterator<String> iterator = obj.keySet().iterator();
builder.push("$set");
while (iterator.hasNext()) {
String key = iterator.next();
if (Constants.ID_PROPERTY.equals(key)) continue;
Object value = obj.get(key);
if (value == null) continue;
builder.add(key, value);
}
return builder.get();
}
public void performBatch(List<BatchUnit> rows) {
if (logger.isTraceEnabled()) {
logger.trace("MongoClientWrapper.performBatch(" + rows + ")");
logger.trace("Batch size to be performed is " + rows.size());
}
// List<Future<? extends Number>> pending = new ArrayList<Future<? extends Number>>();
for (BatchUnit row : rows) {
SpaceDocument spaceDoc = row.getSpaceDocument();
SpaceTypeDescriptor typeDescriptor = types.get(row.getTypeName()).getTypeDescriptor();
SpaceDocumentMapper<DBObject> mapper = getMapper(typeDescriptor);
DBObject obj = mapper.toDBObject(spaceDoc);
DBCollection col = getCollection(row.getTypeName());
switch (row.getDataSyncOperationType()) {
case WRITE:
case UPDATE:
col.save(obj);
break;
case PARTIAL_UPDATE:
DBObject query =
BasicDBObjectBuilder.start()
.add(Constants.ID_PROPERTY, obj.get(Constants.ID_PROPERTY))
.get();
DBObject update = normalize(obj);
col.update(query, update);
break;
// case REMOVE_BY_UID: // Not supported by this implementation
case REMOVE:
col.remove(obj);
break;
default:
throw new IllegalStateException(
"Unsupported data sync operation type: " + row.getDataSyncOperationType());
}
}
/*long totalCount = waitFor(pending);
if (logger.isTraceEnabled()) {
logger.trace("total accepted replies is: " + totalCount);
}*/
}
@Test
public void testBig() {
DBCollection c = _db.getCollection("big1");
c.drop();
String bigString;
{
StringBuilder buf = new StringBuilder(16000);
for (int i = 0; i < 16000; i++) buf.append("x");
bigString = buf.toString();
}
int numToInsert = (15 * 1024 * 1024) / bigString.length();
for (int i = 0; i < numToInsert; i++)
c.save(BasicDBObjectBuilder.start().add("x", i).add("s", bigString).get());
assert (800 < numToInsert);
assertEquals(numToInsert, c.find().count());
assertEquals(numToInsert, c.find().toArray().size());
assertEquals(numToInsert, c.find().limit(800).count());
assertEquals(800, c.find().limit(800).toArray().size());
// negative limit works like negative batchsize, for legacy reason
int x = c.find().limit(-800).toArray().size();
assertLess(x, 800);
DBCursor a = c.find();
assertEquals(numToInsert, a.itcount());
DBCursor b = c.find().batchSize(10);
assertEquals(numToInsert, b.itcount());
assertEquals(10, b.getSizes().get(0).intValue());
assertLess(a.numGetMores(), b.numGetMores());
assertEquals(numToInsert, c.find().batchSize(2).itcount());
assertEquals(numToInsert, c.find().batchSize(1).itcount());
assertEquals(numToInsert, _count(c.find(null, null).skip(0).batchSize(5)));
assertEquals(5, _count(c.find(null, null).skip(0).batchSize(-5)));
}
@Test
public void testLimitAndBatchSize() {
DBCollection c = _db.getCollection("LimitAndBatchSize");
c.drop();
for (int i = 0; i < 1000; i++) c.save(new BasicDBObject("x", i));
DBObject q = BasicDBObjectBuilder.start().push("x").add("$lt", 200).get();
DBCursor cur = c.find(q);
assertEquals(0, cur.getCursorId());
assertEquals(200, cur.itcount());
cur = c.find(q).limit(50);
assertEquals(0, cur.getCursorId());
assertEquals(50, cur.itcount());
cur = c.find(q).batchSize(50);
assertEquals(0, cur.getCursorId());
assertEquals(200, cur.itcount());
cur = c.find(q).batchSize(100).limit(50);
assertEquals(0, cur.getCursorId());
assertEquals(50, cur.itcount());
cur = c.find(q).batchSize(-40);
assertEquals(0, cur.getCursorId());
assertEquals(40, cur.itcount());
cur = c.find(q).limit(-100);
assertEquals(0, cur.getCursorId());
assertEquals(100, cur.itcount());
cur = c.find(q).batchSize(-40).limit(20);
assertEquals(0, cur.getCursorId());
assertEquals(20, cur.itcount());
cur = c.find(q).batchSize(-20).limit(100);
assertEquals(0, cur.getCursorId());
assertEquals(20, cur.itcount());
}
public void indexTweet(Tweet tweet) {
this.connect();
DBObject dbObject =
BasicDBObjectBuilder.start()
// .add("_id", status.getId())
.add("id", tweet.getId())
.add("text", tweet.getText())
.add("created_at", tweet.getCreatedAt())
.add("hashtags", tweet.getHashTags())
.add("userId", tweet.getUserId())
.add("userName", tweet.getUserName())
.add("userScreenName", tweet.getUserScreenName())
.add("coordinates", tweet.getCoordinates())
.get();
// WriteConcern wc = WriteConcern.UNACKNOWLEDGED;
WriteConcern wc = WriteConcern.ACKNOWLEDGED;
wc = wc.continueOnError(true);
WriteResult writeResult = this.dbCollection.insert(dbObject, wc);
}
private static BasicDBObjectBuilder baseNotificationBuilder(String login, String messageType) {
return BasicDBObjectBuilder.start("date", new Date())
.add("login", login)
.add("type", messageType);
}
/**
* Returns the mongo DBObject used to query mongo for events for specified aggregate identifier.
*
* @param aggregateIdentifier Identifier of the aggregate to obtain the mongo DBObject for
* @param firstSequenceNumber number representing the first event to obtain
* @return Created DBObject based on the provided parameters to be used for a query
*/
public static DBObject forAggregate(String aggregateIdentifier, long firstSequenceNumber) {
return BasicDBObjectBuilder.start()
.add(CommitEntry.AGGREGATE_IDENTIFIER_PROPERTY, aggregateIdentifier)
.add(CommitEntry.SEQUENCE_NUMBER_PROPERTY, new BasicDBObject("$gte", firstSequenceNumber))
.get();
}
/**
* Evaluates JavaScript functions on the database server. This is useful if you need to touch a
* lot of data lightly, in which case network transfer could be a bottleneck.
*
* @param code @{code String} representation of JavaScript function
* @param args arguments to pass to the JavaScript function
* @return result of the command execution
* @throws MongoException
*/
public CommandResult doEval(String code, Object... args) {
return command(BasicDBObjectBuilder.start().add("$eval", code).add("args", args).get());
}
class DBTCPConnector implements DBConnector {
static Logger _logger = Logger.getLogger(Bytes.LOGGER.getName() + ".tcp");
static Logger _createLogger = Logger.getLogger(_logger.getName() + ".connect");
public DBTCPConnector(Mongo m, ServerAddress addr) throws MongoException {
_mongo = m;
_portHolder = new DBPortPool.Holder(m._options);
_checkAddress(addr);
_createLogger.info(addr.toString());
if (addr.isPaired()) {
_allHosts = new ArrayList<ServerAddress>(addr.explode());
_createLogger.info("switch to paired mode : " + _allHosts + " -> " + _curAddress);
} else {
_set(addr);
_allHosts = null;
}
}
public DBTCPConnector(Mongo m, ServerAddress... all) throws MongoException {
this(m, Arrays.asList(all));
}
public DBTCPConnector(Mongo m, List<ServerAddress> all) throws MongoException {
_mongo = m;
_portHolder = new DBPortPool.Holder(m._options);
_checkAddress(all);
_allHosts = new ArrayList<ServerAddress>(all); // make a copy so it can't be modified
_createLogger.info(all + " -> " + _curAddress);
}
private static ServerAddress _checkAddress(ServerAddress addr) {
if (addr == null) throw new NullPointerException("address can't be null");
return addr;
}
private static ServerAddress _checkAddress(List<ServerAddress> addrs) {
if (addrs == null) throw new NullPointerException("addresses can't be null");
if (addrs.size() == 0) throw new IllegalArgumentException("need to specify at least 1 address");
return addrs.get(0);
}
/**
* Start a "request".
*
* <p>A "request" is a group of operations in which order matters. Examples include inserting a
* document and then performing a query which expects that document to have been inserted, or
* performing an operation and then using com.mongodb.Mongo.getLastError to perform error-checking
* on that operation. When a thread performs operations in a "request", all operations will be
* performed on the same socket, so they will be correctly ordered.
*/
public void requestStart() {
_threadPort.get().requestStart();
}
/**
* End the current "request", if this thread is in one.
*
* <p>By ending a request when it is safe to do so the built-in connection- pool is allowed to
* reassign requests to different sockets in order to more effectively balance load. See
* requestStart for more information.
*/
public void requestDone() {
_threadPort.get().requestDone();
}
public void requestEnsureConnection() {
_threadPort.get().requestEnsureConnection();
}
WriteResult _checkWriteError(MyPort mp, DBPort port) throws MongoException {
CommandResult e = _mongo.getDB("admin").getLastError();
mp.done(port);
Object foo = e.get("err");
if (foo == null) return new WriteResult(e);
int code = -1;
if (e.get("code") instanceof Number) code = ((Number) e.get("code")).intValue();
String s = foo.toString();
if (code == 11000 || code == 11001 || s.startsWith("E11000") || s.startsWith("E11001"))
throw new MongoException.DuplicateKey(code, s);
throw new MongoException(code, s);
}
public WriteResult say(DB db, OutMessage m, DB.WriteConcern concern) throws MongoException {
MyPort mp = _threadPort.get();
DBPort port = mp.get(true);
port.checkAuth(db);
try {
port.say(m);
if (concern == DB.WriteConcern.STRICT) {
return _checkWriteError(mp, port);
} else {
mp.done(port);
return new WriteResult(db, port);
}
} catch (IOException ioe) {
mp.error(ioe);
_error(ioe);
if (concern == DB.WriteConcern.NONE) {
CommandResult res = new CommandResult();
res.put("ok", false);
res.put("$err", "NETWORK ERROR");
return new WriteResult(res);
}
throw new MongoException.Network("can't say something", ioe);
} catch (MongoException me) {
throw me;
} catch (RuntimeException re) {
mp.error(re);
throw re;
}
}
public Response call(DB db, DBCollection coll, OutMessage m) throws MongoException {
return call(db, coll, m, 2);
}
public Response call(DB db, DBCollection coll, OutMessage m, int retries) throws MongoException {
final MyPort mp = _threadPort.get();
final DBPort port = mp.get(false);
port.checkAuth(db);
Response res = null;
try {
res = port.call(m, coll);
mp.done(port);
} catch (IOException ioe) {
mp.error(ioe);
if (_error(ioe) && retries > 0) {
return call(db, coll, m, retries - 1);
}
throw new MongoException.Network("can't call something", ioe);
} catch (RuntimeException re) {
mp.error(re);
throw re;
}
ServerError err = res.getError();
if (err != null && err.isNotMasterError()) {
_pickCurrent();
if (retries <= 0) {
throw new MongoException("not talking to master and retries used up");
}
return call(db, coll, m, retries - 1);
}
return res;
}
public ServerAddress getAddress() {
return _curAddress;
}
public List<ServerAddress> getAllAddress() {
return _allHosts;
}
public String getConnectPoint() {
return _curAddress.toString();
}
boolean _error(Throwable t) throws MongoException {
if (_allHosts != null) {
System.out.println("paired mode, switching master b/c of: " + t);
t.printStackTrace();
_pickCurrent();
}
return true;
}
class MyPort {
DBPort get(boolean keep) {
_internalStack++;
if (_internalStack > 1) {
if (_last == null) {
System.err.println("_internalStack > 1 and _last is null!");
} else {
return _last;
}
}
if (_port != null) return _port;
try {
DBPort p = _curPortPool.get();
if (keep && _inRequest) _port = p;
_last = p;
return p;
} catch (DBPortPool.NoMoreConnection nmc) {
_internalStack = 0;
throw nmc;
}
}
void done(DBPort p) {
if (_internalStack <= 0) {
int prev = _internalStack;
_reset();
throw new IllegalStateException("done called and _internalStack was: " + _internalStack);
}
_internalStack--;
if (p != _port && _internalStack == 0) _curPortPool.done(p);
if (_internalStack < 0) {
System.err.println("_internalStack < 0 : " + _internalStack);
_internalStack = 0;
}
}
void error(Exception e) {
_curPortPool.remove(_port);
_curPortPool.gotError(e);
_internalStack = 0;
_last = null;
}
void requestEnsureConnection() {
if (!_inRequest) return;
if (_port != null) return;
_port = _curPortPool.get();
}
void requestStart() {
_inRequest = true;
if (_port != null) {
_port = null;
System.err.println("ERROR. somehow _port was not null at requestStart");
}
}
void requestDone() {
if (_port != null) _curPortPool.done(_port);
_port = null;
_inRequest = false;
if (_internalStack > 0) {
System.err.println("_internalStack in requestDone should be 0 is: " + _internalStack);
_internalStack = 0;
}
}
void _reset() {
_internalStack = 0;
_port = null;
_last = null;
}
int _internalStack = 0;
DBPort _port;
DBPort _last;
boolean _inRequest;
}
void _pickInitial() throws MongoException {
if (_curAddress != null) return;
// we need to just get a server to query for ismaster
_pickCurrent();
try {
_logger.info("current address beginning of _pickInitial: " + _curAddress);
DBObject im = isMasterCmd();
if (_isMaster(im)) return;
synchronized (_allHosts) {
Collections.shuffle(_allHosts);
for (ServerAddress a : _allHosts) {
if (_curAddress == a) continue;
_logger.info("remote [" + _curAddress + "] -> [" + a + "]");
_set(a);
im = isMasterCmd();
if (_isMaster(im)) return;
_logger.severe("switched to: " + a + " but isn't master");
}
throw new MongoException("can't find master");
}
} catch (Exception e) {
_logger.log(Level.SEVERE, "can't pick initial master, using random one", e);
}
}
private void _pickCurrent() throws MongoException {
if (_allHosts == null)
throw new MongoException(
"got master/slave issue but not in master/slave mode on the client side");
synchronized (_allHosts) {
Collections.shuffle(_allHosts);
for (int i = 0; i < _allHosts.size(); i++) {
ServerAddress a = _allHosts.get(i);
if (a == _curAddress) continue;
if (_curAddress != null) {
_logger.info("switching from [" + _curAddress + "] to [" + a + "]");
}
_set(a);
return;
}
}
throw new MongoException("couldn't find a new host to swtich too");
}
private boolean _set(ServerAddress addr) {
if (_curAddress == addr) return false;
_curAddress = addr;
_curPortPool = _portHolder.get(addr.getSocketAddress());
return true;
}
public String debugString() {
StringBuilder buf = new StringBuilder("DBTCPConnector: ");
if (_allHosts != null) buf.append("paired : ").append(_allHosts);
else buf.append(_curAddress).append(" ").append(_curAddress._addr);
return buf.toString();
}
DBObject isMasterCmd() {
DBCollection collection = _mongo.getDB("admin").getCollection("$cmd");
Iterator<DBObject> i = collection.__find(_isMaster, null, 0, 1, 0);
if (i == null || !i.hasNext()) throw new MongoException("no result for ismaster query?");
DBObject res = i.next();
if (i.hasNext()) throw new MongoException("what's going on");
return res;
}
boolean _isMaster(DBObject res) {
Object x = res.get("ismaster");
if (x == null) throw new IllegalStateException("ismaster shouldn't be null: " + res);
if (x instanceof Boolean) return (Boolean) x;
if (x instanceof Number) return ((Number) x).intValue() == 1;
throw new IllegalArgumentException("invalid ismaster [" + x + "] : " + x.getClass().getName());
}
public void close() {
_portHolder.close();
}
final Mongo _mongo;
private ServerAddress _curAddress;
private DBPortPool _curPortPool;
private DBPortPool.Holder _portHolder;
private final List<ServerAddress> _allHosts;
private final ThreadLocal<MyPort> _threadPort =
new ThreadLocal<MyPort>() {
protected MyPort initialValue() {
return new MyPort();
}
};
private static final DBObject _isMaster = BasicDBObjectBuilder.start().add("ismaster", 1).get();
}
