public void ensureCaps() {
for (MappedClass mc : mapr.getMappedClasses())
if (mc.getEntityAnnotation() != null && mc.getEntityAnnotation().cap().value() > 0) {
CappedAt cap = mc.getEntityAnnotation().cap();
String collName = mapr.getCollectionName(mc.getClazz());
BasicDBObjectBuilder dbCapOpts = BasicDBObjectBuilder.start("capped", true);
if (cap.value() > 0) dbCapOpts.add("size", cap.value());
if (cap.count() > 0) dbCapOpts.add("max", cap.count());
DB db = getDB();
if (db.getCollectionNames().contains(collName)) {
DBObject dbResult = db.command(BasicDBObjectBuilder.start("collstats", collName).get());
if (dbResult.containsField("capped")) {
// TODO: check the cap options.
log.warning("DBCollection already exists is cap'd already; doing nothing. " + dbResult);
} else {
log.warning(
"DBCollection already exists with same name("
+ collName
+ ") and is not cap'd; not creating cap'd version!");
}
} else {
getDB().createCollection(collName, dbCapOpts.get());
log.debug("Created cap'd DBCollection (" + collName + ") with opts " + dbCapOpts);
}
}
}
@Test
public void testAnotherUpsert() {
DBCollection collection = newCollection();
BasicDBObjectBuilder queryBuilder =
BasicDBObjectBuilder.start()
.push("_id")
.append("f", "ca")
.push("1")
.append("l", 2)
.pop()
.push("t")
.append("t", 11)
.pop()
.pop();
DBObject query = queryBuilder.get();
DBObject update =
BasicDBObjectBuilder.start()
.push("$inc")
.append("n.!", 1)
.append("n.a.b:false", 1)
.pop()
.get();
collection.update(query, update, true, false);
DBObject expected =
queryBuilder.push("n").append("!", 1).push("a").append("b:false", 1).pop().pop().get();
assertEquals(expected, collection.findOne());
}
public static final DBObject createFilter(EventFilter filter) {
BasicDBObjectBuilder builder = new BasicDBObjectBuilder();
if (filter != null) {
if (filter.getUser() != null) {
builder.add("user_id", filter.getUser());
} else if (filter.getProject() != null) {
builder.add("project_id", filter.getProject());
}
if (filter.getMeter() != null) {
builder.add("name", filter.getMeter());
}
if (filter.getStart() != null) {
builder.add("start", new BasicDBObject("$gte", filter.getStart()));
}
if (filter.getEnd() != null) {
builder.add("end", new BasicDBObject("$lt", filter.getEnd()));
}
if (filter.getResource() != null) {
builder.add("resource_id", filter.getResource());
}
if (filter.getSource() != null) {
builder.add("source", filter.getSource());
}
}
return builder.get();
}
public void introduceType(SpaceTypeDescriptor typeDescriptor) {
DBCollection m = getConnection().getCollection(METADATA_COLLECTION_NAME);
BasicDBObjectBuilder builder =
BasicDBObjectBuilder.start().add(Constants.ID_PROPERTY, typeDescriptor.getTypeName());
try {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ObjectOutputStream out = new ObjectOutputStream(bos);
IOUtils.writeObject(
out, SpaceTypeDescriptorVersionedSerializationUtils.toSerializableForm(typeDescriptor));
builder.add(TYPE_DESCRIPTOR_FIELD_NAME, bos.toByteArray());
WriteResult wr = m.save(builder.get());
if (logger.isTraceEnabled()) logger.trace(wr);
indexBuilder.ensureIndexes(typeDescriptor);
} catch (IOException e) {
logger.error(e);
throw new SpaceMongoException(
"error occurs while serialize and save type descriptor: " + typeDescriptor, e);
}
}
@Override
public DBObject toJSON() {
BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
BasicDBList jsonList = new BasicDBList();
Iterator<VoronoiSite> sitesIt = this.sites.iterator();
while (sitesIt.hasNext()) {
jsonList.add(sitesIt.next().toJSON());
}
builder.add("sites", jsonList);
jsonList = new BasicDBList();
Iterator<VoronoiCorner> cornersIt = this.corners.iterator();
while (cornersIt.hasNext()) {
jsonList.add(cornersIt.next().toJSON());
}
builder.add("corners", jsonList);
jsonList = new BasicDBList();
Iterator<VoronoiEdge> edgesIt = this.edges.iterator();
while (edgesIt.hasNext()) {
jsonList.add(edgesIt.next().toJSON());
}
builder.add("edges", jsonList);
return builder.get();
}
/** Does a deep copy of an object to allow for subsequent modification */
public static DBObject copyDBObject(DBObject dbObject) {
DBObject orig = dbObject;
BasicDBObjectBuilder dbObjectBuilder = BasicDBObjectBuilder.start();
for (String field : orig.keySet()) {
Object value = orig.get(field);
dbObjectBuilder.add(field, value);
}
return dbObjectBuilder.get();
}
public void perform(BasicDBObjectBuilder builder) {
BasicDBList list = new BasicDBList();
for (Operator operator : operators) {
BasicDBObjectBuilder operatorBuilder = new BasicDBObjectBuilder();
operator.perform(operatorBuilder);
list.add(operatorBuilder.get());
}
builder.append("$or", list);
}
public static DBObject toDBObject(Person p) {
BasicDBObjectBuilder builder =
BasicDBObjectBuilder.start()
.append("name", p.getName())
.append("surname", p.getSurname())
.append("phone", p.getPhone());
if (p.getId() != null) builder = builder.append("_id", new ObjectId(p.getId()));
return builder.get();
}
/*
* (non-Javadoc)
* @see org.springframework.data.mongodb.core.query.CriteriaDefinition#getCriteriaObject()
*/
@Override
public DBObject getCriteriaObject() {
BasicDBObjectBuilder builder = new BasicDBObjectBuilder();
if (StringUtils.hasText(language)) {
builder.add("$language", language);
}
if (!terms.isEmpty()) {
builder.add("$search", join(terms));
}
return new BasicDBObject("$text", builder.get());
}
/**
* Returns the current CommitEntry as a mongo DBObject.
*
* @return DBObject representing the CommitEntry
*/
public DBObject asDBObject() {
final BasicDBList events = new BasicDBList();
BasicDBObjectBuilder commitBuilder =
BasicDBObjectBuilder.start()
.add(AGGREGATE_IDENTIFIER_PROPERTY, aggregateIdentifier)
.add(SEQUENCE_NUMBER_PROPERTY, firstSequenceNumber)
.add(LAST_SEQUENCE_NUMBER_PROPERTY, lastSequenceNumber)
.add(FIRST_SEQUENCE_NUMBER_PROPERTY, firstSequenceNumber)
.add(TIME_STAMP_PROPERTY, firstTimestamp)
.add(FIRST_TIME_STAMP_PROPERTY, firstTimestamp)
.add(LAST_TIME_STAMP_PROPERTY, lastTimestamp)
.add(EVENTS_PROPERTY, events);
for (EventEntry eventEntry : eventEntries) {
events.add(eventEntry.asDBObject());
}
return commitBuilder.get();
}
protected <T> void ensureIndex(
Class<T> clazz,
String name,
BasicDBObject fields,
boolean unique,
boolean dropDupsOnCreate,
boolean background,
boolean sparse) {
// validate field names and translate them to the stored values
BasicDBObject keys = new BasicDBObject();
for (Entry<String, Object> entry : fields.entrySet()) {
StringBuffer sb = new StringBuffer(entry.getKey());
Mapper.validate(clazz, mapr, sb, FilterOperator.IN, "", true, false);
keys.put(sb.toString(), entry.getValue());
}
BasicDBObjectBuilder keyOpts = new BasicDBObjectBuilder();
if (name != null && name.length() > 0) {
keyOpts.add("name", name);
}
if (unique) {
keyOpts.add("unique", true);
if (dropDupsOnCreate) keyOpts.add("dropDups", true);
}
if (background) keyOpts.add("background", true);
if (sparse) keyOpts.add("sparse", true);
DBCollection dbColl = getCollection(clazz);
BasicDBObject opts = (BasicDBObject) keyOpts.get();
if (opts.isEmpty()) {
log.debug("Ensuring index for " + dbColl.getName() + " with keys:" + keys);
dbColl.ensureIndex(keys);
} else {
log.debug(
"Ensuring index for " + dbColl.getName() + " with keys:" + keys + " and opts:" + opts);
dbColl.ensureIndex(keys, opts);
}
// TODO: remove this once using 2.4 driver does this in ensureIndex
CommandResult cr = dbColl.getDB().getLastError();
cr.throwOnError();
}
@SuppressWarnings({"rawtypes"})
public void ensureIndex(
Class clazz,
String name,
IndexFieldDef[] defs,
boolean unique,
boolean dropDupsOnCreate,
boolean background) {
BasicDBObjectBuilder keys = BasicDBObjectBuilder.start();
for (IndexFieldDef def : defs) {
String fieldName = def.getField();
IndexDirection dir = def.getDirection();
keys.add(fieldName, dir.toIndexValue());
}
ensureIndex(
clazz, name, (BasicDBObject) keys.get(), unique, dropDupsOnCreate, background, false);
}
/**
* Return an iterable of dictionaries containing resource information.
*
* @param user
* @param project
* @param source
*/
public List<Resources.Resource> getResources(String source, String project, String user) {
BasicDBObjectBuilder dboBuilder = new BasicDBObjectBuilder();
if (source != null) {
// dboBuilder.add("source", user);
}
if (project != null) {
dboBuilder.add("project_id", user);
}
if (user != null) {
dboBuilder.add("user_id", user);
}
DBCursor dbCursor = db.getCollection("resource").find(dboBuilder.get());
List<Resources.Resource> resources = new ArrayList<Resources.Resource>();
for (DBObject dbo : dbCursor) {
Resources.Resource resource = new Resources.Resource();
resource.setId((String) dbo.get("_id"));
resource.setProjectId((String) dbo.get("project_id"));
resource.setUserId((String) dbo.get("user_id"));
resource.setTimestamp((Long) dbo.get("timestamp"));
Object metadataDbo = dbo.get("metadata");
// null System.out.println(metadataDbo.getClass().getCanonicalName());
Metadata metadata = new Metadata();
resource.setMetadata(metadata);
List<Meter> meters = new ArrayList<Meter>();
BasicDBList dbList = (BasicDBList) dbo.get("meter");
for (Object o : dbList) {
if (o != null && o instanceof DBObject) {
DBObject meterDbo = (DBObject) o;
Meter meter = new Meter();
meter.setName((String) meterDbo.get("counter_name"));
meter.setType((String) meterDbo.get("counter_type"));
}
}
resource.setMeters(meters);
resources.add(resource);
}
return resources;
}
private static DBObject normalize(DBObject obj) {
BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
Iterator<String> iterator = obj.keySet().iterator();
builder.push("$set");
while (iterator.hasNext()) {
String key = iterator.next();
if (Constants.ID_PROPERTY.equals(key)) continue;
Object value = obj.get(key);
if (value == null) continue;
builder.add(key, value);
}
return builder.get();
}
@SuppressWarnings("rawtypes")
public <T> MapreduceResults<T> mapReduce(
MapreduceType type,
Query query,
String map,
String reduce,
String finalize,
Map<String, Object> scopeFields,
Class<T> outputType) {
Assert.parametersNotNull("map", map);
Assert.parameterNotEmpty(map, "map");
Assert.parametersNotNull("reduce", reduce);
Assert.parameterNotEmpty(reduce, "reduce");
QueryImpl<T> qi = (QueryImpl<T>) query;
DBCollection dbColl = qi.getCollection();
// TODO remove this after testing.
if (dbColl == null) dbColl = getCollection(qi.getEntityClass());
if (log.isTraceEnabled())
log.info(
"Executing mapReduce("
+ dbColl.getName()
+ ") with query("
+ qi.toString()
+ ") map("
+ map
+ ") reduce("
+ reduce
+ ") finalize("
+ finalize
+ ") scope("
+ scopeFields
+ ")");
// TODO replace this with the 2.4 driver impl.
String outColl = mapr.getCollectionName(outputType);
BasicDBObjectBuilder bldr =
BasicDBObjectBuilder.start("mapreduce", mapr.getCollectionName(qi.getEntityClass()));
switch (type) {
case REDUCE:
bldr.push("out").add("reduce", outColl).pop();
break;
case MERGE:
bldr.push("out").add("merge", outColl).pop();
break;
case INLINE:
bldr.push("out").add("inline", 1).pop();
break;
default:
bldr.add("out", outColl);
break;
}
if (qi.getOffset() != 0 || qi.getFieldsObject() != null)
throw new QueryException(
"mapReduce does not allow the offset/retrievedFields query options.");
if (qi.getQueryObject() != null) bldr.add("query", qi.getQueryObject());
if (qi.getLimit() > 0) bldr.add("limit", qi.getLimit());
if (qi.getSortObject() != null) bldr.add("sort", qi.getSortObject());
bldr.add("map", map);
bldr.add("reduce", reduce);
if (finalize != null && finalize.length() > 0) bldr.add("finalize", finalize);
if (scopeFields != null && scopeFields.size() > 0)
bldr.add("scope", mapr.toMongoObject(null, null, scopeFields));
DBObject dbObj = bldr.get();
CommandResult cr = dbColl.getDB().command(dbObj);
cr.throwOnError();
MapreduceResults mrRes =
(MapreduceResults) mapr.fromDBObject(MapreduceResults.class, cr, createCache());
QueryImpl baseQ = null;
if (!MapreduceType.INLINE.equals(type))
baseQ = new QueryImpl(outputType, db.getCollection(mrRes.getOutputCollectionName()), this);
// TODO Handle inline case and create an iterator/able.
mrRes.setBits(type, baseQ);
return mrRes;
}
public DBObject buildDbObject(JSONObject jsonObject) {
if (!check(jsonObject)) {
return null;
}
BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
JSONObject mobile = jsonObject.getJSONObject(FieldName.Tanx.mobile);
JSONObject device = mobile.getJSONObject(FieldName.Tanx.device);
String deviceId = device.getString(FieldName.Tanx.device_id);
// 过滤非法deviceId
if (deviceId.contains("000000000000000") || deviceId.contains(":")) {
logger.info("=== ignore deviceId:{}", deviceId);
return null;
}
// 新增， 不更新的
BasicDBObject setObj =
new BasicDBObject(FieldName.field_deviceId, deviceId)
.append(FieldName.field_deviceIdMd5, DigestUtils.md5Hex(deviceId))
.append(FieldName.field_deviceIdSha1, DigestUtils.sha1Hex(deviceId));
// brand
if (StringUtils.isNotBlank(device.getString(FieldName.Tanx.brand))) {
setObj.append(FieldName.field_brand, device.getString(FieldName.Tanx.brand));
}
// model
if (StringUtils.isNotBlank(device.getString(FieldName.Tanx.model))) {
setObj.append(FieldName.field_model, device.getString(FieldName.Tanx.model));
}
// platform
if (StringUtils.isNotBlank(device.getString(FieldName.Tanx.platform))) {
setObj.append(FieldName.field_platform, device.getString(FieldName.Tanx.platform));
}
// os
if (StringUtils.isNotBlank(device.getString(FieldName.Tanx.os))) {
setObj.append(FieldName.field_os, device.getString(FieldName.Tanx.os));
}
// os_version
if (StringUtils.isNotBlank(device.getString(FieldName.Tanx.os_version))) {
setObj.append(FieldName.field_osVersion, device.getString(FieldName.Tanx.os_version));
}
// mac
String mac = device.getString(FieldName.Tanx.mac);
if (StringUtils.isNotBlank(mac) && mac.contains(":")) {
setObj
.append(FieldName.field_mac, mac)
.append(FieldName.field_macMd5, DigestUtils.md5Hex(mac))
.append(FieldName.field_macSha1, DigestUtils.sha1Hex(mac));
}
// device_size
if (StringUtils.isNotBlank(device.getString(FieldName.Tanx.device_size))) {
setObj.append(FieldName.field_deviceSize, device.getString(FieldName.Tanx.device_size));
}
// builder.add(MongoSink.OP_SET_ON_INSERT, setObj);
BasicDBObject addToSetObj = new BasicDBObject();
// ip
if (StringUtils.isNotBlank(jsonObject.getString(FieldName.Tanx.ip))) {
addToSetObj.append(FieldName.field_ipList, jsonObject.getString(FieldName.Tanx.ip));
}
// 城市名称
if (StringUtils.isNotBlank(jsonObject.getString(FieldName.Tanx.city))) {
addToSetObj.append(FieldName.field_cityList, jsonObject.getString(FieldName.Tanx.city));
}
// 追加的数据，geo
if (StringUtils.isNotBlank(device.getString(FieldName.Tanx.latitude))
&& StringUtils.isNotBlank(device.getString(FieldName.Tanx.longitude))) {
addToSetObj.append(
FieldName.field_geoList,
new BasicDBObject(FieldName.field_latitude, device.getString(FieldName.Tanx.latitude))
.append(FieldName.field_longitude, device.getString(FieldName.Tanx.longitude)));
}
builder.add(MongoSink.OP_ADD_TO_SET, addToSetObj);
// app
if (StringUtils.isNotBlank(mobile.getString(FieldName.Tanx.package_name))) {
BasicDBObject appObj = new BasicDBObject();
appObj.append(FieldName.field_packageName, mobile.getString(FieldName.Tanx.package_name));
if (StringUtils.isNotBlank(mobile.getString(FieldName.Tanx.app_name))) {
appObj.append(FieldName.field_appName, mobile.getString(FieldName.Tanx.app_name));
}
// app分类
if (mobile.getJSONArray(FieldName.Tanx.app_categories) != null
&& !mobile.getJSONArray(FieldName.Tanx.app_categories).isEmpty()) {
appObj.append(
FieldName.field_appCategorys,
mobile
.getJSONArray(FieldName.Tanx.app_categories)
.getJSONObject(0)
.getString(FieldName.Tanx.id));
}
if (StringUtils.isNotBlank(jsonObject.getString(FieldName.bidRequestTime))) {
appObj.append(FieldName.lastUseTime, jsonObject.getString(FieldName.bidRequestTime));
}
// mongo 不支持包含点. 的做key， 替换为_下划线
setObj.append(
FieldName.field_appList + mobile.getString(FieldName.Tanx.package_name).replace(".", "_"),
appObj);
}
if (addToSetObj.isEmpty()) {
return null;
}
// app需要记录最后使用日期， 每次都更新
builder.add(MongoSink.OP_SET, setObj);
return builder.get();
}
