public static void updateMatchingEntities(EntityFeaturePojo entFeature, ObjectId communityId) {
String index = entFeature.getIndex();
long totalFreq = entFeature.getTotalfreq();
long docCount = entFeature.getDoccount();
try {
DBCollection docDb = DbManager.getDocument().getMetadata();
BasicDBObject query1 = new BasicDBObject();
query1.put(EntityPojo.docQuery_index_, index);
query1.put(DocumentPojo.communityId_, communityId);
BasicDBObject multiopB = new BasicDBObject();
multiopB.put(EntityPojo.docUpdate_totalfrequency_, totalFreq);
multiopB.put(EntityPojo.docUpdate_doccount_, docCount);
BasicDBObject multiopA = new BasicDBObject(MongoDbManager.set_, multiopB);
if (_diagnosticMode) {
System.out.println(
"EntityAggregationUtils.updateMatchingEntities: "
+ query1.toString()
+ " / "
+ multiopA.toString());
} else {
synchronized (GenericProcessingController.class) {
// Because this op can be slow, and traverse a lot of disk, need to ensure that
// we don't allow all the threads to hammer it at once (the updates all yield to each
// other
// enough that the disk goes totally crazy)
docDb.update(query1, multiopA, false, true);
DbManager.getDocument().getLastError(DbManager.getDocument().getMetadata().getName());
// (enforce consecutive accesses for this potentially very slow operation)
}
// Was originally checked updatedExisting but for INF-1406, it sometimes seemed to be
// checking the wrong command. I suspect the reason we had this code in here has gone away,
// and it doesn't matter if this update occasionally fails anyway, it will just be out of
// date
// so the check/retry has been removed.
}
} catch (Exception ex) {
logger.error(ex.getMessage(), ex);
}
} // TESTED (by eye, mostly cut-and-paste from test Beta)
/**
* Updates the feature entries for the list of entities that was just extracted including changing
* frequency, adding aliases etc
*
* <p>This method now has 3 steps: 1. Try to update alias 1.a If fail, create new gaz 2. Update
* totalfreq and doccount
*
* @param ents List of entities to update in the entity feature
*/
public static void updateEntityFeatures(
Map<String, Map<ObjectId, EntityFeaturePojo>> entFeatures) {
DBCollection col = DbManager.getFeature().getEntity();
String savedSyncTime = null;
for (Map<ObjectId, EntityFeaturePojo> entCommunity : entFeatures.values()) {
Iterator<Map.Entry<ObjectId, EntityFeaturePojo>> it = entCommunity.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<ObjectId, EntityFeaturePojo> entFeatureKV = it.next();
try {
EntityFeaturePojo entFeature = entFeatureKV.getValue();
long nSavedDocCount = entFeature.getDoccount();
long nSavedFreqCount = entFeature.getTotalfreq();
// (these should be constant across all communities but keep it here
// so can assign it using entFeature, it's v cheap so no need to get once like for sync
// vars)
ObjectId communityID = entFeature.getCommunityId();
if (null != communityID) {
// For each community, see if the entity feature already exists *for that community*
BasicDBObject query =
new BasicDBObject(EntityFeaturePojo.index_, entFeature.getIndex());
query.put(EntityFeaturePojo.communityId_, communityID);
BasicDBObject updateOp = new BasicDBObject();
// Add aliases:
BasicDBObject updateOpA = new BasicDBObject();
BasicDBObject multiopE = new BasicDBObject(MongoDbManager.each_, entFeature.getAlias());
updateOpA.put(EntityFeaturePojo.alias_, multiopE);
// Add link data, if there is any:
if ((null != entFeature.getSemanticLinks())
&& !entFeature.getSemanticLinks().isEmpty()) {
BasicDBObject multiopF =
new BasicDBObject(MongoDbManager.each_, entFeature.getSemanticLinks());
updateOpA.put(EntityFeaturePojo.linkdata_, multiopF);
}
updateOp.put(MongoDbManager.addToSet_, updateOpA);
// Update frequency:
BasicDBObject updateOpB = new BasicDBObject();
updateOpB.put(EntityFeaturePojo.totalfreq_, nSavedFreqCount);
updateOpB.put(EntityFeaturePojo.doccount_, nSavedDocCount);
updateOp.put(MongoDbManager.inc_, updateOpB);
// try to use find/modify to see if something comes back and set doc freq/totalfreq
BasicDBObject fields = new BasicDBObject(EntityFeaturePojo.totalfreq_, 1);
fields.put(EntityFeaturePojo.doccount_, 1);
fields.put(EntityFeaturePojo.alias_, 1);
fields.put(EntityFeaturePojo.linkdata_, 1);
// (slightly annoying, since only want these 2 largish fields if updating freq but won't
// know
// until after i've got this object)
fields.put(EntityFeaturePojo.db_sync_time_, 1);
fields.put(EntityFeaturePojo.db_sync_doccount_, 1);
DBObject dboUpdate = null;
if (_diagnosticMode) {
dboUpdate = col.findOne(query, fields);
} else {
dboUpdate =
col.findAndModify(
query, fields, new BasicDBObject(), false, updateOp, false, true);
// (can use findAndModify because specify index, ie the shard key)
// (returns entity before the changes above, update the feature object below)
// (also atomically creates the object if it doesn't exist so is "distributed-safe")
}
if ((dboUpdate != null) && !dboUpdate.keySet().isEmpty()) {
// (Update the entity feature to be correct so that it can be accurately synchronized
// with the index)
EntityFeaturePojo gp = EntityFeaturePojo.fromDb(dboUpdate, EntityFeaturePojo.class);
entFeature.setTotalfreq(gp.getTotalfreq() + nSavedFreqCount);
entFeature.setDoccount(gp.getDoccount() + nSavedDocCount);
entFeature.setDbSyncDoccount(gp.getDbSyncDoccount());
entFeature.setDbSyncTime(gp.getDbSyncTime());
if (null != gp.getAlias()) {
entFeature.addAllAlias(gp.getAlias());
}
if (null != gp.getSemanticLinks()) {
entFeature.addToSemanticLinks(gp.getSemanticLinks());
}
if (_diagnosticMode) {
System.out.println(
"EntityAggregationUtils.updateEntityFeatures, found: "
+ ((BasicDBObject) gp.toDb()).toString());
System.out.println(
"EntityAggregationUtils.updateEntityFeatures, ^^^ found from query: "
+ query.toString()
+ " / "
+ updateOp.toString());
}
} else // (the object in memory is now an accurate representation of the database, minus
// some fields we'll now add)
{
// Synchronization settings for the newly created object
if (null == savedSyncTime) {
savedSyncTime = Long.toString(System.currentTimeMillis());
}
entFeature.setDbSyncDoccount(nSavedDocCount);
entFeature.setDbSyncTime(savedSyncTime);
// This is all "distributed safe" (apart from the db_syc_xxx and it doesn't matter if
// that is
// out of date, the update will just be slightly out-of-date at worst) since
// (otherwise) these fields are
// only set here, and the findAndModify is atomic
// (Do in raw MongoDB for performance)
BasicDBObject baseFields = new BasicDBObject();
baseFields.put(EntityFeaturePojo.dimension_, entFeature.getDimension().toString());
baseFields.put(EntityFeaturePojo.type_, entFeature.getType());
baseFields.put(
EntityFeaturePojo.disambiguated_name_, entFeature.getDisambiguatedName());
baseFields.put(EntityFeaturePojo.db_sync_doccount_, entFeature.getDbSyncDoccount());
baseFields.put(EntityFeaturePojo.db_sync_time_, entFeature.getDbSyncTime());
if ((null != entFeature.getSemanticLinks())
&& !entFeature.getSemanticLinks().isEmpty()) {
baseFields.put(EntityFeaturePojo.linkdata_, entFeature.getSemanticLinks());
}
// attempt to add geotag (makes necessary checks on util side)
// also add ontology type if geotag is found
EntityGeotagAggregationUtils.addEntityGeo(entFeature);
if (entFeature.getGeotag() != null) {
BasicDBObject geo = new BasicDBObject(GeoPojo.lat_, entFeature.getGeotag().lat);
geo.put(GeoPojo.lon_, entFeature.getGeotag().lon);
baseFields.put(EntityFeaturePojo.geotag_, geo);
if (entFeature.getOntology_type() != null) {
baseFields.put(EntityFeaturePojo.ontology_type_, entFeature.getOntology_type());
}
}
if (!_diagnosticMode) {
// Store the object
col.update(query, new BasicDBObject(MongoDbManager.set_, baseFields));
} else {
System.out.println(
"EntityAggregationUtils.updateEntityFeatures, not found: "
+ query.toString()
+ ": "
+ baseFields.toString());
}
entFeature.setDbSyncTime(null); // (ensures that index re-sync will occur)
}
}
} catch (Exception e) {
// Exception, remove from feature list
it.remove();
// If an exception occurs log the error
logger.error("Exception Message: " + e.getMessage(), e);
}
} // (end loop over communities)
} // (end loop over indexes)
} // TESTED (just by eye - made few changes during re-factoring)
