/**
* Fetches equivalent objects for a particular primary key.
*
* @param pk the PrimaryKey
* @param cld the ClassDescriptor of the PrimaryKey
* @param results a Map to hold results that are to be added to the cache
* @param objectsForCld a List of objects relevant to this PrimaryKey
* @param fetchedObjectIds a Set to hold ids of objects that are fetched, to prefetch from the
* data tracker later
* @throws ObjectStoreException if something goes wrong
*/
protected void doPk(
PrimaryKey pk,
ClassDescriptor cld,
Map<InterMineObject, Set<InterMineObject>> results,
List<InterMineObject> objectsForCld,
Set<Integer> fetchedObjectIds)
throws ObjectStoreException {
Iterator<InterMineObject> objectsForCldIter = objectsForCld.iterator();
while (objectsForCldIter.hasNext()) {
int objCount = 0;
int origObjCount = 0;
Query q = new Query();
QueryClass qc = new QueryClass(cld.getType());
q.addFrom(qc);
q.addToSelect(qc);
ConstraintSet cs = new ConstraintSet(ConstraintOp.AND);
q.setConstraint(cs);
Map<String, Set<Object>> fieldNameToValues = new HashMap<String, Set<Object>>();
for (String fieldName : pk.getFieldNames()) {
try {
QueryField qf = new QueryField(qc, fieldName);
q.addToSelect(qf);
Set<Object> values = new HashSet<Object>();
fieldNameToValues.put(fieldName, values);
cs.addConstraint(new BagConstraint(qf, ConstraintOp.IN, values));
} catch (IllegalArgumentException e) {
QueryForeignKey qf = new QueryForeignKey(qc, fieldName);
q.addToSelect(qf);
Set<Object> values = new HashSet<Object>();
fieldNameToValues.put(fieldName, values);
cs.addConstraint(new BagConstraint(qf, ConstraintOp.IN, values));
}
}
// Now make a map from the primary key values to source objects
Map<List<Object>, InterMineObject> keysToSourceObjects =
new HashMap<List<Object>, InterMineObject>();
while (objectsForCldIter.hasNext() && (objCount < 500)) {
InterMineObject object = objectsForCldIter.next();
origObjCount++;
try {
if (DataLoaderHelper.objectPrimaryKeyNotNull(model, object, cld, pk, source, idMap)) {
List<Collection<Object>> values = new ArrayList<Collection<Object>>();
boolean skipObject = false;
Map<String, Set<Object>> fieldsValues = new HashMap<String, Set<Object>>();
for (String fieldName : pk.getFieldNames()) {
try {
Object value = object.getFieldProxy(fieldName);
Set<Object> fieldValues;
if (value instanceof InterMineObject) {
Integer id = idMap.get(((InterMineObject) value).getId());
if (id == null) {
Set<InterMineObject> eqs = results.get(value);
if (eqs == null) {
value = object.getFieldValue(fieldName);
eqs = queryEquivalentObjects((InterMineObject) value, source);
}
fieldValues = new HashSet<Object>();
for (InterMineObject obj : eqs) {
fieldValues.add(obj.getId());
}
} else {
fieldValues = Collections.singleton((Object) id);
}
} else {
fieldValues = Collections.singleton(value);
}
values.add(fieldValues);
fieldsValues.put(fieldName, fieldValues);
for (Object fieldValue : fieldValues) {
long time = System.currentTimeMillis();
boolean pkQueryFruitless =
hints.pkQueryFruitless(cld.getType(), fieldName, fieldValue);
String summaryName = Util.getFriendlyName(cld.getType()) + "." + fieldName;
if (!savedTimes.containsKey(summaryName)) {
savedTimes.put(summaryName, new Long(System.currentTimeMillis() - time));
savedCounts.put(summaryName, new Integer(0));
}
if (pkQueryFruitless) {
skipObject = true;
}
}
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
if (!skipObject) {
objCount++;
for (String fieldName : pk.getFieldNames()) {
fieldNameToValues.get(fieldName).addAll(fieldsValues.get(fieldName));
}
for (List<Object> valueSet : CollectionUtil.fanOutCombinations(values)) {
if (keysToSourceObjects.containsKey(valueSet)) {
throw new ObjectStoreException(
"Duplicate objects found for pk "
+ cld.getName()
+ "."
+ pk.getName()
+ ": "
+ object);
}
keysToSourceObjects.put(valueSet, object);
}
}
}
} catch (MetaDataException e) {
throw new ObjectStoreException(e);
}
}
// Prune BagConstraints using the hints system.
// boolean emptyQuery = false;
// Iterator<String> fieldNameIter = pk.getFieldNames().iterator();
// while (fieldNameIter.hasNext() && (!emptyQuery)) {
// String fieldName = fieldNameIter.next();
// Set values = fieldNameToValues.get(fieldName);
// Iterator valueIter = values.iterator();
// while (valueIter.hasNext()) {
// if (hints.pkQueryFruitless(cld.getType(), fieldName, valueIter.next())) {
// valueIter.remove();
// }
// }
// if (values.isEmpty()) {
// emptyQuery = true;
// }
// }
if (objCount > 0) {
// Iterate through query, and add objects to results
// long time = System.currentTimeMillis();
int matches = 0;
Results res = lookupOs.execute(q, 2000, false, false, false);
@SuppressWarnings("unchecked")
List<ResultsRow<Object>> tmpRes = (List) res;
for (ResultsRow<Object> row : tmpRes) {
List<Object> values = new ArrayList<Object>();
for (int i = 1; i <= pk.getFieldNames().size(); i++) {
values.add(row.get(i));
}
Set<InterMineObject> set = results.get(keysToSourceObjects.get(values));
if (set != null) {
set.add((InterMineObject) row.get(0));
matches++;
}
fetchedObjectIds.add(((InterMineObject) row.get(0)).getId());
}
// LOG.info("Fetched " + res.size() + " equivalent objects for " + objCount
// + " objects in " + (System.currentTimeMillis() - time) + " ms for "
// + cld.getName() + "." + pk.getName());
}
}
}
/**
* Fetches the equivalent object information for a whole batch of objects.
*
* @param batch the objects
* @throws ObjectStoreException if something goes wrong
*/
protected void getEquivalentsFor(List<ResultsRow<Object>> batch) throws ObjectStoreException {
long time = System.currentTimeMillis();
long time1 = time;
boolean databaseEmpty = hints.databaseEmpty();
if (savedDatabaseEmptyFetch == -1) {
savedDatabaseEmptyFetch = System.currentTimeMillis() - time;
}
if (databaseEmpty) {
savedDatabaseEmpty++;
return;
}
// TODO: add all the objects that are referenced by these objects, and follow primary keys
// We can make use of the ObjectStoreFastCollectionsForTranslatorImpl's ability to work this
// all out for us.
Set<InterMineObject> objects = new HashSet<InterMineObject>();
for (ResultsRow<Object> row : batch) {
for (Object object : row) {
if (object instanceof InterMineObject) {
InterMineObject imo = (InterMineObject) object;
if (idMap.get(imo.getId()) == null) {
objects.add(imo);
for (String fieldName : TypeUtil.getFieldInfos(imo.getClass()).keySet()) {
Object fieldValue;
try {
fieldValue = imo.getFieldProxy(fieldName);
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
if ((fieldValue instanceof InterMineObject)
&& (!(fieldValue instanceof ProxyReference))) {
objects.add((InterMineObject) fieldValue);
} else if (fieldValue instanceof Collection<?>) {
for (Object collectionElement : ((Collection<?>) fieldValue)) {
if ((collectionElement instanceof InterMineObject)
&& (!(collectionElement instanceof ProxyReference))) {
objects.add((InterMineObject) collectionElement);
}
}
}
}
}
}
}
}
objects.removeAll(equivalents.keySet());
// Now objects contains all the objects we need to fetch data for.
Map<InterMineObject, Set<InterMineObject>> results =
new HashMap<InterMineObject, Set<InterMineObject>>();
for (InterMineObject object : objects) {
results.put(object, Collections.synchronizedSet(new HashSet<InterMineObject>()));
}
Map<PrimaryKey, ClassDescriptor> pksToDo = new IdentityHashMap<PrimaryKey, ClassDescriptor>();
Map<ClassDescriptor, List<InterMineObject>> cldToObjectsForCld =
new IdentityHashMap<ClassDescriptor, List<InterMineObject>>();
Map<Class<?>, List<InterMineObject>> categorised = CollectionUtil.groupByClass(objects, false);
Map<ClassDescriptor, Boolean> cldsDone = new IdentityHashMap<ClassDescriptor, Boolean>();
for (Class<?> c : categorised.keySet()) {
Set<ClassDescriptor> classDescriptors = model.getClassDescriptorsForClass(c);
for (ClassDescriptor cld : classDescriptors) {
if (!cldsDone.containsKey(cld)) {
cldsDone.put(cld, Boolean.TRUE);
Set<PrimaryKey> keysForClass;
if (source == null) {
keysForClass = new HashSet<PrimaryKey>(PrimaryKeyUtil.getPrimaryKeys(cld).values());
} else {
keysForClass = DataLoaderHelper.getPrimaryKeys(cld, source, lookupOs);
}
if (!keysForClass.isEmpty()) {
time = System.currentTimeMillis();
boolean classNotExists = hints.classNotExists(cld.getType());
String className = Util.getFriendlyName(cld.getType());
if (!savedTimes.containsKey(className)) {
savedTimes.put(className, new Long(System.currentTimeMillis() - time));
}
if (!classNotExists) {
// LOG.error("Inspecting class " + className);
List<InterMineObject> objectsForCld = new ArrayList<InterMineObject>();
for (Map.Entry<Class<?>, List<InterMineObject>> category : categorised.entrySet()) {
if (cld.getType().isAssignableFrom(category.getKey())) {
objectsForCld.addAll(category.getValue());
}
}
cldToObjectsForCld.put(cld, objectsForCld);
// So now we have a list of objects for this CLD.
for (PrimaryKey pk : keysForClass) {
// LOG.error("Adding pk " + cld.getName() + "." + pk.getName());
pksToDo.put(pk, cld);
}
} else {
// LOG.error("Empty class " + className);
}
}
}
}
}
doPks(pksToDo, results, cldToObjectsForCld, time1);
batchQueried += results.size();
equivalents.putAll(results);
}