/**
* Look a the values of the given primary key in the object and return true if and only if some
* part of the primary key is null. If the primary key contains a reference it is sufficient for
* any of the primary keys of the referenced object to be non-null (ie objectPrimaryKeyIsNull()
* returning true).
*
* @param model the Model in which to find ClassDescriptors
* @param obj the Object to check
* @param cld one of the classes that obj is. Only primary keys for this classes will be checked
* @param pk the primary key to check
* @param source the Source database
* @param idMap an IntToIntMap from source IDs to destination IDs
* @return true if the the given primary key is non-null for the given object
* @throws MetaDataException if anything goes wrong
*/
public static boolean objectPrimaryKeyNotNull(
Model model,
InterMineObject obj,
ClassDescriptor cld,
PrimaryKey pk,
Source source,
IntToIntMap idMap)
throws MetaDataException {
for (String fieldName : pk.getFieldNames()) {
FieldDescriptor fd = cld.getFieldDescriptorByName(fieldName);
if (fd instanceof AttributeDescriptor) {
Object value;
try {
value = obj.getFieldValue(fieldName);
} catch (IllegalAccessException e) {
throw new MetaDataException(
"Failed to get field " + fieldName + " for key " + pk + " from " + obj, e);
}
if (value == null) {
return false;
}
} else if (fd instanceof CollectionDescriptor) {
throw new MetaDataException(
"Primary key "
+ pk.getName()
+ " for class "
+ cld.getName()
+ " cannot contain collection "
+ fd.getName()
+ ": collections cannot be part of a primary key. Please edit"
+ model.getName()
+ "_keyDefs.properties");
} else if (fd instanceof ReferenceDescriptor) {
InterMineObject refObj;
try {
refObj = (InterMineObject) obj.getFieldProxy(fieldName);
} catch (IllegalAccessException e) {
throw new MetaDataException(
"Failed to get field " + fieldName + " for key " + pk + " from " + obj, e);
}
if (refObj == null) {
return false;
}
if ((refObj.getId() != null) && (idMap.get(refObj.getId()) != null)) {
// We have previously loaded the object in this reference.
continue;
}
if (refObj instanceof ProxyReference) {
refObj = ((ProxyReference) refObj).getObject();
}
boolean foundNonNullKey = false;
boolean foundKey = false;
Set<ClassDescriptor> classDescriptors =
model.getClassDescriptorsForClass(refObj.getClass());
CLDS:
for (ClassDescriptor refCld : classDescriptors) {
Set<PrimaryKey> primaryKeys;
if (source == null) {
primaryKeys =
new LinkedHashSet<PrimaryKey>(PrimaryKeyUtil.getPrimaryKeys(refCld).values());
} else {
primaryKeys = DataLoaderHelper.getPrimaryKeys(refCld, source, null);
}
for (PrimaryKey refPK : primaryKeys) {
foundKey = true;
if (objectPrimaryKeyNotNull(model, refObj, refCld, refPK, source, idMap)) {
foundNonNullKey = true;
break CLDS;
}
}
}
if (foundKey && (!foundNonNullKey)) {
return false;
}
}
}
return true;
}
/**
* Fetches the equivalent object information for a whole batch of objects.
*
* @param batch the objects
* @throws ObjectStoreException if something goes wrong
*/
protected void getEquivalentsFor(List<ResultsRow<Object>> batch) throws ObjectStoreException {
long time = System.currentTimeMillis();
long time1 = time;
boolean databaseEmpty = hints.databaseEmpty();
if (savedDatabaseEmptyFetch == -1) {
savedDatabaseEmptyFetch = System.currentTimeMillis() - time;
}
if (databaseEmpty) {
savedDatabaseEmpty++;
return;
}
// TODO: add all the objects that are referenced by these objects, and follow primary keys
// We can make use of the ObjectStoreFastCollectionsForTranslatorImpl's ability to work this
// all out for us.
Set<InterMineObject> objects = new HashSet<InterMineObject>();
for (ResultsRow<Object> row : batch) {
for (Object object : row) {
if (object instanceof InterMineObject) {
InterMineObject imo = (InterMineObject) object;
if (idMap.get(imo.getId()) == null) {
objects.add(imo);
for (String fieldName : TypeUtil.getFieldInfos(imo.getClass()).keySet()) {
Object fieldValue;
try {
fieldValue = imo.getFieldProxy(fieldName);
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
if ((fieldValue instanceof InterMineObject)
&& (!(fieldValue instanceof ProxyReference))) {
objects.add((InterMineObject) fieldValue);
} else if (fieldValue instanceof Collection<?>) {
for (Object collectionElement : ((Collection<?>) fieldValue)) {
if ((collectionElement instanceof InterMineObject)
&& (!(collectionElement instanceof ProxyReference))) {
objects.add((InterMineObject) collectionElement);
}
}
}
}
}
}
}
}
objects.removeAll(equivalents.keySet());
// Now objects contains all the objects we need to fetch data for.
Map<InterMineObject, Set<InterMineObject>> results =
new HashMap<InterMineObject, Set<InterMineObject>>();
for (InterMineObject object : objects) {
results.put(object, Collections.synchronizedSet(new HashSet<InterMineObject>()));
}
Map<PrimaryKey, ClassDescriptor> pksToDo = new IdentityHashMap<PrimaryKey, ClassDescriptor>();
Map<ClassDescriptor, List<InterMineObject>> cldToObjectsForCld =
new IdentityHashMap<ClassDescriptor, List<InterMineObject>>();
Map<Class<?>, List<InterMineObject>> categorised = CollectionUtil.groupByClass(objects, false);
Map<ClassDescriptor, Boolean> cldsDone = new IdentityHashMap<ClassDescriptor, Boolean>();
for (Class<?> c : categorised.keySet()) {
Set<ClassDescriptor> classDescriptors = model.getClassDescriptorsForClass(c);
for (ClassDescriptor cld : classDescriptors) {
if (!cldsDone.containsKey(cld)) {
cldsDone.put(cld, Boolean.TRUE);
Set<PrimaryKey> keysForClass;
if (source == null) {
keysForClass = new HashSet<PrimaryKey>(PrimaryKeyUtil.getPrimaryKeys(cld).values());
} else {
keysForClass = DataLoaderHelper.getPrimaryKeys(cld, source, lookupOs);
}
if (!keysForClass.isEmpty()) {
time = System.currentTimeMillis();
boolean classNotExists = hints.classNotExists(cld.getType());
String className = Util.getFriendlyName(cld.getType());
if (!savedTimes.containsKey(className)) {
savedTimes.put(className, new Long(System.currentTimeMillis() - time));
}
if (!classNotExists) {
// LOG.error("Inspecting class " + className);
List<InterMineObject> objectsForCld = new ArrayList<InterMineObject>();
for (Map.Entry<Class<?>, List<InterMineObject>> category : categorised.entrySet()) {
if (cld.getType().isAssignableFrom(category.getKey())) {
objectsForCld.addAll(category.getValue());
}
}
cldToObjectsForCld.put(cld, objectsForCld);
// So now we have a list of objects for this CLD.
for (PrimaryKey pk : keysForClass) {
// LOG.error("Adding pk " + cld.getName() + "." + pk.getName());
pksToDo.put(pk, cld);
}
} else {
// LOG.error("Empty class " + className);
}
}
}
}
}
doPks(pksToDo, results, cldToObjectsForCld, time1);
batchQueried += results.size();
equivalents.putAll(results);
}
/**
* Return a Set of PrimaryKeys relevant to a given Source for a ClassDescriptor. The Set contains
* all the primary keys that exist on a particular class that are used by the source, without
* performing any recursion. The Model.getClassDescriptorsForClass() method is recommended if you
* wish for all the primary keys of the class' parents as well.
*
* @param cld the ClassDescriptor
* @param source the Source
* @param os the ObjectStore that these PrimaryKeys are used in, for creating indexes
* @return a Set of PrimaryKeys
*/
public static Set<PrimaryKey> getPrimaryKeys(ClassDescriptor cld, Source source, ObjectStore os) {
GetPrimaryKeyCacheKey key = new GetPrimaryKeyCacheKey(cld, source);
synchronized (getPrimaryKeyCache) {
Set<PrimaryKey> keySet = getPrimaryKeyCache.get(key);
if (keySet == null) {
keySet = new LinkedHashSet<PrimaryKey>();
Properties keys = getKeyProperties(source);
if (keys != null) {
if (!verifiedSources.contains(source)) {
String packageNameWithDot =
cld.getName().substring(0, cld.getName().lastIndexOf('.') + 1);
LOG.info(
"Verifying primary key config for source "
+ source
+ ", packageName = "
+ packageNameWithDot);
for (Map.Entry<Object, Object> entry : keys.entrySet()) {
String cldName = (String) entry.getKey();
String keyList = (String) entry.getValue();
if (!cldName.contains(".")) {
ClassDescriptor iCld =
cld.getModel().getClassDescriptorByName(packageNameWithDot + cldName);
if (iCld != null) {
Map<String, PrimaryKey> map = PrimaryKeyUtil.getPrimaryKeys(iCld);
String[] tokens = keyList.split(",");
for (int i = 0; i < tokens.length; i++) {
String token = tokens[i].trim();
if (map.get(token) == null) {
throw new IllegalArgumentException(
"Primary key "
+ token
+ " for class "
+ cldName
+ " required by datasource "
+ source.getName()
+ " in "
+ source.getName()
+ "_keys.properties is not defined in "
+ cld.getModel().getName()
+ "_keyDefs.properties");
}
}
} else {
LOG.warn(
"Ignoring entry for "
+ cldName
+ " in file "
+ cld.getModel().getName()
+ "_keyDefs.properties - not in model!");
}
}
}
verifiedSources.add(source);
}
Map<String, PrimaryKey> map = PrimaryKeyUtil.getPrimaryKeys(cld);
String cldName = TypeUtil.unqualifiedName(cld.getName());
String keyList = (String) keys.get(cldName);
if (keyList != null) {
String[] tokens = keyList.split(",");
for (int i = 0; i < tokens.length; i++) {
String token = tokens[i].trim();
if (map.get(token) == null) {
throw new IllegalArgumentException(
"Primary key "
+ token
+ " for class "
+ cld.getName()
+ " required by data source "
+ source.getName()
+ " in "
+ source.getName()
+ "_keys.properties is not defined in "
+ cld.getModel().getName()
+ "_keyDefs.properties");
} else {
keySet.add(map.get(token));
}
}
}
for (Map.Entry<Object, Object> entry : keys.entrySet()) {
String propKey = (String) entry.getKey();
String fieldList = (String) entry.getValue();
int posOfDot = propKey.indexOf('.');
if (posOfDot > 0) {
String propCldName = propKey.substring(0, posOfDot);
if (cldName.equals(propCldName)) {
String keyName = propKey.substring(posOfDot + 1);
PrimaryKey pk = new PrimaryKey(keyName, fieldList, cld);
if (!keySet.contains(pk)) {
keySet.add(pk);
if (os instanceof ObjectStoreInterMineImpl) {
ObjectStoreInterMineImpl osimi = (ObjectStoreInterMineImpl) os;
DatabaseSchema schema = osimi.getSchema();
ClassDescriptor tableMaster = schema.getTableMaster(cld);
String tableName = DatabaseUtil.getTableName(tableMaster);
List<String> fields = new ArrayList<String>();
for (String field : pk.getFieldNames()) {
String colName = DatabaseUtil.generateSqlCompatibleName(field);
if (tableMaster.getReferenceDescriptorByName(field, true) != null) {
colName += "id";
}
fields.add(colName);
}
String sql =
"CREATE INDEX "
+ tableName
+ "__"
+ keyName
+ " ON "
+ tableName
+ " ("
+ StringUtil.join(fields, ", ")
+ ")";
System.out.println("Creating index: " + sql);
LOG.info("Creating index: " + sql);
Connection conn = null;
try {
conn = osimi.getConnection();
conn.createStatement().execute(sql);
} catch (SQLException e) {
LOG.warn("Index creation failed", e);
} finally {
if (conn != null) {
osimi.releaseConnection(conn);
}
}
}
}
}
}
}
} else {
throw new IllegalArgumentException(
"Unable to find keys for source "
+ source.getName()
+ " in file "
+ source.getName()
+ "_keys.properties");
}
getPrimaryKeyCache.put(key, keySet);
}
return keySet;
}
}