/** * Look a the values of the given primary key in the object and return true if and only if some * part of the primary key is null. If the primary key contains a reference it is sufficient for * any of the primary keys of the referenced object to be non-null (ie objectPrimaryKeyIsNull() * returning true). * * @param model the Model in which to find ClassDescriptors * @param obj the Object to check * @param cld one of the classes that obj is. Only primary keys for this classes will be checked * @param pk the primary key to check * @param source the Source database * @param idMap an IntToIntMap from source IDs to destination IDs * @return true if the the given primary key is non-null for the given object * @throws MetaDataException if anything goes wrong */ public static boolean objectPrimaryKeyNotNull( Model model, InterMineObject obj, ClassDescriptor cld, PrimaryKey pk, Source source, IntToIntMap idMap) throws MetaDataException { for (String fieldName : pk.getFieldNames()) { FieldDescriptor fd = cld.getFieldDescriptorByName(fieldName); if (fd instanceof AttributeDescriptor) { Object value; try { value = obj.getFieldValue(fieldName); } catch (IllegalAccessException e) { throw new MetaDataException( "Failed to get field " + fieldName + " for key " + pk + " from " + obj, e); } if (value == null) { return false; } } else if (fd instanceof CollectionDescriptor) { throw new MetaDataException( "Primary key " + pk.getName() + " for class " + cld.getName() + " cannot contain collection " + fd.getName() + ": collections cannot be part of a primary key. Please edit" + model.getName() + "_keyDefs.properties"); } else if (fd instanceof ReferenceDescriptor) { InterMineObject refObj; try { refObj = (InterMineObject) obj.getFieldProxy(fieldName); } catch (IllegalAccessException e) { throw new MetaDataException( "Failed to get field " + fieldName + " for key " + pk + " from " + obj, e); } if (refObj == null) { return false; } if ((refObj.getId() != null) && (idMap.get(refObj.getId()) != null)) { // We have previously loaded the object in this reference. continue; } if (refObj instanceof ProxyReference) { refObj = ((ProxyReference) refObj).getObject(); } boolean foundNonNullKey = false; boolean foundKey = false; Set<ClassDescriptor> classDescriptors = model.getClassDescriptorsForClass(refObj.getClass()); CLDS: for (ClassDescriptor refCld : classDescriptors) { Set<PrimaryKey> primaryKeys; if (source == null) { primaryKeys = new LinkedHashSet<PrimaryKey>(PrimaryKeyUtil.getPrimaryKeys(refCld).values()); } else { primaryKeys = DataLoaderHelper.getPrimaryKeys(refCld, source, null); } for (PrimaryKey refPK : primaryKeys) { foundKey = true; if (objectPrimaryKeyNotNull(model, refObj, refCld, refPK, source, idMap)) { foundNonNullKey = true; break CLDS; } } } if (foundKey && (!foundNonNullKey)) { return false; } } } return true; }
/** * Fetches the equivalent object information for a whole batch of objects. * * @param batch the objects * @throws ObjectStoreException if something goes wrong */ protected void getEquivalentsFor(List<ResultsRow<Object>> batch) throws ObjectStoreException { long time = System.currentTimeMillis(); long time1 = time; boolean databaseEmpty = hints.databaseEmpty(); if (savedDatabaseEmptyFetch == -1) { savedDatabaseEmptyFetch = System.currentTimeMillis() - time; } if (databaseEmpty) { savedDatabaseEmpty++; return; } // TODO: add all the objects that are referenced by these objects, and follow primary keys // We can make use of the ObjectStoreFastCollectionsForTranslatorImpl's ability to work this // all out for us. Set<InterMineObject> objects = new HashSet<InterMineObject>(); for (ResultsRow<Object> row : batch) { for (Object object : row) { if (object instanceof InterMineObject) { InterMineObject imo = (InterMineObject) object; if (idMap.get(imo.getId()) == null) { objects.add(imo); for (String fieldName : TypeUtil.getFieldInfos(imo.getClass()).keySet()) { Object fieldValue; try { fieldValue = imo.getFieldProxy(fieldName); } catch (IllegalAccessException e) { throw new RuntimeException(e); } if ((fieldValue instanceof InterMineObject) && (!(fieldValue instanceof ProxyReference))) { objects.add((InterMineObject) fieldValue); } else if (fieldValue instanceof Collection<?>) { for (Object collectionElement : ((Collection<?>) fieldValue)) { if ((collectionElement instanceof InterMineObject) && (!(collectionElement instanceof ProxyReference))) { objects.add((InterMineObject) collectionElement); } } } } } } } } objects.removeAll(equivalents.keySet()); // Now objects contains all the objects we need to fetch data for. Map<InterMineObject, Set<InterMineObject>> results = new HashMap<InterMineObject, Set<InterMineObject>>(); for (InterMineObject object : objects) { results.put(object, Collections.synchronizedSet(new HashSet<InterMineObject>())); } Map<PrimaryKey, ClassDescriptor> pksToDo = new IdentityHashMap<PrimaryKey, ClassDescriptor>(); Map<ClassDescriptor, List<InterMineObject>> cldToObjectsForCld = new IdentityHashMap<ClassDescriptor, List<InterMineObject>>(); Map<Class<?>, List<InterMineObject>> categorised = CollectionUtil.groupByClass(objects, false); Map<ClassDescriptor, Boolean> cldsDone = new IdentityHashMap<ClassDescriptor, Boolean>(); for (Class<?> c : categorised.keySet()) { Set<ClassDescriptor> classDescriptors = model.getClassDescriptorsForClass(c); for (ClassDescriptor cld : classDescriptors) { if (!cldsDone.containsKey(cld)) { cldsDone.put(cld, Boolean.TRUE); Set<PrimaryKey> keysForClass; if (source == null) { keysForClass = new HashSet<PrimaryKey>(PrimaryKeyUtil.getPrimaryKeys(cld).values()); } else { keysForClass = DataLoaderHelper.getPrimaryKeys(cld, source, lookupOs); } if (!keysForClass.isEmpty()) { time = System.currentTimeMillis(); boolean classNotExists = hints.classNotExists(cld.getType()); String className = Util.getFriendlyName(cld.getType()); if (!savedTimes.containsKey(className)) { savedTimes.put(className, new Long(System.currentTimeMillis() - time)); } if (!classNotExists) { // LOG.error("Inspecting class " + className); List<InterMineObject> objectsForCld = new ArrayList<InterMineObject>(); for (Map.Entry<Class<?>, List<InterMineObject>> category : categorised.entrySet()) { if (cld.getType().isAssignableFrom(category.getKey())) { objectsForCld.addAll(category.getValue()); } } cldToObjectsForCld.put(cld, objectsForCld); // So now we have a list of objects for this CLD. for (PrimaryKey pk : keysForClass) { // LOG.error("Adding pk " + cld.getName() + "." + pk.getName()); pksToDo.put(pk, cld); } } else { // LOG.error("Empty class " + className); } } } } } doPks(pksToDo, results, cldToObjectsForCld, time1); batchQueried += results.size(); equivalents.putAll(results); }
/** * Return a Set of PrimaryKeys relevant to a given Source for a ClassDescriptor. The Set contains * all the primary keys that exist on a particular class that are used by the source, without * performing any recursion. The Model.getClassDescriptorsForClass() method is recommended if you * wish for all the primary keys of the class' parents as well. * * @param cld the ClassDescriptor * @param source the Source * @param os the ObjectStore that these PrimaryKeys are used in, for creating indexes * @return a Set of PrimaryKeys */ public static Set<PrimaryKey> getPrimaryKeys(ClassDescriptor cld, Source source, ObjectStore os) { GetPrimaryKeyCacheKey key = new GetPrimaryKeyCacheKey(cld, source); synchronized (getPrimaryKeyCache) { Set<PrimaryKey> keySet = getPrimaryKeyCache.get(key); if (keySet == null) { keySet = new LinkedHashSet<PrimaryKey>(); Properties keys = getKeyProperties(source); if (keys != null) { if (!verifiedSources.contains(source)) { String packageNameWithDot = cld.getName().substring(0, cld.getName().lastIndexOf('.') + 1); LOG.info( "Verifying primary key config for source " + source + ", packageName = " + packageNameWithDot); for (Map.Entry<Object, Object> entry : keys.entrySet()) { String cldName = (String) entry.getKey(); String keyList = (String) entry.getValue(); if (!cldName.contains(".")) { ClassDescriptor iCld = cld.getModel().getClassDescriptorByName(packageNameWithDot + cldName); if (iCld != null) { Map<String, PrimaryKey> map = PrimaryKeyUtil.getPrimaryKeys(iCld); String[] tokens = keyList.split(","); for (int i = 0; i < tokens.length; i++) { String token = tokens[i].trim(); if (map.get(token) == null) { throw new IllegalArgumentException( "Primary key " + token + " for class " + cldName + " required by datasource " + source.getName() + " in " + source.getName() + "_keys.properties is not defined in " + cld.getModel().getName() + "_keyDefs.properties"); } } } else { LOG.warn( "Ignoring entry for " + cldName + " in file " + cld.getModel().getName() + "_keyDefs.properties - not in model!"); } } } verifiedSources.add(source); } Map<String, PrimaryKey> map = PrimaryKeyUtil.getPrimaryKeys(cld); String cldName = TypeUtil.unqualifiedName(cld.getName()); String keyList = (String) keys.get(cldName); if (keyList != null) { String[] tokens = keyList.split(","); for (int i = 0; i < tokens.length; i++) { String token = tokens[i].trim(); if (map.get(token) == null) { throw new IllegalArgumentException( "Primary key " + token + " for class " + cld.getName() + " required by data source " + source.getName() + " in " + source.getName() + "_keys.properties is not defined in " + cld.getModel().getName() + "_keyDefs.properties"); } else { keySet.add(map.get(token)); } } } for (Map.Entry<Object, Object> entry : keys.entrySet()) { String propKey = (String) entry.getKey(); String fieldList = (String) entry.getValue(); int posOfDot = propKey.indexOf('.'); if (posOfDot > 0) { String propCldName = propKey.substring(0, posOfDot); if (cldName.equals(propCldName)) { String keyName = propKey.substring(posOfDot + 1); PrimaryKey pk = new PrimaryKey(keyName, fieldList, cld); if (!keySet.contains(pk)) { keySet.add(pk); if (os instanceof ObjectStoreInterMineImpl) { ObjectStoreInterMineImpl osimi = (ObjectStoreInterMineImpl) os; DatabaseSchema schema = osimi.getSchema(); ClassDescriptor tableMaster = schema.getTableMaster(cld); String tableName = DatabaseUtil.getTableName(tableMaster); List<String> fields = new ArrayList<String>(); for (String field : pk.getFieldNames()) { String colName = DatabaseUtil.generateSqlCompatibleName(field); if (tableMaster.getReferenceDescriptorByName(field, true) != null) { colName += "id"; } fields.add(colName); } String sql = "CREATE INDEX " + tableName + "__" + keyName + " ON " + tableName + " (" + StringUtil.join(fields, ", ") + ")"; System.out.println("Creating index: " + sql); LOG.info("Creating index: " + sql); Connection conn = null; try { conn = osimi.getConnection(); conn.createStatement().execute(sql); } catch (SQLException e) { LOG.warn("Index creation failed", e); } finally { if (conn != null) { osimi.releaseConnection(conn); } } } } } } } } else { throw new IllegalArgumentException( "Unable to find keys for source " + source.getName() + " in file " + source.getName() + "_keys.properties"); } getPrimaryKeyCache.put(key, keySet); } return keySet; } }