Example #1
0
 /**
  * Fetches equivalent objects for a particular primary key.
  *
  * @param pk the PrimaryKey
  * @param cld the ClassDescriptor of the PrimaryKey
  * @param results a Map to hold results that are to be added to the cache
  * @param objectsForCld a List of objects relevant to this PrimaryKey
  * @param fetchedObjectIds a Set to hold ids of objects that are fetched, to prefetch from the
  *     data tracker later
  * @throws ObjectStoreException if something goes wrong
  */
 protected void doPk(
     PrimaryKey pk,
     ClassDescriptor cld,
     Map<InterMineObject, Set<InterMineObject>> results,
     List<InterMineObject> objectsForCld,
     Set<Integer> fetchedObjectIds)
     throws ObjectStoreException {
   Iterator<InterMineObject> objectsForCldIter = objectsForCld.iterator();
   while (objectsForCldIter.hasNext()) {
     int objCount = 0;
     int origObjCount = 0;
     Query q = new Query();
     QueryClass qc = new QueryClass(cld.getType());
     q.addFrom(qc);
     q.addToSelect(qc);
     ConstraintSet cs = new ConstraintSet(ConstraintOp.AND);
     q.setConstraint(cs);
     Map<String, Set<Object>> fieldNameToValues = new HashMap<String, Set<Object>>();
     for (String fieldName : pk.getFieldNames()) {
       try {
         QueryField qf = new QueryField(qc, fieldName);
         q.addToSelect(qf);
         Set<Object> values = new HashSet<Object>();
         fieldNameToValues.put(fieldName, values);
         cs.addConstraint(new BagConstraint(qf, ConstraintOp.IN, values));
       } catch (IllegalArgumentException e) {
         QueryForeignKey qf = new QueryForeignKey(qc, fieldName);
         q.addToSelect(qf);
         Set<Object> values = new HashSet<Object>();
         fieldNameToValues.put(fieldName, values);
         cs.addConstraint(new BagConstraint(qf, ConstraintOp.IN, values));
       }
     }
     // Now make a map from the primary key values to source objects
     Map<List<Object>, InterMineObject> keysToSourceObjects =
         new HashMap<List<Object>, InterMineObject>();
     while (objectsForCldIter.hasNext() && (objCount < 500)) {
       InterMineObject object = objectsForCldIter.next();
       origObjCount++;
       try {
         if (DataLoaderHelper.objectPrimaryKeyNotNull(model, object, cld, pk, source, idMap)) {
           List<Collection<Object>> values = new ArrayList<Collection<Object>>();
           boolean skipObject = false;
           Map<String, Set<Object>> fieldsValues = new HashMap<String, Set<Object>>();
           for (String fieldName : pk.getFieldNames()) {
             try {
               Object value = object.getFieldProxy(fieldName);
               Set<Object> fieldValues;
               if (value instanceof InterMineObject) {
                 Integer id = idMap.get(((InterMineObject) value).getId());
                 if (id == null) {
                   Set<InterMineObject> eqs = results.get(value);
                   if (eqs == null) {
                     value = object.getFieldValue(fieldName);
                     eqs = queryEquivalentObjects((InterMineObject) value, source);
                   }
                   fieldValues = new HashSet<Object>();
                   for (InterMineObject obj : eqs) {
                     fieldValues.add(obj.getId());
                   }
                 } else {
                   fieldValues = Collections.singleton((Object) id);
                 }
               } else {
                 fieldValues = Collections.singleton(value);
               }
               values.add(fieldValues);
               fieldsValues.put(fieldName, fieldValues);
               for (Object fieldValue : fieldValues) {
                 long time = System.currentTimeMillis();
                 boolean pkQueryFruitless =
                     hints.pkQueryFruitless(cld.getType(), fieldName, fieldValue);
                 String summaryName = Util.getFriendlyName(cld.getType()) + "." + fieldName;
                 if (!savedTimes.containsKey(summaryName)) {
                   savedTimes.put(summaryName, new Long(System.currentTimeMillis() - time));
                   savedCounts.put(summaryName, new Integer(0));
                 }
                 if (pkQueryFruitless) {
                   skipObject = true;
                 }
               }
             } catch (IllegalAccessException e) {
               throw new RuntimeException(e);
             }
           }
           if (!skipObject) {
             objCount++;
             for (String fieldName : pk.getFieldNames()) {
               fieldNameToValues.get(fieldName).addAll(fieldsValues.get(fieldName));
             }
             for (List<Object> valueSet : CollectionUtil.fanOutCombinations(values)) {
               if (keysToSourceObjects.containsKey(valueSet)) {
                 throw new ObjectStoreException(
                     "Duplicate objects found for pk "
                         + cld.getName()
                         + "."
                         + pk.getName()
                         + ": "
                         + object);
               }
               keysToSourceObjects.put(valueSet, object);
             }
           }
         }
       } catch (MetaDataException e) {
         throw new ObjectStoreException(e);
       }
     }
     // Prune BagConstraints using the hints system.
     // boolean emptyQuery = false;
     // Iterator<String> fieldNameIter = pk.getFieldNames().iterator();
     // while (fieldNameIter.hasNext() && (!emptyQuery)) {
     //    String fieldName = fieldNameIter.next();
     //    Set values = fieldNameToValues.get(fieldName);
     // Iterator valueIter = values.iterator();
     // while (valueIter.hasNext()) {
     //    if (hints.pkQueryFruitless(cld.getType(), fieldName, valueIter.next())) {
     //        valueIter.remove();
     //    }
     // }
     //    if (values.isEmpty()) {
     //        emptyQuery = true;
     //    }
     // }
     if (objCount > 0) {
       // Iterate through query, and add objects to results
       // long time = System.currentTimeMillis();
       int matches = 0;
       Results res = lookupOs.execute(q, 2000, false, false, false);
       @SuppressWarnings("unchecked")
       List<ResultsRow<Object>> tmpRes = (List) res;
       for (ResultsRow<Object> row : tmpRes) {
         List<Object> values = new ArrayList<Object>();
         for (int i = 1; i <= pk.getFieldNames().size(); i++) {
           values.add(row.get(i));
         }
         Set<InterMineObject> set = results.get(keysToSourceObjects.get(values));
         if (set != null) {
           set.add((InterMineObject) row.get(0));
           matches++;
         }
         fetchedObjectIds.add(((InterMineObject) row.get(0)).getId());
       }
       // LOG.info("Fetched " + res.size() + " equivalent objects for " + objCount
       //        + " objects in " + (System.currentTimeMillis() - time) + " ms for "
       //        + cld.getName() + "." + pk.getName());
     }
   }
 }
Example #2
0
  /**
   * Fetches the equivalent object information for a whole batch of objects.
   *
   * @param batch the objects
   * @throws ObjectStoreException if something goes wrong
   */
  protected void getEquivalentsFor(List<ResultsRow<Object>> batch) throws ObjectStoreException {
    long time = System.currentTimeMillis();
    long time1 = time;
    boolean databaseEmpty = hints.databaseEmpty();
    if (savedDatabaseEmptyFetch == -1) {
      savedDatabaseEmptyFetch = System.currentTimeMillis() - time;
    }
    if (databaseEmpty) {
      savedDatabaseEmpty++;
      return;
    }
    // TODO: add all the objects that are referenced by these objects, and follow primary keys
    // We can make use of the ObjectStoreFastCollectionsForTranslatorImpl's ability to work this
    // all out for us.
    Set<InterMineObject> objects = new HashSet<InterMineObject>();
    for (ResultsRow<Object> row : batch) {
      for (Object object : row) {
        if (object instanceof InterMineObject) {
          InterMineObject imo = (InterMineObject) object;
          if (idMap.get(imo.getId()) == null) {
            objects.add(imo);
            for (String fieldName : TypeUtil.getFieldInfos(imo.getClass()).keySet()) {
              Object fieldValue;
              try {
                fieldValue = imo.getFieldProxy(fieldName);
              } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
              }
              if ((fieldValue instanceof InterMineObject)
                  && (!(fieldValue instanceof ProxyReference))) {
                objects.add((InterMineObject) fieldValue);
              } else if (fieldValue instanceof Collection<?>) {
                for (Object collectionElement : ((Collection<?>) fieldValue)) {
                  if ((collectionElement instanceof InterMineObject)
                      && (!(collectionElement instanceof ProxyReference))) {
                    objects.add((InterMineObject) collectionElement);
                  }
                }
              }
            }
          }
        }
      }
    }
    objects.removeAll(equivalents.keySet());
    // Now objects contains all the objects we need to fetch data for.
    Map<InterMineObject, Set<InterMineObject>> results =
        new HashMap<InterMineObject, Set<InterMineObject>>();
    for (InterMineObject object : objects) {
      results.put(object, Collections.synchronizedSet(new HashSet<InterMineObject>()));
    }

    Map<PrimaryKey, ClassDescriptor> pksToDo = new IdentityHashMap<PrimaryKey, ClassDescriptor>();
    Map<ClassDescriptor, List<InterMineObject>> cldToObjectsForCld =
        new IdentityHashMap<ClassDescriptor, List<InterMineObject>>();
    Map<Class<?>, List<InterMineObject>> categorised = CollectionUtil.groupByClass(objects, false);
    Map<ClassDescriptor, Boolean> cldsDone = new IdentityHashMap<ClassDescriptor, Boolean>();
    for (Class<?> c : categorised.keySet()) {
      Set<ClassDescriptor> classDescriptors = model.getClassDescriptorsForClass(c);
      for (ClassDescriptor cld : classDescriptors) {
        if (!cldsDone.containsKey(cld)) {
          cldsDone.put(cld, Boolean.TRUE);
          Set<PrimaryKey> keysForClass;
          if (source == null) {
            keysForClass = new HashSet<PrimaryKey>(PrimaryKeyUtil.getPrimaryKeys(cld).values());
          } else {
            keysForClass = DataLoaderHelper.getPrimaryKeys(cld, source, lookupOs);
          }
          if (!keysForClass.isEmpty()) {
            time = System.currentTimeMillis();
            boolean classNotExists = hints.classNotExists(cld.getType());
            String className = Util.getFriendlyName(cld.getType());
            if (!savedTimes.containsKey(className)) {
              savedTimes.put(className, new Long(System.currentTimeMillis() - time));
            }
            if (!classNotExists) {
              // LOG.error("Inspecting class " + className);
              List<InterMineObject> objectsForCld = new ArrayList<InterMineObject>();
              for (Map.Entry<Class<?>, List<InterMineObject>> category : categorised.entrySet()) {
                if (cld.getType().isAssignableFrom(category.getKey())) {
                  objectsForCld.addAll(category.getValue());
                }
              }
              cldToObjectsForCld.put(cld, objectsForCld);
              // So now we have a list of objects for this CLD.
              for (PrimaryKey pk : keysForClass) {
                // LOG.error("Adding pk " + cld.getName() + "." + pk.getName());
                pksToDo.put(pk, cld);
              }
            } else {
              // LOG.error("Empty class " + className);
            }
          }
        }
      }
    }
    doPks(pksToDo, results, cldToObjectsForCld, time1);
    batchQueried += results.size();
    equivalents.putAll(results);
  }