/**
   * Look a the values of the given primary key in the object and return true if and only if some
   * part of the primary key is null. If the primary key contains a reference it is sufficient for
   * any of the primary keys of the referenced object to be non-null (ie objectPrimaryKeyIsNull()
   * returning true).
   *
   * @param model the Model in which to find ClassDescriptors
   * @param obj the Object to check
   * @param cld one of the classes that obj is. Only primary keys for this classes will be checked
   * @param pk the primary key to check
   * @param source the Source database
   * @param idMap an IntToIntMap from source IDs to destination IDs
   * @return true if the the given primary key is non-null for the given object
   * @throws MetaDataException if anything goes wrong
   */
  public static boolean objectPrimaryKeyNotNull(
      Model model,
      InterMineObject obj,
      ClassDescriptor cld,
      PrimaryKey pk,
      Source source,
      IntToIntMap idMap)
      throws MetaDataException {
    for (String fieldName : pk.getFieldNames()) {
      FieldDescriptor fd = cld.getFieldDescriptorByName(fieldName);
      if (fd instanceof AttributeDescriptor) {
        Object value;
        try {
          value = obj.getFieldValue(fieldName);
        } catch (IllegalAccessException e) {
          throw new MetaDataException(
              "Failed to get field " + fieldName + " for key " + pk + " from " + obj, e);
        }
        if (value == null) {
          return false;
        }
      } else if (fd instanceof CollectionDescriptor) {
        throw new MetaDataException(
            "Primary key "
                + pk.getName()
                + " for class "
                + cld.getName()
                + " cannot contain collection "
                + fd.getName()
                + ": collections cannot be part of a primary key. Please edit"
                + model.getName()
                + "_keyDefs.properties");
      } else if (fd instanceof ReferenceDescriptor) {
        InterMineObject refObj;
        try {
          refObj = (InterMineObject) obj.getFieldProxy(fieldName);
        } catch (IllegalAccessException e) {
          throw new MetaDataException(
              "Failed to get field " + fieldName + " for key " + pk + " from " + obj, e);
        }
        if (refObj == null) {
          return false;
        }

        if ((refObj.getId() != null) && (idMap.get(refObj.getId()) != null)) {
          // We have previously loaded the object in this reference.
          continue;
        }

        if (refObj instanceof ProxyReference) {
          refObj = ((ProxyReference) refObj).getObject();
        }

        boolean foundNonNullKey = false;
        boolean foundKey = false;
        Set<ClassDescriptor> classDescriptors =
            model.getClassDescriptorsForClass(refObj.getClass());

        CLDS:
        for (ClassDescriptor refCld : classDescriptors) {
          Set<PrimaryKey> primaryKeys;

          if (source == null) {
            primaryKeys =
                new LinkedHashSet<PrimaryKey>(PrimaryKeyUtil.getPrimaryKeys(refCld).values());
          } else {
            primaryKeys = DataLoaderHelper.getPrimaryKeys(refCld, source, null);
          }

          for (PrimaryKey refPK : primaryKeys) {
            foundKey = true;

            if (objectPrimaryKeyNotNull(model, refObj, refCld, refPK, source, idMap)) {
              foundNonNullKey = true;
              break CLDS;
            }
          }
        }

        if (foundKey && (!foundNonNullKey)) {
          return false;
        }
      }
    }

    return true;
  }
  /**
   * Fetches the equivalent object information for a whole batch of objects.
   *
   * @param batch the objects
   * @throws ObjectStoreException if something goes wrong
   */
  protected void getEquivalentsFor(List<ResultsRow<Object>> batch) throws ObjectStoreException {
    long time = System.currentTimeMillis();
    long time1 = time;
    boolean databaseEmpty = hints.databaseEmpty();
    if (savedDatabaseEmptyFetch == -1) {
      savedDatabaseEmptyFetch = System.currentTimeMillis() - time;
    }
    if (databaseEmpty) {
      savedDatabaseEmpty++;
      return;
    }
    // TODO: add all the objects that are referenced by these objects, and follow primary keys
    // We can make use of the ObjectStoreFastCollectionsForTranslatorImpl's ability to work this
    // all out for us.
    Set<InterMineObject> objects = new HashSet<InterMineObject>();
    for (ResultsRow<Object> row : batch) {
      for (Object object : row) {
        if (object instanceof InterMineObject) {
          InterMineObject imo = (InterMineObject) object;
          if (idMap.get(imo.getId()) == null) {
            objects.add(imo);
            for (String fieldName : TypeUtil.getFieldInfos(imo.getClass()).keySet()) {
              Object fieldValue;
              try {
                fieldValue = imo.getFieldProxy(fieldName);
              } catch (IllegalAccessException e) {
                throw new RuntimeException(e);
              }
              if ((fieldValue instanceof InterMineObject)
                  && (!(fieldValue instanceof ProxyReference))) {
                objects.add((InterMineObject) fieldValue);
              } else if (fieldValue instanceof Collection<?>) {
                for (Object collectionElement : ((Collection<?>) fieldValue)) {
                  if ((collectionElement instanceof InterMineObject)
                      && (!(collectionElement instanceof ProxyReference))) {
                    objects.add((InterMineObject) collectionElement);
                  }
                }
              }
            }
          }
        }
      }
    }
    objects.removeAll(equivalents.keySet());
    // Now objects contains all the objects we need to fetch data for.
    Map<InterMineObject, Set<InterMineObject>> results =
        new HashMap<InterMineObject, Set<InterMineObject>>();
    for (InterMineObject object : objects) {
      results.put(object, Collections.synchronizedSet(new HashSet<InterMineObject>()));
    }

    Map<PrimaryKey, ClassDescriptor> pksToDo = new IdentityHashMap<PrimaryKey, ClassDescriptor>();
    Map<ClassDescriptor, List<InterMineObject>> cldToObjectsForCld =
        new IdentityHashMap<ClassDescriptor, List<InterMineObject>>();
    Map<Class<?>, List<InterMineObject>> categorised = CollectionUtil.groupByClass(objects, false);
    Map<ClassDescriptor, Boolean> cldsDone = new IdentityHashMap<ClassDescriptor, Boolean>();
    for (Class<?> c : categorised.keySet()) {
      Set<ClassDescriptor> classDescriptors = model.getClassDescriptorsForClass(c);
      for (ClassDescriptor cld : classDescriptors) {
        if (!cldsDone.containsKey(cld)) {
          cldsDone.put(cld, Boolean.TRUE);
          Set<PrimaryKey> keysForClass;
          if (source == null) {
            keysForClass = new HashSet<PrimaryKey>(PrimaryKeyUtil.getPrimaryKeys(cld).values());
          } else {
            keysForClass = DataLoaderHelper.getPrimaryKeys(cld, source, lookupOs);
          }
          if (!keysForClass.isEmpty()) {
            time = System.currentTimeMillis();
            boolean classNotExists = hints.classNotExists(cld.getType());
            String className = Util.getFriendlyName(cld.getType());
            if (!savedTimes.containsKey(className)) {
              savedTimes.put(className, new Long(System.currentTimeMillis() - time));
            }
            if (!classNotExists) {
              // LOG.error("Inspecting class " + className);
              List<InterMineObject> objectsForCld = new ArrayList<InterMineObject>();
              for (Map.Entry<Class<?>, List<InterMineObject>> category : categorised.entrySet()) {
                if (cld.getType().isAssignableFrom(category.getKey())) {
                  objectsForCld.addAll(category.getValue());
                }
              }
              cldToObjectsForCld.put(cld, objectsForCld);
              // So now we have a list of objects for this CLD.
              for (PrimaryKey pk : keysForClass) {
                // LOG.error("Adding pk " + cld.getName() + "." + pk.getName());
                pksToDo.put(pk, cld);
              }
            } else {
              // LOG.error("Empty class " + className);
            }
          }
        }
      }
    }
    doPks(pksToDo, results, cldToObjectsForCld, time1);
    batchQueried += results.size();
    equivalents.putAll(results);
  }
  /**
   * Return a Set of PrimaryKeys relevant to a given Source for a ClassDescriptor. The Set contains
   * all the primary keys that exist on a particular class that are used by the source, without
   * performing any recursion. The Model.getClassDescriptorsForClass() method is recommended if you
   * wish for all the primary keys of the class' parents as well.
   *
   * @param cld the ClassDescriptor
   * @param source the Source
   * @param os the ObjectStore that these PrimaryKeys are used in, for creating indexes
   * @return a Set of PrimaryKeys
   */
  public static Set<PrimaryKey> getPrimaryKeys(ClassDescriptor cld, Source source, ObjectStore os) {
    GetPrimaryKeyCacheKey key = new GetPrimaryKeyCacheKey(cld, source);
    synchronized (getPrimaryKeyCache) {
      Set<PrimaryKey> keySet = getPrimaryKeyCache.get(key);
      if (keySet == null) {
        keySet = new LinkedHashSet<PrimaryKey>();
        Properties keys = getKeyProperties(source);
        if (keys != null) {
          if (!verifiedSources.contains(source)) {
            String packageNameWithDot =
                cld.getName().substring(0, cld.getName().lastIndexOf('.') + 1);
            LOG.info(
                "Verifying primary key config for source "
                    + source
                    + ", packageName = "
                    + packageNameWithDot);
            for (Map.Entry<Object, Object> entry : keys.entrySet()) {
              String cldName = (String) entry.getKey();
              String keyList = (String) entry.getValue();
              if (!cldName.contains(".")) {
                ClassDescriptor iCld =
                    cld.getModel().getClassDescriptorByName(packageNameWithDot + cldName);
                if (iCld != null) {
                  Map<String, PrimaryKey> map = PrimaryKeyUtil.getPrimaryKeys(iCld);

                  String[] tokens = keyList.split(",");
                  for (int i = 0; i < tokens.length; i++) {
                    String token = tokens[i].trim();
                    if (map.get(token) == null) {
                      throw new IllegalArgumentException(
                          "Primary key "
                              + token
                              + " for class "
                              + cldName
                              + " required by datasource "
                              + source.getName()
                              + " in "
                              + source.getName()
                              + "_keys.properties is not defined in "
                              + cld.getModel().getName()
                              + "_keyDefs.properties");
                    }
                  }
                } else {
                  LOG.warn(
                      "Ignoring entry for "
                          + cldName
                          + " in file "
                          + cld.getModel().getName()
                          + "_keyDefs.properties - not in model!");
                }
              }
            }
            verifiedSources.add(source);
          }
          Map<String, PrimaryKey> map = PrimaryKeyUtil.getPrimaryKeys(cld);
          String cldName = TypeUtil.unqualifiedName(cld.getName());
          String keyList = (String) keys.get(cldName);
          if (keyList != null) {
            String[] tokens = keyList.split(",");
            for (int i = 0; i < tokens.length; i++) {
              String token = tokens[i].trim();
              if (map.get(token) == null) {
                throw new IllegalArgumentException(
                    "Primary key "
                        + token
                        + " for class "
                        + cld.getName()
                        + " required by data source "
                        + source.getName()
                        + " in "
                        + source.getName()
                        + "_keys.properties is not defined in "
                        + cld.getModel().getName()
                        + "_keyDefs.properties");
              } else {
                keySet.add(map.get(token));
              }
            }
          }
          for (Map.Entry<Object, Object> entry : keys.entrySet()) {
            String propKey = (String) entry.getKey();
            String fieldList = (String) entry.getValue();
            int posOfDot = propKey.indexOf('.');
            if (posOfDot > 0) {
              String propCldName = propKey.substring(0, posOfDot);
              if (cldName.equals(propCldName)) {
                String keyName = propKey.substring(posOfDot + 1);
                PrimaryKey pk = new PrimaryKey(keyName, fieldList, cld);
                if (!keySet.contains(pk)) {
                  keySet.add(pk);
                  if (os instanceof ObjectStoreInterMineImpl) {
                    ObjectStoreInterMineImpl osimi = (ObjectStoreInterMineImpl) os;
                    DatabaseSchema schema = osimi.getSchema();
                    ClassDescriptor tableMaster = schema.getTableMaster(cld);
                    String tableName = DatabaseUtil.getTableName(tableMaster);
                    List<String> fields = new ArrayList<String>();

                    for (String field : pk.getFieldNames()) {
                      String colName = DatabaseUtil.generateSqlCompatibleName(field);
                      if (tableMaster.getReferenceDescriptorByName(field, true) != null) {
                        colName += "id";
                      }
                      fields.add(colName);
                    }
                    String sql =
                        "CREATE INDEX "
                            + tableName
                            + "__"
                            + keyName
                            + " ON "
                            + tableName
                            + " ("
                            + StringUtil.join(fields, ", ")
                            + ")";
                    System.out.println("Creating index: " + sql);
                    LOG.info("Creating index: " + sql);
                    Connection conn = null;
                    try {
                      conn = osimi.getConnection();
                      conn.createStatement().execute(sql);
                    } catch (SQLException e) {
                      LOG.warn("Index creation failed", e);
                    } finally {
                      if (conn != null) {
                        osimi.releaseConnection(conn);
                      }
                    }
                  }
                }
              }
            }
          }
        } else {
          throw new IllegalArgumentException(
              "Unable to find keys for source "
                  + source.getName()
                  + " in file "
                  + source.getName()
                  + "_keys.properties");
        }
        getPrimaryKeyCache.put(key, keySet);
      }
      return keySet;
    }
  }