@Override
  public EntitiesValidationReport validateImport(RepositoryCollection source) {
    EntitiesValidationReportImpl report = new EntitiesValidationReportImpl();

    // compare the data sheets against metadata in store or imported file
    Map<String, DefaultEntityMetaData> metaDataMap = getEntityMetaData(source);

    for (String sheet : source.getEntityNames())
      if (!"entities".equals(sheet) && !"attributes".equals(sheet)) {
        // check if sheet is known?
        if (metaDataMap.containsKey(sheet)) report.getSheetsImportable().put(sheet, true);
        else report.getSheetsImportable().put(sheet, false);

        // check the fields
        Repository s = source.getRepositoryByEntityName(sheet);
        EntityMetaData target = metaDataMap.get(sheet);

        if (target != null) {
          List<String> fieldsAvailable = new ArrayList<String>();
          List<String> fieldsImportable = new ArrayList<String>();
          List<String> fieldsRequired = new ArrayList<String>();
          List<String> fieldsUnknown = new ArrayList<String>();

          for (AttributeMetaData att : s.getEntityMetaData().getAttributes()) {
            if (target.getAttribute(att.getName()) == null) fieldsUnknown.add(att.getName());
            else fieldsImportable.add(att.getName());
          }
          for (AttributeMetaData att : target.getAttributes()) {
            if (!fieldsImportable.contains(att.getName())) {
              if (!att.isNillable()) fieldsRequired.add(att.getName());
              else fieldsAvailable.add(att.getName());
            }
          }

          report.getFieldsAvailable().put(sheet, fieldsAvailable);
          report.getFieldsRequired().put(sheet, fieldsRequired);
          report.getFieldsUnknown().put(sheet, fieldsUnknown);
          report.getFieldsImportable().put(sheet, fieldsImportable);
        }
      }
    return report;
  }
  public Collection<EntityMetaData> loadEMD(RepositoryCollection emdFormattedRepos) {
    // extract entity metadata
    Map<String, DefaultEntityMetaData> entities =
        new LinkedHashMap<String, DefaultEntityMetaData>();
    for (Entity e : emdFormattedRepos.getRepositoryByEntityName("entities")) {
      DefaultEntityMetaData emd = new DefaultEntityMetaData(e.getString("name"));
      if (e.getBoolean("abstract")) emd.setAbstract(true);
    }

    // extract extends relations
    for (Entity e : emdFormattedRepos.getRepositoryByEntityName("entities")) {
      if (e.get("extends") != null) {
        DefaultEntityMetaData emd = entities.get(e.get("name"));
        emd.setExtends(entities.get(e.get("extends")));
      }
    }

    Collection<EntityMetaData> result = new ArrayList<EntityMetaData>();
    result.addAll(entities.values());
    return result;
  }
  @Test
  public void integrationTestMetaData() {
    DataServiceImpl dataServiceImpl = Mockito.mock(DataServiceImpl.class);
    when(dataServiceImpl.hasRepository("attributes")).thenReturn(Boolean.FALSE); // To skip the
    // canIntegrateEntityMetadataCheck
    // test
    MetaDataServiceImpl metaDataService = new MetaDataServiceImpl(dataServiceImpl);
    RepositoryCollection repositoryCollection = Mockito.mock(RepositoryCollection.class);

    when(repositoryCollection.getEntityNames()).thenReturn(Lists.newArrayList("attributes"));

    DefaultEntityMetaData newEntityMetaData = new DefaultEntityMetaData("attributes");
    newEntityMetaData.addAttribute("ID");
    Repository repo1 = Mockito.mock(Repository.class);
    when(repositoryCollection.getRepository("attributes")).thenReturn(repo1);
    when(repo1.getEntityMetaData()).thenReturn(newEntityMetaData);

    LinkedHashMap<String, Boolean> entitiesImportable = new LinkedHashMap<String, Boolean>();
    entitiesImportable.put("attributes", true);

    assertEquals(entitiesImportable, metaDataService.integrationTestMetaData(repositoryCollection));
  }
  @Override
  @Transactional(rollbackFor = IOException.class)
  public EntityImportReport doImport(RepositoryCollection source, DatabaseAction databaseAction)
      throws IOException {
    if (store == null) throw new RuntimeException("store was not set");

    EntityImportReport report = new EntityImportReport();

    // TODO: need to change order

    Map<String, DefaultEntityMetaData> metadata = getEntityMetaData(source);

    for (Entry<String, DefaultEntityMetaData> entry : metadata.entrySet()) {
      String name = entry.getKey();
      if (!"entities".equals(name) && !"attributes".equals(name)) {
        Repository from = source.getRepositoryByEntityName(name);

        // TODO check if compatible with metadata

        // create repo if needed
        MysqlRepository to = (MysqlRepository) store.getRepositoryByEntityName(name);

        if (to == null) {
          logger.debug("tyring to create: " + name);

          EntityMetaData em = metadata.get(name);
          if (em == null) throw new IllegalArgumentException("Unknown entity: " + name);
          store.add(em);

          to = (MysqlRepository) store.getRepositoryByEntityName(name);
        }

        // import

        report.getNrImportedEntitiesMap().put(name, to.add(from));
      }
    }

    return report;
  }
  public Collection<EntityMetaData> loadOMX(RepositoryCollection omx) {
    // extract attribute metadata
    Map<String, AttributeMetaData> attributes = new LinkedHashMap<String, AttributeMetaData>();
    for (Entity e : omx.getRepositoryByEntityName("observablefeature")) {
      logger.debug("found observablefeature: " + e);

      DefaultAttributeMetaData att = new DefaultAttributeMetaData(e.getString("name"));
      if (e.get("dataType") != null)
        att.setDataType(MolgenisFieldTypes.getType(e.getString("dataType")));
      attributes.put(e.getString("identifier"), att);
      if (e.get("description") != null) att.setDescription(e.getString("description"));
      // TODO unit! if(e.get("unit") != null)

      if ("xref".equals(e.get("dataType")) || "mref".equals(e.get("dataType"))) {
        // TODO: cannot solve!!!
        att.setRefEntity(omxEntities.get("Characteristic"));
      }
      if ("categorical".equals(e.get("dataType"))) {
        att.setRefEntity(omxEntities.get("Category"));
      }
    }
    // TODO: fix categorical!

    // extract protocols as entities(abstract=true)
    Map<String, EntityMetaData> entities = new LinkedHashMap<String, EntityMetaData>();
    for (Entity e : omx.getRepositoryByEntityName("protocol")) {
      // skip all null entities
      if (hasValues(e)) {
        logger.debug("found protocol: " + e);

        DefaultEntityMetaData ent = new DefaultEntityMetaData(e.getString("identifier")); // alas
        ent.setLabel(e.getString("name"));
        ent.setAbstract(true);

        // add attributes
        if (e.get("features_identifier") != null)
          for (String attIdentifier : e.getList("features_identifier")) {
            if (attributes.get(attIdentifier) == null)
              throw new RuntimeException("attribute '" + attIdentifier + "' unknown");
            ent.addAttributeMetaData(attributes.get(attIdentifier));
          }

        entities.put(e.getString("identifier"), ent);
      }
    }

    for (Entity e : omx.getRepositoryByEntityName("protocol")) {
      // add subprotocols as compound
      if (e.get("subprotocols_identifier") != null) {
        for (String subProtocol : e.getList("subprotocols_identifier")) {
          DefaultAttributeMetaData att = new DefaultAttributeMetaData(subProtocol);
          att.setDataType(MolgenisFieldTypes.COMPOUND);
          att.setRefEntity(entities.get(subProtocol));
          ((DefaultEntityMetaData) entities.get(e.get("identifier"))).addAttributeMetaData(att);
        }
      }
    }

    // create dataset as entities
    for (Entity e : omx.getRepositoryByEntityName("dataset")) {
      logger.debug("found dataset: " + e);

      DefaultEntityMetaData ent = new DefaultEntityMetaData(e.getString("identifier"));
      ent.setLabel(e.getString("name"));
      // dataset 'extends' protocol
      ent.setExtends(entities.get(e.getString("protocolused_identifier")));
      entities.put(e.getString("identifier"), ent);
    }

    return entities.values();
  }
  @Override
  @Transactional(rollbackFor = IOException.class)
  public EntityImportReport doImport(
      RepositoryCollection repositories, DatabaseAction databaseAction) throws IOException {
    // All new repository identifiers
    List<String> newRepoIdentifiers = new ArrayList<String>();

    // First import entities, the data sheets are ignored in the entitiesimporter
    EntityImportReport importReport = entitiesImporter.importEntities(repositories, databaseAction);

    // RULE: Feature can only belong to one Protocol in a DataSet. Check it (see issue #1136)
    checkFeatureCanOnlyBelongToOneProtocolForOneDataSet();

    // Import data sheets
    for (String name : repositories.getEntityNames()) {
      Repository repository = repositories.getRepositoryByEntityName(name);

      if (repository.getName().startsWith(DATASET_SHEET_PREFIX)) {
        // Import DataSet sheet, create new OmxRepository
        String identifier = repository.getName().substring(DATASET_SHEET_PREFIX.length());

        if (!dataService.hasRepository(identifier)) {

          dataService.addRepository(
              new AggregateableCrudRepositorySecurityDecorator(
                  new OmxRepository(dataService, searchService, identifier, entityValidator)));
          newRepoIdentifiers.add(identifier);

          DataSet dataSet =
              dataService.findOne(
                  DataSet.ENTITY_NAME,
                  new QueryImpl().eq(DataSet.IDENTIFIER, identifier),
                  DataSet.class);

          List<Protocol> protocols =
              ProtocolUtils.getProtocolDescendants(dataSet.getProtocolUsed());
          List<ObservableFeature> categoricalFeatures = new ArrayList<ObservableFeature>();
          for (Protocol protocol : protocols) {
            List<ObservableFeature> observableFeatures = protocol.getFeatures();
            if (observableFeatures != null) {
              for (ObservableFeature observableFeature : observableFeatures) {
                String dataType = observableFeature.getDataType();
                FieldType type = MolgenisFieldTypes.getType(dataType);
                if (type.getEnumType() == FieldTypeEnum.CATEGORICAL) {
                  categoricalFeatures.add(observableFeature);
                }
              }
            }
          }
          for (ObservableFeature categoricalFeature : categoricalFeatures) {
            if (!dataService.hasRepository(
                OmxLookupTableEntityMetaData.createOmxLookupTableEntityMetaDataName(
                    categoricalFeature.getIdentifier()))) {
              dataService.addRepository(
                  new OmxLookupTableRepository(
                      dataService, categoricalFeature.getIdentifier(), queryResolver));
              newRepoIdentifiers.add(
                  OmxLookupTableEntityMetaData.createOmxLookupTableEntityMetaDataName(
                      categoricalFeature.getIdentifier()));
            }
          }
        }

        // Check if all column names in the excel sheet exist as attributes of the entity
        Set<ConstraintViolation> violations = Sets.newLinkedHashSet();
        EntityMetaData meta = dataService.getEntityMetaData(identifier);
        for (AttributeMetaData attr : repository.getEntityMetaData().getAttributes()) {
          if (meta.getAttribute(attr.getName()) == null) {
            String message =
                String.format(
                    "Unknown attributename '%s' for entity '%s'. Sheet: '%s'",
                    attr.getName(), meta.getName(), repository.getName());
            violations.add(new ConstraintViolation(message, attr.getName(), null, null, meta, 0));
          }
        }

        if (!violations.isEmpty()) {
          throw new MolgenisValidationException(violations);
        }

        // Import data into new OmxRepository
        try {
          dataService.add(identifier, repository);
        } catch (MolgenisValidationException e) {
          // Add sheet info
          for (ConstraintViolation violation : e.getViolations()) {
            if (violation.getRownr() > 0) {

              // Rownr +1 for header
              violation.setImportInfo(
                  String.format(
                      "Sheet: '%s', row: %d", repository.getName(), violation.getRownr() + 1));
            } else {
              violation.setImportInfo(String.format("Sheet: '%s'", repository.getName()));
            }
          }

          for (String newRepoIdentifier : newRepoIdentifiers) {
            dataService.removeRepository(newRepoIdentifier);
          }

          throw e;
        }

        int count = (int) RepositoryUtils.count(repository);
        importReport.addEntityCount(identifier, count);
        importReport.addNrImported(count);
      }
    }

    return importReport;
  }
  @Override
  public Map<String, DefaultEntityMetaData> getEntityMetaData(RepositoryCollection source) {
    // TODO: this task is actually a 'merge' instead of 'import'
    // so we need to consider both new metadata as existing ...

    Map<String, DefaultEntityMetaData> entities =
        new LinkedHashMap<String, DefaultEntityMetaData>();

    // load attributes first (because entities are optional).
    for (Entity a : source.getRepositoryByEntityName("attributes")) {
      int i = 1;
      String entityName = a.getString("entity");

      // required
      if (entityName == null) throw new IllegalArgumentException("attributes.entity is missing");
      if (a.get("name") == null) throw new IllegalArgumentException("attributes.name is missing");

      // create entity if not yet defined
      if (entities.get(entityName) == null)
        entities.put(entityName, new DefaultEntityMetaData(entityName));
      DefaultEntityMetaData md = entities.get(entityName);

      DefaultAttributeMetaData am = new DefaultAttributeMetaData(a.getString("name"));

      if (a.get("dataType") != null) {
        FieldType t = MolgenisFieldTypes.getType(a.getString("dataType"));
        if (t == null)
          throw new IllegalArgumentException(
              "attributes.type error on line " + i + ": " + a.getString("dataType") + " unknown");
        am.setDataType(t);
      }
      if (a.get("nillable") != null) am.setNillable(a.getBoolean("nillable"));
      if (a.get("auto") != null) am.setAuto(a.getBoolean("auto"));
      if (a.get("idAttribute") != null) am.setIdAttribute(a.getBoolean("idAttribute"));

      md.addAttributeMetaData(am);
    }

    // load all entities (optional)
    if (source.getRepositoryByEntityName("entities") != null) {
      int i = 1;
      for (Entity e : source.getRepositoryByEntityName("entities")) {
        i++;
        String entityName = e.getString("name");

        // required
        if (entityName == null)
          throw new IllegalArgumentException("entity.name is missing on line " + i);

        if (entities.get(entityName) == null)
          entities.put(entityName, new DefaultEntityMetaData(entityName));
        DefaultEntityMetaData md = entities.get(entityName);

        if (e.get("description") != null) md.setDescription(e.getString("description"));
      }
    }

    // re-iterate to map the mrefs/xref refEntity (or give error if not found)
    // TODO: consider also those in existing db
    int i = 1;
    for (Entity a : source.getRepositoryByEntityName("attributes")) {
      i++;
      if (a.get("refEntity") != null) {
        DefaultEntityMetaData em = entities.get(a.getString("entity"));
        DefaultAttributeMetaData am =
            (DefaultAttributeMetaData) em.getAttribute(a.getString("name"));

        if (entities.get(a.getString("refEntity")) == null) {
          throw new IllegalArgumentException(
              "attributes.refEntity error on line "
                  + i
                  + ": "
                  + a.getString("refEntity")
                  + " unknown");
        }

        am.setRefEntity(entities.get(a.getString("refEntity")));
      }
    }

    return entities;
  }