@Override
  @RunAsSystem
  public void initialize() {
    Resource resource = new ClassPathResource("roc-curve.R");
    if (resource.exists()) {
      long count =
          dataService.count(
              Script.ENTITY_NAME, new QueryImpl().eq(Script.NAME, ROC_CURVE_SCRIPT_NAME));
      if (count == 0) {
        Entity scriptType =
            dataService.findOne(ScriptType.ENTITY_NAME, new QueryImpl().eq(ScriptType.NAME, "R"));

        if (scriptType == null) throw new UnknownEntityException("ScriptType R does not exist!");

        String scriptContent;
        try {
          scriptContent =
              FileCopyUtils.copyToString(new InputStreamReader(resource.getInputStream(), "UTF-8"));

        } catch (IOException e) {
          throw new UncheckedIOException(e);
        }

        if (dataService.count(
                ScriptParameter.ENTITY_NAME,
                new QueryImpl().eq(ScriptParameter.NAME, ROC_CURVE_SCRIPT_PARAMETER))
            == 0) {
          dataService.add(
              ScriptParameter.ENTITY_NAME,
              new MapEntity(ImmutableMap.of(ScriptParameter.NAME, ROC_CURVE_SCRIPT_PARAMETER)));
        }

        Entity scriptParameterEntity =
            dataService.findOne(
                ScriptParameter.ENTITY_NAME,
                new QueryImpl().eq(ScriptParameter.NAME, ROC_CURVE_SCRIPT_PARAMETER));

        MapEntity scriptEntity = new MapEntity();
        scriptEntity.set(Script.NAME, ROC_CURVE_SCRIPT_NAME);
        scriptEntity.set(Script.GENERATE_TOKEN, true);
        scriptEntity.set(Script.TYPE, scriptType);
        scriptEntity.set(Script.RESULT_FILE_EXTENSION, "png");
        scriptEntity.set(Script.CONTENT, scriptContent);
        scriptEntity.set(Script.PARAMETERS, Arrays.asList(scriptParameterEntity));
        dataService.add(Script.ENTITY_NAME, scriptEntity);

        LOG.info("Script entity \"roc\" has been added to the database!");
      } else {
        LOG.info("Script entity \"roc\" already exists in the database!");
      }
    } else {
      LOG.info("R script \"roc-curve.R\" does not exist on classpath!");
    }
  }
  @RequestMapping(value = "/{id:.+}", method = GET)
  public void getFile(@PathVariable("id") String id, HttpServletResponse response)
      throws IOException {
    FileMeta fileMeta = dataService.findOne(FileMeta.ENTITY_NAME, id, FileMeta.class);
    if (fileMeta == null) {
      response.setStatus(HttpStatus.NOT_FOUND.value());
    } else {

      java.io.File fileStoreFile = fileStore.getFile(fileMeta.getFilename());

      // if file meta data exists for this file
      String outputFilename = fileMeta.getFilename();

      String contentType = fileMeta.getContentType();
      if (contentType != null) {
        response.setContentType(contentType);
      }

      Long size = fileMeta.getSize();
      if (size != null) {
        response.setContentLength(size.intValue());
      }

      response.setHeader(
          "Content-Disposition", "attachment; filename=" + outputFilename.replace(" ", "_"));

      InputStream is = new FileInputStream(fileStoreFile);
      try {
        FileCopyUtils.copy(is, response.getOutputStream());
      } finally {
        is.close();
      }
    }
  }
  private List<? extends Authority> getUserPermissions(String userId, String authorityPrefix) {
    MolgenisUser molgenisUser =
        dataService.findOne(MolgenisUser.ENTITY_NAME, userId, MolgenisUser.class);
    if (molgenisUser == null) throw new RuntimeException("unknown user id [" + userId + "]");
    List<Authority> userPermissions = getUserPermissions(molgenisUser, authorityPrefix);

    Iterable<MolgenisGroupMember> groupMembersIt =
        dataService.findAll(
            MolgenisGroupMember.ENTITY_NAME,
            new QueryImpl().eq(MolgenisGroupMember.MOLGENISUSER, molgenisUser),
            MolgenisGroupMember.class);

    if (groupMembersIt == null) {
      return Collections.emptyList();
    }

    List<MolgenisGroupMember> groupMembers = Lists.newArrayList(groupMembersIt);

    if (groupMembers != null && !groupMembers.isEmpty()) {
      List<MolgenisGroup> molgenisGroups =
          Lists.transform(
              groupMembers,
              new Function<MolgenisGroupMember, MolgenisGroup>() {
                @Override
                public MolgenisGroup apply(MolgenisGroupMember molgenisGroupMember) {
                  return molgenisGroupMember.getMolgenisGroup();
                }
              });
      List<Authority> groupAuthorities = getGroupPermissions(molgenisGroups, authorityPrefix);
      if (groupAuthorities != null && !groupAuthorities.isEmpty())
        userPermissions.addAll(groupAuthorities);
    }

    return userPermissions;
  }
 @Override
 public Catalog getCatalog(String id) throws UnknownCatalogException {
   Protocol protocol =
       dataService.findOne(
           Protocol.ENTITY_NAME, new QueryImpl().eq(Protocol.ID, id), Protocol.class);
   if (protocol == null) throw new UnknownCatalogException("Catalog [" + id + "] does not exist");
   return new OmxCatalog(protocol, dataService);
 }
 @Override
 public boolean isCatalogOfStudyDefinitionLoaded(String id)
     throws UnknownCatalogException, UnknownStudyDefinitionException {
   StudyDataRequest studyDataRequest =
       dataService.findOne(
           StudyDataRequest.ENTITY_NAME,
           new QueryImpl().eq(StudyDataRequest.ID, Integer.valueOf(id)),
           StudyDataRequest.class);
   if (studyDataRequest == null)
     throw new UnknownStudyDefinitionException("Study definition [" + id + "] does not exist");
   Protocol protocol =
       dataService.findOne(
           Protocol.ENTITY_NAME, new QueryImpl().eq(Protocol.ID, id), Protocol.class);
   if (protocol == null)
     throw new UnknownCatalogException("No catalog defined for study definition [" + id + "]");
   return protocol.getActive();
 }
 @Override
 public void unloadCatalog(String id) throws UnknownCatalogException {
   Protocol protocol =
       dataService.findOne(
           Protocol.ENTITY_NAME, new QueryImpl().eq(Protocol.ID, id), Protocol.class);
   if (protocol == null) throw new UnknownCatalogException("Catalog [" + id + "] does not exist");
   setProtocolActive(protocol, false);
 }
 @Override
 public boolean isCatalogActivated(String id) throws UnknownCatalogException {
   Protocol protocol =
       dataService.findOne(
           Protocol.ENTITY_NAME, new QueryImpl().eq(Protocol.ID, id), Protocol.class);
   if (protocol == null) throw new UnknownCatalogException("Catalog [" + id + "] does not exist");
   return protocol.getActive();
 }
  /**
   * @param entityName The name of the entity to update
   * @param attributeName The name of the attribute to update
   * @param request EntityCollectionBatchRequestV2
   * @param response HttpServletResponse
   * @throws Exception
   */
  @RequestMapping(value = "/{entityName}/{attributeName}", method = PUT)
  @ResponseStatus(OK)
  public synchronized void updateAttribute(
      @PathVariable("entityName") String entityName,
      @PathVariable("attributeName") String attributeName,
      @RequestBody @Valid EntityCollectionBatchRequestV2 request,
      HttpServletResponse response)
      throws Exception {
    final EntityMetaData meta = dataService.getEntityMetaData(entityName);
    if (meta == null) {
      throw createUnknownEntityException(entityName);
    }

    try {
      AttributeMetaData attr = meta.getAttribute(attributeName);
      if (attr == null) {
        throw createUnknownAttributeException(entityName, attributeName);
      }

      if (attr.isReadonly()) {
        throw createMolgenisDataAccessExceptionReadOnlyAttribute(entityName, attributeName);
      }

      final List<Entity> entities =
          request
              .getEntities()
              .stream()
              .filter(e -> e.size() == 2)
              .map(e -> this.restService.toEntity(meta, e))
              .collect(Collectors.toList());
      if (entities.size() != request.getEntities().size()) {
        throw createMolgenisDataExceptionIdentifierAndValue();
      }

      final List<Entity> updatedEntities = new ArrayList<Entity>();
      int count = 0;
      for (Entity entity : entities) {
        String id = checkForEntityId(entity, count);

        Entity originalEntity = dataService.findOne(entityName, id);
        if (originalEntity == null) {
          throw createUnknownEntityExceptionNotValidId(id);
        }

        Object value = this.restService.toEntityValue(attr, entity.get(attributeName));
        originalEntity.set(attributeName, value);
        updatedEntities.add(originalEntity);
        count++;
      }

      // update all entities
      this.dataService.update(entityName, updatedEntities.stream());
      response.setStatus(HttpServletResponse.SC_OK);
    } catch (Exception e) {
      response.setStatus(HttpServletResponse.SC_NO_CONTENT);
      throw e;
    }
  }
  @Test
  public void deleteNoMetadata() throws IOException {
    when(dataService.findOne(
            DataSet.ENTITY_NAME, new QueryImpl().eq(DataSet.IDENTIFIER, "dataset1")))
        .thenReturn(dataset);

    dataSetDeleterServiceImpl.deleteData("dataset1");
    verify(dataService, Mockito.times(0)).delete(DataSet.ENTITY_NAME, dataset);
  }
 @Test
 public void testUpdateUnknown() {
   mappingProject.setIdentifier("mappingProjectID");
   when(dataService.findOne(ENTITY_NAME, "mappingProjectID")).thenReturn(null);
   try {
     mappingProjectRepositoryImpl.update(mappingProject);
     fail("Expected exception");
   } catch (MolgenisDataException expected) {
     assertEquals(expected.getMessage(), "MappingProject does not exist");
   }
 }
 @Override
 @PreAuthorize("hasAnyRole('ROLE_SU')")
 @Transactional(readOnly = true)
 public Permissions getGroupEntityClassPermissions(String groupId) {
   MolgenisGroup molgenisGroup =
       dataService.findOne(MolgenisGroup.ENTITY_NAME, groupId, MolgenisGroup.class);
   if (molgenisGroup == null) throw new RuntimeException("unknown group id [" + groupId + "]");
   List<Authority> groupPermissions = getGroupPermissions(molgenisGroup);
   Permissions permissions =
       createPermissions(groupPermissions, SecurityUtils.AUTHORITY_ENTITY_PREFIX);
   permissions.setGroupId(groupId);
   return permissions;
 }
  @Test
  public void removeToken() {
    MolgenisToken molgenisToken = new MolgenisToken();
    molgenisToken.setToken("token");

    when(dataService.findOne(
            MolgenisToken.ENTITY_NAME,
            new QueryImpl().eq(MolgenisToken.TOKEN, "token"),
            MolgenisToken.class))
        .thenReturn(molgenisToken);

    tokenService.removeToken("token");
    verify(dataService).delete(MolgenisToken.ENTITY_NAME, molgenisToken);
  }
 @Override
 public Catalog getCatalogOfStudyDefinition(String id)
     throws UnknownCatalogException, UnknownStudyDefinitionException {
   StudyDataRequest studyDataRequest =
       dataService.findOne(
           StudyDataRequest.ENTITY_NAME,
           new QueryImpl().eq(StudyDataRequest.ID, Integer.valueOf(id)),
           StudyDataRequest.class);
   if (studyDataRequest == null)
     throw new UnknownStudyDefinitionException("Study definition [" + id + "] does not exist");
   Protocol protocol = studyDataRequest.getProtocol();
   if (protocol == null) throw new UnknownCatalogException("Catalog [" + id + "] does not exist");
   return new OmxCatalog(protocol, dataService);
 }
  @Test(expectedExceptions = UnknownTokenException.class)
  public void findUserByTokenExpired() {
    MolgenisToken molgenisToken = new MolgenisToken();
    molgenisToken.setToken("token");
    molgenisToken.setExpirationDate(DateUtils.addDays(new Date(), -1));

    when(dataService.findOne(
            MolgenisToken.ENTITY_NAME,
            new QueryImpl().eq(MolgenisToken.TOKEN, "token"),
            MolgenisToken.class))
        .thenReturn(molgenisToken);

    tokenService.findUserByToken("token");
  }
  @Test
  public void generateAndStoreToken() {
    MolgenisUser user = new MolgenisUser();

    when(dataService.findOne(
            MolgenisUser.ENTITY_NAME,
            new QueryImpl().eq(MolgenisUser.USERNAME, "admin"),
            MolgenisUser.class))
        .thenReturn(user);

    when(tokenGenerator.generateToken()).thenReturn("token");
    assertEquals(tokenService.generateAndStoreToken("admin", "description"), "token");

    MolgenisToken molgenisToken = new MolgenisToken();
    molgenisToken.setToken("token");
    verify(dataService).add(MolgenisToken.ENTITY_NAME, molgenisToken);
  }
  @RequestMapping(value = "/{entityName}/{id:.+}", method = POST, params = "_method=GET")
  @ResponseBody
  public Map<String, Object> retrieveEntityPost(
      @PathVariable("entityName") String entityName,
      @PathVariable("id") Object id,
      @RequestParam(value = "attrs", required = false) AttributeFilter attributeFilter) {
    EntityMetaData entityMeta = dataService.getEntityMetaData(entityName);
    Fetch fetch =
        AttributeFilterToFetchConverter.convert(
            attributeFilter, entityMeta, languageService.getCurrentUserLanguageCode());

    Entity entity = dataService.findOne(entityName, id, fetch);
    if (entity == null) {
      throw new UnknownEntityException(entityName + " [" + id + "] not found");
    }

    return createEntityResponse(entity, fetch, true);
  }
  private void replaceUserPermissions(
      List<UserAuthority> entityAuthorities, String userId, String authorityType) {
    MolgenisUser molgenisUser =
        dataService.findOne(MolgenisUser.ENTITY_NAME, userId, MolgenisUser.class);
    if (molgenisUser == null) throw new RuntimeException("unknown user id [" + userId + "]");

    // inject user
    for (UserAuthority entityAuthority : entityAuthorities)
      entityAuthority.setMolgenisUser(molgenisUser);

    // delete old plugin authorities
    List<? extends Authority> oldEntityAuthorities =
        getUserPermissions(molgenisUser, authorityType);
    if (oldEntityAuthorities != null && !oldEntityAuthorities.isEmpty())
      dataService.delete(UserAuthority.ENTITY_NAME, oldEntityAuthorities);

    // insert new plugin authorities
    if (!entityAuthorities.isEmpty()) dataService.add(UserAuthority.ENTITY_NAME, entityAuthorities);
  }
  private void replaceGroupPermissions(
      List<GroupAuthority> entityAuthorities, String groupId, String authorityPrefix) {
    MolgenisGroup molgenisGroup =
        dataService.findOne(MolgenisGroup.ENTITY_NAME, groupId, MolgenisGroup.class);
    if (molgenisGroup == null) throw new RuntimeException("unknown group id [" + groupId + "]");

    // inject user
    for (GroupAuthority entityAuthority : entityAuthorities)
      entityAuthority.setMolgenisGroup(molgenisGroup);

    // delete old plugin authorities
    List<Authority> oldEntityAuthorities = getGroupPermissions(molgenisGroup, authorityPrefix);
    if (oldEntityAuthorities != null && !oldEntityAuthorities.isEmpty())
      dataService.delete(GroupAuthority.ENTITY_NAME, oldEntityAuthorities);

    // insert new plugin authorities
    if (!entityAuthorities.isEmpty())
      dataService.add(GroupAuthority.ENTITY_NAME, entityAuthorities);
  }
  /**
   * A helper function that gets identifiers of all the attributes from one entityMetaData
   *
   * @param sourceEntityMetaData
   * @return
   */
  public List<String> getAttributeIdentifiers(EntityMetaData sourceEntityMetaData) {
    Entity entityMetaDataEntity =
        dataService.findOne(
            EntityMetaDataMetaData.ENTITY_NAME,
            new QueryImpl().eq(EntityMetaDataMetaData.FULL_NAME, sourceEntityMetaData.getName()));

    if (entityMetaDataEntity == null)
      throw new MolgenisDataAccessException(
          "Could not find EntityMetaDataEntity by the name of " + sourceEntityMetaData.getName());

    return FluentIterable.from(entityMetaDataEntity.getEntities(EntityMetaDataMetaData.ATTRIBUTES))
        .transform(
            new Function<Entity, String>() {
              public String apply(Entity attributeEntity) {
                return attributeEntity.getString(AttributeMetaDataMetaData.IDENTIFIER);
              }
            })
        .toList();
  }
  @Test
  public void findUserByToken() {
    MolgenisToken molgenisToken = new MolgenisToken();
    molgenisToken.setToken("token");
    MolgenisUser user = new MolgenisUser();
    user.setUsername("admin");
    molgenisToken.setMolgenisUser(user);

    when(dataService.findOne(
            MolgenisToken.ENTITY_NAME,
            new QueryImpl().eq(MolgenisToken.TOKEN, "token"),
            MolgenisToken.class))
        .thenReturn(molgenisToken);

    UserDetails userDetails =
        new User("admin", "admin", Arrays.asList(new SimpleGrantedAuthority("admin")));
    when(userDetailsService.loadUserByUsername("admin")).thenReturn(userDetails);

    assertEquals(tokenService.findUserByToken("token"), userDetails);
  }
  @RequestMapping(
      value = "/verify",
      method = RequestMethod.POST,
      headers = "Content-Type=multipart/form-data")
  public void verify(
      @RequestParam(value = "selectedDataSet", required = false) String selectedDataSetId,
      @RequestParam Part file,
      HttpServletResponse response,
      Model model)
      throws IOException {
    EntitySource reader = null;
    ExcelWriter<Entity> excelWriterRanks = null;

    try {
      if (selectedDataSetId != null) {
        String origFileName = FileUploadUtils.getOriginalFileName(file);
        File uploadFile = fileStore.store(file.getInputStream(), origFileName);
        response.setContentType("application/vnd.ms-excel");
        response.addHeader(
            "Content-Disposition",
            "attachment; filename=" + getCsvFileName(file.getName() + "-ranks"));
        excelWriterRanks = new ExcelWriter<Entity>(response.getOutputStream());
        excelWriterRanks.addCellProcessor(new LowerCaseProcessor(true, false));

        Writable<Entity> sheetWriterRank = null;
        Writable<Entity> sheetWriterRankStatistics = null;
        Writable<Entity> sheetWriteBiobankRanks = null;
        Writable<Entity> sheetWriteSpssInput = null;

        reader = new ExcelEntitySourceFactory().create(uploadFile);
        Repository<? extends Entity> inputSheet = reader.getRepositoryByEntityName("Sheet1");

        List<String> biobankNames = new ArrayList<String>();
        for (AttributeMetaData attr : inputSheet.getAttributes()) {
          biobankNames.add(attr.getName());
        }
        String firstColumn = biobankNames.get(0);
        biobankNames.remove(0);

        // First column has to correspond to the selected dataset
        DataSet ds = dataService.findOne(DataSet.ENTITY_NAME, Integer.parseInt(selectedDataSetId));

        if (ds.getName().equalsIgnoreCase(firstColumn)) {
          Map<String, Map<String, List<String>>> maunalMappings =
              new HashMap<String, Map<String, List<String>>>();
          for (Entity row : inputSheet) {
            String variableName = row.getString(firstColumn);
            if (!maunalMappings.containsKey(variableName))
              maunalMappings.put(variableName, new HashMap<String, List<String>>());
            for (String biobank : biobankNames) {
              if (row.get(biobank) != null) {
                String mappingString = row.get(biobank).toString();
                if (!maunalMappings.containsKey(variableName)) {
                  maunalMappings.put(variableName, new HashMap<String, List<String>>());
                }
                if (!maunalMappings.get(variableName).containsKey(biobank.toLowerCase())) {
                  maunalMappings
                      .get(variableName)
                      .put(biobank.toLowerCase(), new ArrayList<String>());
                }
                maunalMappings
                    .get(variableName)
                    .get(biobank.toLowerCase())
                    .addAll(Arrays.asList(mappingString.split(",")));
              }
            }
          }

          List<String> lowerCaseBiobankNames = new ArrayList<String>();
          for (String element : biobankNames) {
            lowerCaseBiobankNames.add(element.toLowerCase());
          }

          List<DataSet> dataSets =
              dataService.findAllAsList(
                  DataSet.ENTITY_NAME, new QueryImpl().in(DataSet.NAME, lowerCaseBiobankNames));

          lowerCaseBiobankNames.add(0, firstColumn.toLowerCase());
          sheetWriterRank = excelWriterRanks.createWritable("result", lowerCaseBiobankNames);

          Map<String, Map<String, List<Integer>>> rankCollection =
              new HashMap<String, Map<String, List<Integer>>>();
          List<Object> allRanks = new ArrayList<Object>();

          for (Entry<String, Map<String, List<String>>> entry : maunalMappings.entrySet()) {
            String variableName = entry.getKey();
            List<String> ranks = new ArrayList<String>();
            ranks.add(variableName);
            Map<String, List<String>> mappingDetail = entry.getValue();
            List<ObservableFeature> features =
                dataService.findAllAsList(
                    ObservableFeature.ENTITY_NAME,
                    new QueryImpl().eq(ObservableFeature.NAME, variableName));
            String description = features.get(0).getDescription();
            if (!rankCollection.containsKey(description))
              rankCollection.put(description, new HashMap<String, List<Integer>>());

            if (!features.isEmpty()) {
              Entity row = new MapEntity();
              row.set(firstColumn.toLowerCase(), description);

              for (DataSet dataSet : dataSets) {
                List<Integer> ranksBiobank = new ArrayList<Integer>();
                if (mappingDetail.containsKey(dataSet.getName().toLowerCase())) {
                  Map<String, Hit> mappedFeatureIds =
                      findFeaturesFromIndex(
                          "name",
                          mappingDetail.get(dataSet.getName().toLowerCase()),
                          dataSet.getId());

                  String mappingDataSetIdentifier =
                      SecurityUtils.getCurrentUsername()
                          + "-"
                          + selectedDataSetId
                          + "-"
                          + dataSet.getId();

                  Query q =
                      new QueryImpl()
                          .eq("store_mapping_feature", features.get(0).getId())
                          .pageSize(50)
                          .sort(new Sort(Direction.DESC, "store_mapping_score"));

                  SearchRequest searchRequest =
                      new SearchRequest(mappingDataSetIdentifier, q, null);

                  SearchResult result = searchService.search(searchRequest);

                  if (mappedFeatureIds.size() == 0) {
                    row.set(dataSet.getName().toLowerCase(), "N/A2");
                    continue;
                  }

                  List<String> ids = new ArrayList<String>();
                  for (Hit hit : result.getSearchHits()) {
                    Map<String, Object> columnValueMap = hit.getColumnValueMap();
                    ids.add(columnValueMap.get("store_mapping_mapped_feature").toString());
                  }
                  Map<String, Hit> featureInfos = findFeaturesFromIndex("id", ids, dataSet.getId());

                  String previousDescription = null;
                  int rank = 0;
                  for (Hit hit : result.getSearchHits()) {
                    Map<String, Object> columnValueMap = hit.getColumnValueMap();
                    String mappedFeatureId =
                        columnValueMap.get("store_mapping_mapped_feature").toString();
                    String mappedFeatureDescription =
                        featureInfos
                            .get(mappedFeatureId)
                            .getColumnValueMap()
                            .get("description")
                            .toString()
                            .replaceAll("[^0-9a-zA-Z ]", " ");

                    rank++;
                    if (previousDescription != null
                        && previousDescription.equalsIgnoreCase(mappedFeatureDescription)) rank--;

                    if (mappedFeatureIds.containsKey(mappedFeatureId)) {
                      ranksBiobank.add(rank);
                      allRanks.add(rank);
                      mappedFeatureIds.remove(mappedFeatureId);
                    }
                    previousDescription = mappedFeatureDescription;
                  }
                  if (mappedFeatureIds.size() == 0) {
                    String output = StringUtils.join(ranksBiobank, ',');
                    if (ranksBiobank.size() > 1) {
                      output += " (" + averageRank(ranksBiobank) + ")";
                    }
                    row.set(dataSet.getName().toLowerCase(), output);
                  } else {
                    for (int i = 0; i < mappedFeatureIds.size(); i++) allRanks.add("Not mapped");
                    row.set(dataSet.getName().toLowerCase(), "Not mapped");
                    ranksBiobank.clear();
                  }
                } else row.set(dataSet.getName().toLowerCase(), "N/A1");

                rankCollection.get(description).put(dataSet.getName().toLowerCase(), ranksBiobank);
              }
              sheetWriterRank.add(row);
            }
          }

          Map<String, List<Integer>> rankCollectionPerBiobank =
              new HashMap<String, List<Integer>>();
          {
            sheetWriterRankStatistics =
                excelWriterRanks.createWritable(
                    "rank statistics",
                    Arrays.asList(
                        firstColumn.toLowerCase(),
                        "average rank",
                        "round-up rank",
                        "median rank",
                        "minium",
                        "maximum"));

            for (Entry<String, Map<String, List<Integer>>> entry : rankCollection.entrySet()) {
              String variableName = entry.getKey();
              Entity row = new MapEntity();
              row.set(firstColumn.toLowerCase(), variableName);
              List<Integer> rankAllBiobanks = new ArrayList<Integer>();
              for (Entry<String, List<Integer>> rankBiobanks : entry.getValue().entrySet()) {
                if (!rankCollectionPerBiobank.containsKey(rankBiobanks.getKey()))
                  rankCollectionPerBiobank.put(rankBiobanks.getKey(), new ArrayList<Integer>());
                rankCollectionPerBiobank.get(rankBiobanks.getKey()).addAll(rankBiobanks.getValue());
                rankAllBiobanks.addAll(rankBiobanks.getValue());
              }

              row.set("average rank", averageRank(rankAllBiobanks));
              row.set("round-up rank", Math.ceil(averageRank(rankAllBiobanks)));
              Collections.sort(rankAllBiobanks);
              if (!rankAllBiobanks.isEmpty()) {
                row.set("minium", rankAllBiobanks.get(0));
                row.set("maximum", rankAllBiobanks.get(rankAllBiobanks.size() - 1));

                double medianRank = 0;
                if (rankAllBiobanks.size() % 2 == 0) {
                  medianRank =
                      (double)
                              (rankAllBiobanks.get(rankAllBiobanks.size() / 2 - 1)
                                  + rankAllBiobanks.get(rankAllBiobanks.size() / 2))
                          / 2;
                } else {
                  medianRank = rankAllBiobanks.get(rankAllBiobanks.size() / 2);
                }
                row.set("median rank", medianRank);
              }

              sheetWriterRankStatistics.add(row);
            }
          }

          {
            lowerCaseBiobankNames.remove(0);
            sheetWriteBiobankRanks =
                excelWriterRanks.createWritable("biobank average ranks", lowerCaseBiobankNames);
            Entity entity = new MapEntity();
            for (Entry<String, List<Integer>> entry : rankCollectionPerBiobank.entrySet()) {
              entity.set(entry.getKey(), averageRank(entry.getValue()));
            }
            sheetWriteBiobankRanks.add(entity);
          }

          {
            sheetWriteSpssInput =
                excelWriterRanks.createWritable("spss ranks", Arrays.asList("rank"));
            for (Object rank : allRanks) {
              Entity entity = new MapEntity("rank", rank);
              sheetWriteSpssInput.add(entity);
            }
          }
        }
      }
    } finally {
      if (reader != null) reader.close();
      if (excelWriterRanks != null) IOUtils.closeQuietly(excelWriterRanks);
    }
  }
  @Override
  @Transactional(rollbackFor = IOException.class)
  public EntityImportReport doImport(
      RepositoryCollection repositories, DatabaseAction databaseAction) throws IOException {
    // All new repository identifiers
    List<String> newRepoIdentifiers = new ArrayList<String>();

    // First import entities, the data sheets are ignored in the entitiesimporter
    EntityImportReport importReport = entitiesImporter.importEntities(repositories, databaseAction);

    // RULE: Feature can only belong to one Protocol in a DataSet. Check it (see issue #1136)
    checkFeatureCanOnlyBelongToOneProtocolForOneDataSet();

    // Import data sheets
    for (String name : repositories.getEntityNames()) {
      Repository repository = repositories.getRepositoryByEntityName(name);

      if (repository.getName().startsWith(DATASET_SHEET_PREFIX)) {
        // Import DataSet sheet, create new OmxRepository
        String identifier = repository.getName().substring(DATASET_SHEET_PREFIX.length());

        if (!dataService.hasRepository(identifier)) {

          dataService.addRepository(
              new AggregateableCrudRepositorySecurityDecorator(
                  new OmxRepository(dataService, searchService, identifier, entityValidator)));
          newRepoIdentifiers.add(identifier);

          DataSet dataSet =
              dataService.findOne(
                  DataSet.ENTITY_NAME,
                  new QueryImpl().eq(DataSet.IDENTIFIER, identifier),
                  DataSet.class);

          List<Protocol> protocols =
              ProtocolUtils.getProtocolDescendants(dataSet.getProtocolUsed());
          List<ObservableFeature> categoricalFeatures = new ArrayList<ObservableFeature>();
          for (Protocol protocol : protocols) {
            List<ObservableFeature> observableFeatures = protocol.getFeatures();
            if (observableFeatures != null) {
              for (ObservableFeature observableFeature : observableFeatures) {
                String dataType = observableFeature.getDataType();
                FieldType type = MolgenisFieldTypes.getType(dataType);
                if (type.getEnumType() == FieldTypeEnum.CATEGORICAL) {
                  categoricalFeatures.add(observableFeature);
                }
              }
            }
          }
          for (ObservableFeature categoricalFeature : categoricalFeatures) {
            if (!dataService.hasRepository(
                OmxLookupTableEntityMetaData.createOmxLookupTableEntityMetaDataName(
                    categoricalFeature.getIdentifier()))) {
              dataService.addRepository(
                  new OmxLookupTableRepository(
                      dataService, categoricalFeature.getIdentifier(), queryResolver));
              newRepoIdentifiers.add(
                  OmxLookupTableEntityMetaData.createOmxLookupTableEntityMetaDataName(
                      categoricalFeature.getIdentifier()));
            }
          }
        }

        // Check if all column names in the excel sheet exist as attributes of the entity
        Set<ConstraintViolation> violations = Sets.newLinkedHashSet();
        EntityMetaData meta = dataService.getEntityMetaData(identifier);
        for (AttributeMetaData attr : repository.getEntityMetaData().getAttributes()) {
          if (meta.getAttribute(attr.getName()) == null) {
            String message =
                String.format(
                    "Unknown attributename '%s' for entity '%s'. Sheet: '%s'",
                    attr.getName(), meta.getName(), repository.getName());
            violations.add(new ConstraintViolation(message, attr.getName(), null, null, meta, 0));
          }
        }

        if (!violations.isEmpty()) {
          throw new MolgenisValidationException(violations);
        }

        // Import data into new OmxRepository
        try {
          dataService.add(identifier, repository);
        } catch (MolgenisValidationException e) {
          // Add sheet info
          for (ConstraintViolation violation : e.getViolations()) {
            if (violation.getRownr() > 0) {

              // Rownr +1 for header
              violation.setImportInfo(
                  String.format(
                      "Sheet: '%s', row: %d", repository.getName(), violation.getRownr() + 1));
            } else {
              violation.setImportInfo(String.format("Sheet: '%s'", repository.getName()));
            }
          }

          for (String newRepoIdentifier : newRepoIdentifiers) {
            dataService.removeRepository(newRepoIdentifier);
          }

          throw e;
        }

        int count = (int) RepositoryUtils.count(repository);
        importReport.addEntityCount(identifier, count);
        importReport.addNrImported(count);
      }
    }

    return importReport;
  }