@Override public EntitiesValidationReport validateImport(RepositoryCollection source) { EntitiesValidationReportImpl report = new EntitiesValidationReportImpl(); // compare the data sheets against metadata in store or imported file Map<String, DefaultEntityMetaData> metaDataMap = getEntityMetaData(source); for (String sheet : source.getEntityNames()) if (!"entities".equals(sheet) && !"attributes".equals(sheet)) { // check if sheet is known? if (metaDataMap.containsKey(sheet)) report.getSheetsImportable().put(sheet, true); else report.getSheetsImportable().put(sheet, false); // check the fields Repository s = source.getRepositoryByEntityName(sheet); EntityMetaData target = metaDataMap.get(sheet); if (target != null) { List<String> fieldsAvailable = new ArrayList<String>(); List<String> fieldsImportable = new ArrayList<String>(); List<String> fieldsRequired = new ArrayList<String>(); List<String> fieldsUnknown = new ArrayList<String>(); for (AttributeMetaData att : s.getEntityMetaData().getAttributes()) { if (target.getAttribute(att.getName()) == null) fieldsUnknown.add(att.getName()); else fieldsImportable.add(att.getName()); } for (AttributeMetaData att : target.getAttributes()) { if (!fieldsImportable.contains(att.getName())) { if (!att.isNillable()) fieldsRequired.add(att.getName()); else fieldsAvailable.add(att.getName()); } } report.getFieldsAvailable().put(sheet, fieldsAvailable); report.getFieldsRequired().put(sheet, fieldsRequired); report.getFieldsUnknown().put(sheet, fieldsUnknown); report.getFieldsImportable().put(sheet, fieldsImportable); } } return report; }
@Test public void integrationTestMetaData() { DataServiceImpl dataServiceImpl = Mockito.mock(DataServiceImpl.class); when(dataServiceImpl.hasRepository("attributes")).thenReturn(Boolean.FALSE); // To skip the // canIntegrateEntityMetadataCheck // test MetaDataServiceImpl metaDataService = new MetaDataServiceImpl(dataServiceImpl); RepositoryCollection repositoryCollection = Mockito.mock(RepositoryCollection.class); when(repositoryCollection.getEntityNames()).thenReturn(Lists.newArrayList("attributes")); DefaultEntityMetaData newEntityMetaData = new DefaultEntityMetaData("attributes"); newEntityMetaData.addAttribute("ID"); Repository repo1 = Mockito.mock(Repository.class); when(repositoryCollection.getRepository("attributes")).thenReturn(repo1); when(repo1.getEntityMetaData()).thenReturn(newEntityMetaData); LinkedHashMap<String, Boolean> entitiesImportable = new LinkedHashMap<String, Boolean>(); entitiesImportable.put("attributes", true); assertEquals(entitiesImportable, metaDataService.integrationTestMetaData(repositoryCollection)); }
@Override @Transactional(rollbackFor = IOException.class) public EntityImportReport doImport( RepositoryCollection repositories, DatabaseAction databaseAction) throws IOException { // All new repository identifiers List<String> newRepoIdentifiers = new ArrayList<String>(); // First import entities, the data sheets are ignored in the entitiesimporter EntityImportReport importReport = entitiesImporter.importEntities(repositories, databaseAction); // RULE: Feature can only belong to one Protocol in a DataSet. Check it (see issue #1136) checkFeatureCanOnlyBelongToOneProtocolForOneDataSet(); // Import data sheets for (String name : repositories.getEntityNames()) { Repository repository = repositories.getRepositoryByEntityName(name); if (repository.getName().startsWith(DATASET_SHEET_PREFIX)) { // Import DataSet sheet, create new OmxRepository String identifier = repository.getName().substring(DATASET_SHEET_PREFIX.length()); if (!dataService.hasRepository(identifier)) { dataService.addRepository( new AggregateableCrudRepositorySecurityDecorator( new OmxRepository(dataService, searchService, identifier, entityValidator))); newRepoIdentifiers.add(identifier); DataSet dataSet = dataService.findOne( DataSet.ENTITY_NAME, new QueryImpl().eq(DataSet.IDENTIFIER, identifier), DataSet.class); List<Protocol> protocols = ProtocolUtils.getProtocolDescendants(dataSet.getProtocolUsed()); List<ObservableFeature> categoricalFeatures = new ArrayList<ObservableFeature>(); for (Protocol protocol : protocols) { List<ObservableFeature> observableFeatures = protocol.getFeatures(); if (observableFeatures != null) { for (ObservableFeature observableFeature : observableFeatures) { String dataType = observableFeature.getDataType(); FieldType type = MolgenisFieldTypes.getType(dataType); if (type.getEnumType() == FieldTypeEnum.CATEGORICAL) { categoricalFeatures.add(observableFeature); } } } } for (ObservableFeature categoricalFeature : categoricalFeatures) { if (!dataService.hasRepository( OmxLookupTableEntityMetaData.createOmxLookupTableEntityMetaDataName( categoricalFeature.getIdentifier()))) { dataService.addRepository( new OmxLookupTableRepository( dataService, categoricalFeature.getIdentifier(), queryResolver)); newRepoIdentifiers.add( OmxLookupTableEntityMetaData.createOmxLookupTableEntityMetaDataName( categoricalFeature.getIdentifier())); } } } // Check if all column names in the excel sheet exist as attributes of the entity Set<ConstraintViolation> violations = Sets.newLinkedHashSet(); EntityMetaData meta = dataService.getEntityMetaData(identifier); for (AttributeMetaData attr : repository.getEntityMetaData().getAttributes()) { if (meta.getAttribute(attr.getName()) == null) { String message = String.format( "Unknown attributename '%s' for entity '%s'. Sheet: '%s'", attr.getName(), meta.getName(), repository.getName()); violations.add(new ConstraintViolation(message, attr.getName(), null, null, meta, 0)); } } if (!violations.isEmpty()) { throw new MolgenisValidationException(violations); } // Import data into new OmxRepository try { dataService.add(identifier, repository); } catch (MolgenisValidationException e) { // Add sheet info for (ConstraintViolation violation : e.getViolations()) { if (violation.getRownr() > 0) { // Rownr +1 for header violation.setImportInfo( String.format( "Sheet: '%s', row: %d", repository.getName(), violation.getRownr() + 1)); } else { violation.setImportInfo(String.format("Sheet: '%s'", repository.getName())); } } for (String newRepoIdentifier : newRepoIdentifiers) { dataService.removeRepository(newRepoIdentifier); } throw e; } int count = (int) RepositoryUtils.count(repository); importReport.addEntityCount(identifier, count); importReport.addNrImported(count); } } return importReport; }