@Override
public EntitiesValidationReport validateImport(RepositoryCollection source) {
EntitiesValidationReportImpl report = new EntitiesValidationReportImpl();
// compare the data sheets against metadata in store or imported file
Map<String, DefaultEntityMetaData> metaDataMap = getEntityMetaData(source);
for (String sheet : source.getEntityNames())
if (!"entities".equals(sheet) && !"attributes".equals(sheet)) {
// check if sheet is known?
if (metaDataMap.containsKey(sheet)) report.getSheetsImportable().put(sheet, true);
else report.getSheetsImportable().put(sheet, false);
// check the fields
Repository s = source.getRepositoryByEntityName(sheet);
EntityMetaData target = metaDataMap.get(sheet);
if (target != null) {
List<String> fieldsAvailable = new ArrayList<String>();
List<String> fieldsImportable = new ArrayList<String>();
List<String> fieldsRequired = new ArrayList<String>();
List<String> fieldsUnknown = new ArrayList<String>();
for (AttributeMetaData att : s.getEntityMetaData().getAttributes()) {
if (target.getAttribute(att.getName()) == null) fieldsUnknown.add(att.getName());
else fieldsImportable.add(att.getName());
}
for (AttributeMetaData att : target.getAttributes()) {
if (!fieldsImportable.contains(att.getName())) {
if (!att.isNillable()) fieldsRequired.add(att.getName());
else fieldsAvailable.add(att.getName());
}
}
report.getFieldsAvailable().put(sheet, fieldsAvailable);
report.getFieldsRequired().put(sheet, fieldsRequired);
report.getFieldsUnknown().put(sheet, fieldsUnknown);
report.getFieldsImportable().put(sheet, fieldsImportable);
}
}
return report;
}
@Test
public void integrationTestMetaData() {
DataServiceImpl dataServiceImpl = Mockito.mock(DataServiceImpl.class);
when(dataServiceImpl.hasRepository("attributes")).thenReturn(Boolean.FALSE); // To skip the
// canIntegrateEntityMetadataCheck
// test
MetaDataServiceImpl metaDataService = new MetaDataServiceImpl(dataServiceImpl);
RepositoryCollection repositoryCollection = Mockito.mock(RepositoryCollection.class);
when(repositoryCollection.getEntityNames()).thenReturn(Lists.newArrayList("attributes"));
DefaultEntityMetaData newEntityMetaData = new DefaultEntityMetaData("attributes");
newEntityMetaData.addAttribute("ID");
Repository repo1 = Mockito.mock(Repository.class);
when(repositoryCollection.getRepository("attributes")).thenReturn(repo1);
when(repo1.getEntityMetaData()).thenReturn(newEntityMetaData);
LinkedHashMap<String, Boolean> entitiesImportable = new LinkedHashMap<String, Boolean>();
entitiesImportable.put("attributes", true);
assertEquals(entitiesImportable, metaDataService.integrationTestMetaData(repositoryCollection));
}
@Override
@Transactional(rollbackFor = IOException.class)
public EntityImportReport doImport(
RepositoryCollection repositories, DatabaseAction databaseAction) throws IOException {
// All new repository identifiers
List<String> newRepoIdentifiers = new ArrayList<String>();
// First import entities, the data sheets are ignored in the entitiesimporter
EntityImportReport importReport = entitiesImporter.importEntities(repositories, databaseAction);
// RULE: Feature can only belong to one Protocol in a DataSet. Check it (see issue #1136)
checkFeatureCanOnlyBelongToOneProtocolForOneDataSet();
// Import data sheets
for (String name : repositories.getEntityNames()) {
Repository repository = repositories.getRepositoryByEntityName(name);
if (repository.getName().startsWith(DATASET_SHEET_PREFIX)) {
// Import DataSet sheet, create new OmxRepository
String identifier = repository.getName().substring(DATASET_SHEET_PREFIX.length());
if (!dataService.hasRepository(identifier)) {
dataService.addRepository(
new AggregateableCrudRepositorySecurityDecorator(
new OmxRepository(dataService, searchService, identifier, entityValidator)));
newRepoIdentifiers.add(identifier);
DataSet dataSet =
dataService.findOne(
DataSet.ENTITY_NAME,
new QueryImpl().eq(DataSet.IDENTIFIER, identifier),
DataSet.class);
List<Protocol> protocols =
ProtocolUtils.getProtocolDescendants(dataSet.getProtocolUsed());
List<ObservableFeature> categoricalFeatures = new ArrayList<ObservableFeature>();
for (Protocol protocol : protocols) {
List<ObservableFeature> observableFeatures = protocol.getFeatures();
if (observableFeatures != null) {
for (ObservableFeature observableFeature : observableFeatures) {
String dataType = observableFeature.getDataType();
FieldType type = MolgenisFieldTypes.getType(dataType);
if (type.getEnumType() == FieldTypeEnum.CATEGORICAL) {
categoricalFeatures.add(observableFeature);
}
}
}
}
for (ObservableFeature categoricalFeature : categoricalFeatures) {
if (!dataService.hasRepository(
OmxLookupTableEntityMetaData.createOmxLookupTableEntityMetaDataName(
categoricalFeature.getIdentifier()))) {
dataService.addRepository(
new OmxLookupTableRepository(
dataService, categoricalFeature.getIdentifier(), queryResolver));
newRepoIdentifiers.add(
OmxLookupTableEntityMetaData.createOmxLookupTableEntityMetaDataName(
categoricalFeature.getIdentifier()));
}
}
}
// Check if all column names in the excel sheet exist as attributes of the entity
Set<ConstraintViolation> violations = Sets.newLinkedHashSet();
EntityMetaData meta = dataService.getEntityMetaData(identifier);
for (AttributeMetaData attr : repository.getEntityMetaData().getAttributes()) {
if (meta.getAttribute(attr.getName()) == null) {
String message =
String.format(
"Unknown attributename '%s' for entity '%s'. Sheet: '%s'",
attr.getName(), meta.getName(), repository.getName());
violations.add(new ConstraintViolation(message, attr.getName(), null, null, meta, 0));
}
}
if (!violations.isEmpty()) {
throw new MolgenisValidationException(violations);
}
// Import data into new OmxRepository
try {
dataService.add(identifier, repository);
} catch (MolgenisValidationException e) {
// Add sheet info
for (ConstraintViolation violation : e.getViolations()) {
if (violation.getRownr() > 0) {
// Rownr +1 for header
violation.setImportInfo(
String.format(
"Sheet: '%s', row: %d", repository.getName(), violation.getRownr() + 1));
} else {
violation.setImportInfo(String.format("Sheet: '%s'", repository.getName()));
}
}
for (String newRepoIdentifier : newRepoIdentifiers) {
dataService.removeRepository(newRepoIdentifier);
}
throw e;
}
int count = (int) RepositoryUtils.count(repository);
importReport.addEntityCount(identifier, count);
importReport.addNrImported(count);
}
}
return importReport;
}
