@Override public EntitiesValidationReport validateImport(RepositoryCollection source) { EntitiesValidationReportImpl report = new EntitiesValidationReportImpl(); // compare the data sheets against metadata in store or imported file Map<String, DefaultEntityMetaData> metaDataMap = getEntityMetaData(source); for (String sheet : source.getEntityNames()) if (!"entities".equals(sheet) && !"attributes".equals(sheet)) { // check if sheet is known? if (metaDataMap.containsKey(sheet)) report.getSheetsImportable().put(sheet, true); else report.getSheetsImportable().put(sheet, false); // check the fields Repository s = source.getRepositoryByEntityName(sheet); EntityMetaData target = metaDataMap.get(sheet); if (target != null) { List<String> fieldsAvailable = new ArrayList<String>(); List<String> fieldsImportable = new ArrayList<String>(); List<String> fieldsRequired = new ArrayList<String>(); List<String> fieldsUnknown = new ArrayList<String>(); for (AttributeMetaData att : s.getEntityMetaData().getAttributes()) { if (target.getAttribute(att.getName()) == null) fieldsUnknown.add(att.getName()); else fieldsImportable.add(att.getName()); } for (AttributeMetaData att : target.getAttributes()) { if (!fieldsImportable.contains(att.getName())) { if (!att.isNillable()) fieldsRequired.add(att.getName()); else fieldsAvailable.add(att.getName()); } } report.getFieldsAvailable().put(sheet, fieldsAvailable); report.getFieldsRequired().put(sheet, fieldsRequired); report.getFieldsUnknown().put(sheet, fieldsUnknown); report.getFieldsImportable().put(sheet, fieldsImportable); } } return report; }
public Collection<EntityMetaData> loadEMD(RepositoryCollection emdFormattedRepos) { // extract entity metadata Map<String, DefaultEntityMetaData> entities = new LinkedHashMap<String, DefaultEntityMetaData>(); for (Entity e : emdFormattedRepos.getRepositoryByEntityName("entities")) { DefaultEntityMetaData emd = new DefaultEntityMetaData(e.getString("name")); if (e.getBoolean("abstract")) emd.setAbstract(true); } // extract extends relations for (Entity e : emdFormattedRepos.getRepositoryByEntityName("entities")) { if (e.get("extends") != null) { DefaultEntityMetaData emd = entities.get(e.get("name")); emd.setExtends(entities.get(e.get("extends"))); } } Collection<EntityMetaData> result = new ArrayList<EntityMetaData>(); result.addAll(entities.values()); return result; }
@Test public void integrationTestMetaData() { DataServiceImpl dataServiceImpl = Mockito.mock(DataServiceImpl.class); when(dataServiceImpl.hasRepository("attributes")).thenReturn(Boolean.FALSE); // To skip the // canIntegrateEntityMetadataCheck // test MetaDataServiceImpl metaDataService = new MetaDataServiceImpl(dataServiceImpl); RepositoryCollection repositoryCollection = Mockito.mock(RepositoryCollection.class); when(repositoryCollection.getEntityNames()).thenReturn(Lists.newArrayList("attributes")); DefaultEntityMetaData newEntityMetaData = new DefaultEntityMetaData("attributes"); newEntityMetaData.addAttribute("ID"); Repository repo1 = Mockito.mock(Repository.class); when(repositoryCollection.getRepository("attributes")).thenReturn(repo1); when(repo1.getEntityMetaData()).thenReturn(newEntityMetaData); LinkedHashMap<String, Boolean> entitiesImportable = new LinkedHashMap<String, Boolean>(); entitiesImportable.put("attributes", true); assertEquals(entitiesImportable, metaDataService.integrationTestMetaData(repositoryCollection)); }
@Override @Transactional(rollbackFor = IOException.class) public EntityImportReport doImport(RepositoryCollection source, DatabaseAction databaseAction) throws IOException { if (store == null) throw new RuntimeException("store was not set"); EntityImportReport report = new EntityImportReport(); // TODO: need to change order Map<String, DefaultEntityMetaData> metadata = getEntityMetaData(source); for (Entry<String, DefaultEntityMetaData> entry : metadata.entrySet()) { String name = entry.getKey(); if (!"entities".equals(name) && !"attributes".equals(name)) { Repository from = source.getRepositoryByEntityName(name); // TODO check if compatible with metadata // create repo if needed MysqlRepository to = (MysqlRepository) store.getRepositoryByEntityName(name); if (to == null) { logger.debug("tyring to create: " + name); EntityMetaData em = metadata.get(name); if (em == null) throw new IllegalArgumentException("Unknown entity: " + name); store.add(em); to = (MysqlRepository) store.getRepositoryByEntityName(name); } // import report.getNrImportedEntitiesMap().put(name, to.add(from)); } } return report; }
public Collection<EntityMetaData> loadOMX(RepositoryCollection omx) { // extract attribute metadata Map<String, AttributeMetaData> attributes = new LinkedHashMap<String, AttributeMetaData>(); for (Entity e : omx.getRepositoryByEntityName("observablefeature")) { logger.debug("found observablefeature: " + e); DefaultAttributeMetaData att = new DefaultAttributeMetaData(e.getString("name")); if (e.get("dataType") != null) att.setDataType(MolgenisFieldTypes.getType(e.getString("dataType"))); attributes.put(e.getString("identifier"), att); if (e.get("description") != null) att.setDescription(e.getString("description")); // TODO unit! if(e.get("unit") != null) if ("xref".equals(e.get("dataType")) || "mref".equals(e.get("dataType"))) { // TODO: cannot solve!!! att.setRefEntity(omxEntities.get("Characteristic")); } if ("categorical".equals(e.get("dataType"))) { att.setRefEntity(omxEntities.get("Category")); } } // TODO: fix categorical! // extract protocols as entities(abstract=true) Map<String, EntityMetaData> entities = new LinkedHashMap<String, EntityMetaData>(); for (Entity e : omx.getRepositoryByEntityName("protocol")) { // skip all null entities if (hasValues(e)) { logger.debug("found protocol: " + e); DefaultEntityMetaData ent = new DefaultEntityMetaData(e.getString("identifier")); // alas ent.setLabel(e.getString("name")); ent.setAbstract(true); // add attributes if (e.get("features_identifier") != null) for (String attIdentifier : e.getList("features_identifier")) { if (attributes.get(attIdentifier) == null) throw new RuntimeException("attribute '" + attIdentifier + "' unknown"); ent.addAttributeMetaData(attributes.get(attIdentifier)); } entities.put(e.getString("identifier"), ent); } } for (Entity e : omx.getRepositoryByEntityName("protocol")) { // add subprotocols as compound if (e.get("subprotocols_identifier") != null) { for (String subProtocol : e.getList("subprotocols_identifier")) { DefaultAttributeMetaData att = new DefaultAttributeMetaData(subProtocol); att.setDataType(MolgenisFieldTypes.COMPOUND); att.setRefEntity(entities.get(subProtocol)); ((DefaultEntityMetaData) entities.get(e.get("identifier"))).addAttributeMetaData(att); } } } // create dataset as entities for (Entity e : omx.getRepositoryByEntityName("dataset")) { logger.debug("found dataset: " + e); DefaultEntityMetaData ent = new DefaultEntityMetaData(e.getString("identifier")); ent.setLabel(e.getString("name")); // dataset 'extends' protocol ent.setExtends(entities.get(e.getString("protocolused_identifier"))); entities.put(e.getString("identifier"), ent); } return entities.values(); }
@Override @Transactional(rollbackFor = IOException.class) public EntityImportReport doImport( RepositoryCollection repositories, DatabaseAction databaseAction) throws IOException { // All new repository identifiers List<String> newRepoIdentifiers = new ArrayList<String>(); // First import entities, the data sheets are ignored in the entitiesimporter EntityImportReport importReport = entitiesImporter.importEntities(repositories, databaseAction); // RULE: Feature can only belong to one Protocol in a DataSet. Check it (see issue #1136) checkFeatureCanOnlyBelongToOneProtocolForOneDataSet(); // Import data sheets for (String name : repositories.getEntityNames()) { Repository repository = repositories.getRepositoryByEntityName(name); if (repository.getName().startsWith(DATASET_SHEET_PREFIX)) { // Import DataSet sheet, create new OmxRepository String identifier = repository.getName().substring(DATASET_SHEET_PREFIX.length()); if (!dataService.hasRepository(identifier)) { dataService.addRepository( new AggregateableCrudRepositorySecurityDecorator( new OmxRepository(dataService, searchService, identifier, entityValidator))); newRepoIdentifiers.add(identifier); DataSet dataSet = dataService.findOne( DataSet.ENTITY_NAME, new QueryImpl().eq(DataSet.IDENTIFIER, identifier), DataSet.class); List<Protocol> protocols = ProtocolUtils.getProtocolDescendants(dataSet.getProtocolUsed()); List<ObservableFeature> categoricalFeatures = new ArrayList<ObservableFeature>(); for (Protocol protocol : protocols) { List<ObservableFeature> observableFeatures = protocol.getFeatures(); if (observableFeatures != null) { for (ObservableFeature observableFeature : observableFeatures) { String dataType = observableFeature.getDataType(); FieldType type = MolgenisFieldTypes.getType(dataType); if (type.getEnumType() == FieldTypeEnum.CATEGORICAL) { categoricalFeatures.add(observableFeature); } } } } for (ObservableFeature categoricalFeature : categoricalFeatures) { if (!dataService.hasRepository( OmxLookupTableEntityMetaData.createOmxLookupTableEntityMetaDataName( categoricalFeature.getIdentifier()))) { dataService.addRepository( new OmxLookupTableRepository( dataService, categoricalFeature.getIdentifier(), queryResolver)); newRepoIdentifiers.add( OmxLookupTableEntityMetaData.createOmxLookupTableEntityMetaDataName( categoricalFeature.getIdentifier())); } } } // Check if all column names in the excel sheet exist as attributes of the entity Set<ConstraintViolation> violations = Sets.newLinkedHashSet(); EntityMetaData meta = dataService.getEntityMetaData(identifier); for (AttributeMetaData attr : repository.getEntityMetaData().getAttributes()) { if (meta.getAttribute(attr.getName()) == null) { String message = String.format( "Unknown attributename '%s' for entity '%s'. Sheet: '%s'", attr.getName(), meta.getName(), repository.getName()); violations.add(new ConstraintViolation(message, attr.getName(), null, null, meta, 0)); } } if (!violations.isEmpty()) { throw new MolgenisValidationException(violations); } // Import data into new OmxRepository try { dataService.add(identifier, repository); } catch (MolgenisValidationException e) { // Add sheet info for (ConstraintViolation violation : e.getViolations()) { if (violation.getRownr() > 0) { // Rownr +1 for header violation.setImportInfo( String.format( "Sheet: '%s', row: %d", repository.getName(), violation.getRownr() + 1)); } else { violation.setImportInfo(String.format("Sheet: '%s'", repository.getName())); } } for (String newRepoIdentifier : newRepoIdentifiers) { dataService.removeRepository(newRepoIdentifier); } throw e; } int count = (int) RepositoryUtils.count(repository); importReport.addEntityCount(identifier, count); importReport.addNrImported(count); } } return importReport; }
@Override public Map<String, DefaultEntityMetaData> getEntityMetaData(RepositoryCollection source) { // TODO: this task is actually a 'merge' instead of 'import' // so we need to consider both new metadata as existing ... Map<String, DefaultEntityMetaData> entities = new LinkedHashMap<String, DefaultEntityMetaData>(); // load attributes first (because entities are optional). for (Entity a : source.getRepositoryByEntityName("attributes")) { int i = 1; String entityName = a.getString("entity"); // required if (entityName == null) throw new IllegalArgumentException("attributes.entity is missing"); if (a.get("name") == null) throw new IllegalArgumentException("attributes.name is missing"); // create entity if not yet defined if (entities.get(entityName) == null) entities.put(entityName, new DefaultEntityMetaData(entityName)); DefaultEntityMetaData md = entities.get(entityName); DefaultAttributeMetaData am = new DefaultAttributeMetaData(a.getString("name")); if (a.get("dataType") != null) { FieldType t = MolgenisFieldTypes.getType(a.getString("dataType")); if (t == null) throw new IllegalArgumentException( "attributes.type error on line " + i + ": " + a.getString("dataType") + " unknown"); am.setDataType(t); } if (a.get("nillable") != null) am.setNillable(a.getBoolean("nillable")); if (a.get("auto") != null) am.setAuto(a.getBoolean("auto")); if (a.get("idAttribute") != null) am.setIdAttribute(a.getBoolean("idAttribute")); md.addAttributeMetaData(am); } // load all entities (optional) if (source.getRepositoryByEntityName("entities") != null) { int i = 1; for (Entity e : source.getRepositoryByEntityName("entities")) { i++; String entityName = e.getString("name"); // required if (entityName == null) throw new IllegalArgumentException("entity.name is missing on line " + i); if (entities.get(entityName) == null) entities.put(entityName, new DefaultEntityMetaData(entityName)); DefaultEntityMetaData md = entities.get(entityName); if (e.get("description") != null) md.setDescription(e.getString("description")); } } // re-iterate to map the mrefs/xref refEntity (or give error if not found) // TODO: consider also those in existing db int i = 1; for (Entity a : source.getRepositoryByEntityName("attributes")) { i++; if (a.get("refEntity") != null) { DefaultEntityMetaData em = entities.get(a.getString("entity")); DefaultAttributeMetaData am = (DefaultAttributeMetaData) em.getAttribute(a.getString("name")); if (entities.get(a.getString("refEntity")) == null) { throw new IllegalArgumentException( "attributes.refEntity error on line " + i + ": " + a.getString("refEntity") + " unknown"); } am.setRefEntity(entities.get(a.getString("refEntity"))); } } return entities; }