@RequestMapping(method = RequestMethod.GET)
public String init(
@RequestParam(value = "selectedDataSet", required = false) String selectedDataSetId,
Model model) {
Iterable<DataSet> allDataSets = dataService.findAll(DataSet.ENTITY_NAME, new QueryImpl());
List<DataSet> dataSets = new ArrayList<DataSet>();
for (DataSet dataSet : allDataSets) {
if (selectedDataSetId == null) selectedDataSetId = dataSet.getId().toString();
if (!dataSet.getProtocolUsed().getIdentifier().equals(PROTOCOL_IDENTIFIER))
dataSets.add(dataSet);
}
model.addAttribute("dataSets", dataSets);
List<String> mappedDataSets = new ArrayList<String>();
if (selectedDataSetId != null) {
model.addAttribute("selectedDataSet", selectedDataSetId);
Iterable<DataSet> it =
dataService.findAll(
DataSet.ENTITY_NAME, new QueryImpl().like(DataSet.IDENTIFIER, selectedDataSetId));
for (DataSet dataSet : it) {
if (dataSet
.getIdentifier()
.startsWith(SecurityUtils.getCurrentUsername() + "-" + selectedDataSetId)) {
String[] dataSetIds = dataSet.getIdentifier().toString().split("-");
if (dataSetIds.length > 1) mappedDataSets.add(dataSetIds[2]);
}
}
}
model.addAttribute("mappedDataSets", mappedDataSets);
return "EvaluationPlugin";
}
private List<? extends Authority> getUserPermissions(String userId, String authorityPrefix) {
MolgenisUser molgenisUser =
dataService.findOne(MolgenisUser.ENTITY_NAME, userId, MolgenisUser.class);
if (molgenisUser == null) throw new RuntimeException("unknown user id [" + userId + "]");
List<Authority> userPermissions = getUserPermissions(molgenisUser, authorityPrefix);
Iterable<MolgenisGroupMember> groupMembersIt =
dataService.findAll(
MolgenisGroupMember.ENTITY_NAME,
new QueryImpl().eq(MolgenisGroupMember.MOLGENISUSER, molgenisUser),
MolgenisGroupMember.class);
if (groupMembersIt == null) {
return Collections.emptyList();
}
List<MolgenisGroupMember> groupMembers = Lists.newArrayList(groupMembersIt);
if (groupMembers != null && !groupMembers.isEmpty()) {
List<MolgenisGroup> molgenisGroups =
Lists.transform(
groupMembers,
new Function<MolgenisGroupMember, MolgenisGroup>() {
@Override
public MolgenisGroup apply(MolgenisGroupMember molgenisGroupMember) {
return molgenisGroupMember.getMolgenisGroup();
}
});
List<Authority> groupAuthorities = getGroupPermissions(molgenisGroups, authorityPrefix);
if (groupAuthorities != null && !groupAuthorities.isEmpty())
userPermissions.addAll(groupAuthorities);
}
return userPermissions;
}
@Override
public void onApplicationEvent(ContextRefreshedEvent event) {
// Run it as System
// Remember the original context
SecurityContext origCtx = SecurityContextHolder.getContext();
try {
// Set a SystemSecurityToken
SecurityContextHolder.setContext(SecurityContextHolder.createEmptyContext());
SecurityContextHolder.getContext().setAuthentication(new SystemSecurityToken());
Query q =
new QueryImpl()
.eq(ObservableFeature.DATATYPE, FieldTypeEnum.CATEGORICAL.toString().toLowerCase());
for (ObservableFeature categoricalFeature :
dataService.findAll(ObservableFeature.ENTITY_NAME, q, ObservableFeature.class)) {
OmxLookupTableRepository repo =
new OmxLookupTableRepository(
dataService, categoricalFeature.getIdentifier(), queryResolver);
dataService.addRepository(repo);
}
} finally {
// Set the original context back when method is finished
SecurityContextHolder.setContext(origCtx);
}
}
// RULE: Feature can only belong to one Protocol in a DataSet.(see issue #1136)
private void checkFeatureCanOnlyBelongToOneProtocolForOneDataSet() {
// RULE: Feature can only belong to one Protocol in a DataSet. Check it (see issue #1136)
Iterable<DataSet> dataSets = dataService.findAll(DataSet.ENTITY_NAME, DataSet.class);
for (DataSet dataSet : dataSets) {
List<Protocol> dataSetProtocols =
ProtocolUtils.getProtocolDescendants(dataSet.getProtocolUsed(), true);
for (Protocol protocol : dataSetProtocols) {
for (ObservableFeature feature : protocol.getFeatures()) {
for (Protocol p : dataSetProtocols) {
if (!p.equals(protocol) && p.getFeatures().contains(feature)) {
String message =
String.format(
"An ObservableFeature can only belong to one Protocol but feature '%s' belongs to both '%s' and '%s'",
feature.getIdentifier(), p.getIdentifier(), protocol.getIdentifier());
throw new MolgenisValidationException(
Sets.newHashSet(
new ConstraintViolation(
message, feature.getIdentifier(), feature, null, null, 0)));
}
}
}
}
}
}
@Test
public void testFindAll() {
Query q = new QueryImpl();
q.eq(OWNER, "flup");
when(dataService.findAll(ENTITY_NAME)).thenReturn(Stream.of(mappingProjectEntity));
when(userService.getUser("flup")).thenReturn(owner);
when(mappingTargetRepository.toMappingTargets(mappingTargetEntities))
.thenReturn(asList(mappingTarget1, mappingTarget2));
List<MappingProject> result = mappingProjectRepositoryImpl.getAllMappingProjects();
mappingProject.setIdentifier("mappingProjectID");
assertEquals(result, asList(mappingProject));
}
private void registerNewIndexActionForDirtyJobs(IndexJobExecution indexJobExecution) {
String id = indexJobExecution.getIndexActionJobID();
dataService
.findAll(
IndexActionMetaData.INDEX_ACTION,
new QueryImpl<IndexAction>().eq(IndexActionMetaData.INDEX_ACTION_GROUP_ATTR, id),
IndexAction.class)
.forEach(
action ->
indexActionRegisterService.register(
action.getEntityFullName(), action.getEntityId()));
dataService.delete(IndexJobExecutionMeta.INDEX_JOB_EXECUTION, indexJobExecution);
}
@Test
public void testGenerateRules() {
EntityType targetRefEntityType = createCategoricalRefEntityType("HOP_HYPERTENSION");
Entity targetEntity1 =
new DynamicEntity(
targetRefEntityType, of("code", 0, "label", "Never had high blood pressure "));
Entity targetEntity2 =
new DynamicEntity(
targetRefEntityType, of("code", 1, "label", "Ever had high blood pressure "));
Entity targetEntity3 =
new DynamicEntity(targetRefEntityType, of("code", 9, "label", "Missing"));
Mockito.when(dataService.findAll(targetRefEntityType.getName()))
.thenReturn(Stream.of(targetEntity1, targetEntity2, targetEntity3));
targetAttribute =
attrMetaFactory.create().setName("History of Hypertension").setDataType(CATEGORICAL);
targetAttribute.setRefEntity(targetRefEntityType);
EntityType sourceRefEntityType = createCategoricalRefEntityType("High_blood_pressure_ref");
Entity sourceEntity1 = new DynamicEntity(targetRefEntityType, of("code", 1, "label", "yes"));
Entity sourceEntity2 = new DynamicEntity(targetRefEntityType, of("code", 2, "label", "no"));
Entity sourceEntity3 =
new DynamicEntity(targetRefEntityType, of("code", 3, "label", "I do not know"));
Mockito.when(dataService.findAll(sourceRefEntityType.getName()))
.thenReturn(Stream.of(sourceEntity1, sourceEntity2, sourceEntity3));
sourceAttribute =
attrMetaFactory.create().setName("High_blood_pressure").setDataType(CATEGORICAL);
sourceAttribute.setRefEntity(sourceRefEntityType);
String generatedAlgorithm =
categoryAlgorithmGenerator.generate(
targetAttribute, singletonList(sourceAttribute), targetEntityType, sourceEntityType);
String expectedAlgorithm =
"$('High_blood_pressure').map({\"1\":\"1\",\"2\":\"0\",\"3\":\"9\"}, null, null).value();";
Assert.assertEquals(generatedAlgorithm, expectedAlgorithm);
}
@RequestMapping(
value = "/aggregate",
method = RequestMethod.POST,
produces = "application/json",
consumes = "application/json")
@ResponseBody
public AggregateResponse aggregate(@Valid @RequestBody AggregateRequest request) {
// TODO create utility class to extract info from entity/attribute uris
String[] attributeUriTokens = request.getAttributeUri().split("/");
String entityName = attributeUriTokens[3];
String attributeName = attributeUriTokens[5];
QueryImpl q = request.getQ() != null ? new QueryImpl(request.getQ()) : new QueryImpl();
EntityMetaData entityMeta = dataService.getEntityMetaData(entityName);
AttributeMetaData attributeMeta = entityMeta.getAttribute(attributeName);
FieldTypeEnum dataType = attributeMeta.getDataType().getEnumType();
if (dataType != FieldTypeEnum.BOOL && dataType != FieldTypeEnum.CATEGORICAL) {
throw new RuntimeException("Unsupported data type " + dataType);
}
EntityMetaData refEntityMeta = null;
String refAttributeName = null;
if (dataType == FieldTypeEnum.CATEGORICAL) {
refEntityMeta = attributeMeta.getRefEntity();
refAttributeName = refEntityMeta.getLabelAttribute().getName();
}
Map<String, Integer> aggregateMap = new HashMap<String, Integer>();
for (Entity entity : dataService.findAll(entityName, q)) {
String val;
switch (dataType) {
case BOOL:
val = entity.getString(attributeName);
break;
case CATEGORICAL:
Entity refEntity = (Entity) entity.get(attributeName);
val = refEntity.getString(refAttributeName);
break;
default:
throw new RuntimeException("Unsupported data type " + dataType);
}
Integer count = aggregateMap.get(val);
if (count == null) aggregateMap.put(val, 1);
else aggregateMap.put(val, count + 1);
}
return new AggregateResponse(aggregateMap);
}
@Override
public Iterable<Entity> findAll(String name, Query q) {
if (resources.containsKey(name)) {
// Don't check isAvailable() yet, it's too costly.
try {
return resources.get(name).findAll(q);
} catch (Exception ex) {
// something went wrong, find out what is the cause
if (resources.get(name).isAvailable()) {
LOG.error("Error querying Resource {}.", name);
throw ex;
}
LOG.warn("Resource {} is unavailable, trying dataService instead.", name);
}
}
return dataService.findAll(name, q);
}
@Override
public Iterable<CatalogMeta> getCatalogs() {
Iterable<Protocol> protocols =
dataService.findAll(
Protocol.ENTITY_NAME, new QueryImpl().eq(Protocol.ROOT, true), Protocol.class);
return Iterables.transform(
protocols,
new Function<Protocol, CatalogMeta>() {
@Override
public CatalogMeta apply(Protocol protocol) {
CatalogMeta catalogMeta =
new CatalogMeta(protocol.getId().toString(), protocol.getName());
catalogMeta.setDescription(protocol.getDescription());
return catalogMeta;
}
});
}
@RequestMapping(value = "/download", method = POST)
public void download(
@RequestParam("dataRequest") String dataRequestStr, HttpServletResponse response)
throws IOException {
// Workaround because binding with @RequestBody is not possible:
// http://stackoverflow.com/a/9970672
dataRequestStr = URLDecoder.decode(dataRequestStr, "UTF-8");
logger.info("Download request: [" + dataRequestStr + "]");
DataRequest dataRequest =
new GsonHttpMessageConverter().getGson().fromJson(dataRequestStr, DataRequest.class);
String entityName = dataRequest.getEntityName();
EntityMetaData entityMetaData = dataService.getEntityMetaData(entityName);
final Set<String> attributes = new HashSet<String>(dataRequest.getAttributeNames());
String fileName =
entityName + '_' + new SimpleDateFormat("yyyy-MM-dd").format(new Date()) + ".csv";
response.setContentType("text/csv");
response.addHeader("Content-Disposition", "attachment; filename=" + fileName);
CsvWriter csvWriter = new CsvWriter(response.getOutputStream());
try {
csvWriter.writeAttributeNames(
Iterables.transform(
Iterables.filter(
entityMetaData.getAtomicAttributes(),
new Predicate<AttributeMetaData>() {
@Override
public boolean apply(AttributeMetaData attributeMetaData) {
return attributes.contains(attributeMetaData.getName());
}
}),
new Function<AttributeMetaData, String>() {
@Override
public String apply(AttributeMetaData attributeMetaData) {
return attributeMetaData.getName();
}
}));
csvWriter.add(dataService.findAll(entityName, dataRequest.getQuery()));
} finally {
csvWriter.close();
}
}
private List<Authority> getGroupPermissions(
List<MolgenisGroup> molgenisGroups, final String authorityPrefix) {
Iterable<GroupAuthority> authorities =
dataService.findAll(
GroupAuthority.ENTITY_NAME,
new QueryImpl().in(GroupAuthority.MOLGENISGROUP, molgenisGroups),
GroupAuthority.class);
return Lists.<Authority>newArrayList(
Iterables.filter(
authorities,
new Predicate<Authority>() {
@Override
public boolean apply(Authority authority) {
return authorityPrefix != null
? authority.getRole().startsWith(authorityPrefix)
: true;
}
}));
}
public void bootstrap() {
if (!searchService.hasMapping(AttributeMetadata.ATTRIBUTE_META_DATA)) {
LOG.debug(
"No index for Attribute found, asuming missing index, schedule (re)index for all entities");
metaDataService
.getRepositories()
.forEach(repo -> indexActionRegisterService.register(repo.getName(), null));
LOG.debug("Done scheduling (re)index jobs for all entities");
} else {
LOG.debug("Index for Attribute found, index is present, no (re)index needed");
List<IndexJobExecution> failedIndexJobs =
dataService
.findAll(
IndexJobExecutionMeta.INDEX_JOB_EXECUTION,
new QueryImpl<IndexJobExecution>().eq(JobExecutionMetaData.STATUS, FAILED),
IndexJobExecution.class)
.collect(Collectors.toList());
failedIndexJobs.forEach(job -> registerNewIndexActionForDirtyJobs(job));
}
}
private List<Authority> getUserPermissions(
MolgenisUser molgenisUser, final String authorityPrefix) {
Iterable<UserAuthority> authorities =
dataService.findAll(
UserAuthority.ENTITY_NAME,
new QueryImpl().eq(UserAuthority.MOLGENISUSER, molgenisUser),
UserAuthority.class);
return Lists.<Authority>newArrayList(
Iterables.filter(
authorities,
new Predicate<Authority>() {
@Override
public boolean apply(Authority authority) {
return authorityPrefix != null
? authority.getRole().startsWith(authorityPrefix)
: true;
}
}));
}
private void initReferenceValidation(ValidationResource validationResource) {
// get reference attrs
List<AttributeMetaData> refAttrs =
StreamSupport.stream(getEntityMetaData().getAtomicAttributes().spliterator(), false)
.filter(
attr ->
(attr.getDataType() instanceof XrefField
|| attr.getDataType() instanceof MrefField)
&& attr.getExpression() == null)
.collect(Collectors.toList());
// get referenced entity ids
if (!refAttrs.isEmpty()) {
Map<String, HugeSet<Object>> refEntitiesIds = new HashMap<>();
refAttrs.forEach(
refAttr -> {
EntityMetaData refEntityMeta = refAttr.getRefEntity();
String refEntityName = refEntityMeta.getName();
HugeSet<Object> refEntityIds = refEntitiesIds.get(refEntityName);
if (refEntityIds == null) {
refEntityIds = new HugeSet<>();
refEntitiesIds.put(refEntityName, refEntityIds);
Query q =
new QueryImpl()
.fetch(new Fetch().field(refEntityMeta.getIdAttribute().getName()));
for (Iterator<Entity> it = dataService.findAll(refEntityName, q).iterator();
it.hasNext(); ) {
refEntityIds.add(it.next().getIdValue());
}
}
});
validationResource.setRefEntitiesIds(refEntitiesIds);
}
validationResource.setRefAttrs(refAttrs);
}
@BeforeMethod
public void init() {
dataService = Mockito.mock(DataService.class);
categoryAlgorithmGenerator = new OneToOneCategoryAlgorithmGenerator(dataService);
EntityType targetRefEntityType = createCategoricalRefEntityType("POTATO_REF");
Entity targetEntity1 =
new DynamicEntity(targetRefEntityType, of("code", 1, "label", "Almost daily + daily"));
Entity targetEntity2 =
new DynamicEntity(targetRefEntityType, of("code", 2, "label", "Several times a week"));
Entity targetEntity3 =
new DynamicEntity(targetRefEntityType, of("code", 3, "label", "About once a week"));
Entity targetEntity4 =
new DynamicEntity(
targetRefEntityType, of("code", 4, "label", "Never + fewer than once a week"));
Entity targetEntity5 =
new DynamicEntity(targetRefEntityType, of("code", 9, "label", "missing"));
targetAttribute =
attrMetaFactory
.create()
.setName("Current Consumption Frequency of Potatoes")
.setDataType(CATEGORICAL);
targetAttribute.setRefEntity(targetRefEntityType);
Mockito.when(dataService.findAll(targetRefEntityType.getName()))
.thenReturn(
Stream.of(targetEntity1, targetEntity2, targetEntity3, targetEntity4, targetEntity5));
targetEntityType = entityTypeFactory.create("target");
targetEntityType.addAttribute(targetAttribute);
EntityType sourceRefEntityType = createCategoricalRefEntityType("LifeLines_POTATO_REF");
Entity sourceEntity1 =
new DynamicEntity(targetRefEntityType, of("code", 1, "label", "Not this month"));
Entity sourceEntity2 =
new DynamicEntity(targetRefEntityType, of("code", 2, "label", "1 day per month"));
Entity sourceEntity3 =
new DynamicEntity(targetRefEntityType, of("code", 3, "label", "2-3 days per month"));
Entity sourceEntity4 =
new DynamicEntity(targetRefEntityType, of("code", 4, "label", "1 day per week"));
Entity sourceEntity5 =
new DynamicEntity(targetRefEntityType, of("code", 5, "label", "2-3 days per week"));
Entity sourceEntity6 =
new DynamicEntity(targetRefEntityType, of("code", 6, "label", "4-5 days per week"));
Entity sourceEntity7 =
new DynamicEntity(targetRefEntityType, of("code", 7, "label", "6-7 days per week"));
Entity sourceEntity8 =
new DynamicEntity(targetRefEntityType, of("code", 8, "label", "9 days per week"));
sourceAttribute = attrMetaFactory.create().setName("MESHED_POTATO").setDataType(CATEGORICAL);
sourceAttribute.setLabel(
"How often did you eat boiled or mashed potatoes (also in stew) in the past month? Baked potatoes are asked later");
sourceAttribute.setRefEntity(sourceRefEntityType);
Mockito.when(dataService.findAll(sourceRefEntityType.getName()))
.thenReturn(
Stream.of(
sourceEntity1,
sourceEntity2,
sourceEntity3,
sourceEntity4,
sourceEntity5,
sourceEntity6,
sourceEntity7,
sourceEntity8));
sourceEntityType = entityTypeFactory.create("source");
sourceEntityType.addAttributes(Lists.newArrayList(sourceAttribute));
}
@Override
@PreAuthorize("hasAnyRole('ROLE_SU')")
@Transactional(readOnly = true)
public List<MolgenisGroup> getGroups() {
return Lists.newArrayList(dataService.findAll(MolgenisGroup.ENTITY_NAME, MolgenisGroup.class));
}
@Test
public void testFindAttributes() {
DefaultEntityMetaData sourceEntityMetaData = new DefaultEntityMetaData("sourceEntityMetaData");
EntityMetaData targetEntityMetaData = new DefaultEntityMetaData("targetEntityMetaData");
DefaultAttributeMetaData targetAttribute = new DefaultAttributeMetaData("targetAttribute");
// Mock the id's of the attribute entities that should be searched
List<String> attributeIdentifiers = Arrays.asList("1", "2");
when(semanticSearchServiceHelper.getAttributeIdentifiers(sourceEntityMetaData))
.thenReturn(attributeIdentifiers);
// Mock the createDisMaxQueryRule method
List<QueryRule> rules = new ArrayList<QueryRule>();
QueryRule targetQueryRuleLabel =
new QueryRule(AttributeMetaDataMetaData.LABEL, Operator.FUZZY_MATCH, "height");
rules.add(targetQueryRuleLabel);
QueryRule targetQueryRuleOntologyTermTag =
new QueryRule(AttributeMetaDataMetaData.LABEL, Operator.FUZZY_MATCH, "standing height");
rules.add(targetQueryRuleOntologyTermTag);
QueryRule targetQueryRuleOntologyTermTagSyn =
new QueryRule(AttributeMetaDataMetaData.LABEL, Operator.FUZZY_MATCH, "length");
rules.add(targetQueryRuleOntologyTermTagSyn);
QueryRule disMaxQueryRule = new QueryRule(rules);
disMaxQueryRule.setOperator(Operator.DIS_MAX);
when(semanticSearchServiceHelper.createDisMaxQueryRuleForAttribute(
targetEntityMetaData, targetAttribute))
.thenReturn(disMaxQueryRule);
MapEntity entity1 =
new MapEntity(
ImmutableMap.of(
AttributeMetaDataMetaData.NAME,
"height_0",
AttributeMetaDataMetaData.LABEL,
"height",
AttributeMetaDataMetaData.DESCRIPTION,
"this is a height measurement in m!"));
List<Entity> attributeMetaDataEntities = Arrays.<Entity>asList(entity1);
List<QueryRule> disMaxQueryRules =
Lists.newArrayList(
new QueryRule(AttributeMetaDataMetaData.IDENTIFIER, Operator.IN, attributeIdentifiers),
new QueryRule(Operator.AND),
disMaxQueryRule);
AttributeMetaData attributeHeight = new DefaultAttributeMetaData("height_0");
AttributeMetaData attributeWeight = new DefaultAttributeMetaData("weight_0");
sourceEntityMetaData.addAttributeMetaData(attributeHeight);
sourceEntityMetaData.addAttributeMetaData(attributeWeight);
// Case 1
when(dataService.findAll(
AttributeMetaDataMetaData.ENTITY_NAME, new QueryImpl(disMaxQueryRules)))
.thenReturn(attributeMetaDataEntities);
Iterable<AttributeMetaData> termsActual1 =
semanticSearchService.findAttributes(
sourceEntityMetaData, targetEntityMetaData, targetAttribute);
Iterable<AttributeMetaData> termsExpected1 = Arrays.<AttributeMetaData>asList(attributeHeight);
assertEquals(termsActual1, termsExpected1);
// Case 2
when(dataService.findAll(
AttributeMetaDataMetaData.ENTITY_NAME, new QueryImpl(disMaxQueryRules)))
.thenReturn(Arrays.<Entity>asList());
Iterable<AttributeMetaData> termsActual2 =
semanticSearchService.findAttributes(
sourceEntityMetaData, targetEntityMetaData, targetAttribute);
Iterable<AttributeMetaData> termsExpected2 = Arrays.<AttributeMetaData>asList();
assertEquals(termsActual2, termsExpected2);
Mockito.reset(ontologyService);
attribute.setDescription("Standing height (Ångstrøm)");
when(ontologyService.findOntologyTerms(
ontologies, ImmutableSet.of("standing", "height", "ångstrøm"), 100))
.thenReturn(ontologyTerms);
Hit<OntologyTerm> result = semanticSearchService.findTags(attribute, ontologies);
assertEquals(result, Hit.<OntologyTerm>create(standingHeight, 0.76471f));
}