public static HashMap<Integer, List<Integer>> groupSimilarUMLSConsByCommonToken( EncodedEntityStructure umlsGroup, int srcId, AnnotationMapping am) { HashMap<Integer, List<Integer>> umlsGroups = new HashMap<Integer, List<Integer>>(); Map<Integer, Set<Integer>> conceptToEvidenceSet = new HashMap<Integer, Set<Integer>>(); for (int targetId : umlsGroup.getObjIds().keySet()) { long anid = CantorDecoder.code(srcId, targetId); Set<Integer> evidence = am.getEvidenceMap().get(anid); conceptToEvidenceSet.put(targetId, evidence); for (int tid : evidence) { List<Integer> group = umlsGroups.get(tid); if (group == null) { group = new ArrayList<Integer>(); umlsGroups.put(tid, group); } group.add(targetId); } } Map<Integer, Set<Integer>> conceptTosupersets = getSupersetConceptRelationships(conceptToEvidenceSet); for (Entry<Integer, List<Integer>> entry : umlsGroups.entrySet()) { for (int i = 0; i < entry.getValue().size(); i++) { int concept = entry.getValue().get(i); if (conceptTosupersets.containsKey(concept)) { boolean found = false; Set<Integer> supersetConcepts = conceptTosupersets.get(concept); for (int superConcept : entry.getValue()) { if (supersetConcepts.contains(superConcept)) { found = true; break; } } if (found) { entry.getValue().remove(i); i--; } } } } return umlsGroups; }
public static HashMap<Integer, List<Integer>> groupSimilarUMLSCons( EncodedEntityStructure umlsGroup, Set<GenericProperty> preRanAtts, int size, FormRepository gi) { EncodedAnnotationMapping set = new EncodedAnnotationMapping(); Set<Integer> srcs = new HashSet<Integer>(); srcs.add(umlsGroup.getStructureId()); // Int2FloatMap idfMap = TFIDFTokenWeightGenerator.getInstance().generateIDFValues(srcs, size); // Map<String,Object> externalObjects = new HashMap<String,Object>(); // externalObjects.put(TFIDFMatcher.IDF_MAP_SOURCE, idfMap); // externalObjects.put(TFIDFMatcher.IDF_MAP_TARGET, idfMap); // externalObjects.put(TFIDFMatcher.TFIDF_SOURCE_SEPARATED,false); long time = System.currentTimeMillis(); MatchOperator mop = new MatchOperator( RegisteredMatcher.TRIGRAM_MATCHER, AggregationFunction.MAX, preRanAtts, preRanAtts, 0.35f); // mop.setGlobalObjects(externalObjects); ExecutionTree tree = new ExecutionTree(); tree.addOperator(mop); try { set = gi.getMatchManager().matchEncoded(umlsGroup, umlsGroup, tree, null); log.debug("matched group"); } catch (MatchingExecutionException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } HashMap<Integer, Node> nodeMap = new HashMap<Integer, Node>(); boolean isChange = true; int counter = 0; HashSet<EntityAnnotation> toRemoveCorrs = new HashSet<EntityAnnotation>(); for (EntityAnnotation c : set.getAnnotations()) { if (c.getSrcId() == c.getTargetId()) { toRemoveCorrs.add(c); } } for (EntityAnnotation c : toRemoveCorrs) { set.removeAnnotation(c.getSrcId(), c.getTargetId()); } while (isChange && counter < 10000) { counter++; if (set.getNumberOfAnnotations() != 0) { for (EntityAnnotation cor : set.getAnnotations()) { if (cor.getSrcId() != cor.getTargetId()) { Node n = nodeMap.get(cor.getSrcId()); if (n == null) { n = new Node(); n.ownId = cor.getSrcId(); n.minId = n.ownId; nodeMap.put(n.ownId, n); } Node n2 = nodeMap.get(cor.getTargetId()); if (n2 == null) { n2 = new Node(); n2.ownId = cor.getTargetId(); n2.minId = cor.getTargetId(); nodeMap.put(n2.ownId, n2); } int min = Math.min(n.minId, n2.minId); if (n2.minId == min && n.minId == min) { isChange = false; } else { isChange = true; } n.minId = min; n2.minId = min; } } } else { for (int id : umlsGroup.getObjIds().keySet()) { Node n = new Node(); n.ownId = id; n.minId = n.ownId; nodeMap.put(id, n); } isChange = false; } } HashMap<Integer, List<Integer>> umlsGroups = new HashMap<Integer, List<Integer>>(); HashSet<Integer> notInGroup = new HashSet<Integer>(); for (int id : umlsGroup.getObjIds().keySet()) { notInGroup.add(id); } notInGroup.removeAll(nodeMap.keySet()); for (Entry<Integer, Node> e : nodeMap.entrySet()) { List<Integer> list = umlsGroups.get(e.getValue().minId); if (list == null) { list = new ArrayList<Integer>(); umlsGroups.put(e.getValue().minId, list); } list.add(e.getKey()); } for (Integer nig : notInGroup) { List<Integer> list = new ArrayList<Integer>(); umlsGroups.put(nig, list); list.add(nig); } return umlsGroups; }
public static HashMap<Integer, List<Integer>> groupSimilarUMLSConsByCliques( EncodedEntityStructure umlsGroup, Set<GenericProperty> preRanAtts, int size, FormRepository gi) { EncodedAnnotationMapping set = new EncodedAnnotationMapping(); Set<Integer> srcs = new HashSet<Integer>(); srcs.add(umlsGroup.getStructureId()); // Int2FloatMap idfMap = TFIDFTokenWeightGenerator.getInstance().generateIDFValues(srcs, size); // Map<String,Object> externalObjects = new HashMap<String,Object>(); // externalObjects.put(TFIDFMatcher.IDF_MAP_SOURCE, idfMap); // externalObjects.put(TFIDFMatcher.IDF_MAP_TARGET, idfMap); // externalObjects.put(TFIDFMatcher.TFIDF_SOURCE_SEPARATED,false); // long time = System.currentTimeMillis(); MatchOperator mop = new MatchOperator( RegisteredMatcher.TRIGRAM_MATCHER, AggregationFunction.MAX, preRanAtts, preRanAtts, 0.35f); // mop.setGlobalObjects(externalObjects); ExecutionTree tree = new ExecutionTree(); tree.addOperator(mop); try { set = gi.getMatchManager().matchEncoded(umlsGroup, umlsGroup, tree, null); } catch (MatchingExecutionException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } HashSet<EntityAnnotation> toRemoveCorrs = new HashSet<EntityAnnotation>(); Set<Integer> nodes = new HashSet<Integer>(); HashMap<Integer, List<Integer>> edges = new HashMap<Integer, List<Integer>>(); for (EntityAnnotation c : set.getAnnotations()) { if (c.getSrcId() == c.getTargetId()) { toRemoveCorrs.add(c); } else { nodes.add(c.getSrcId()); nodes.add(c.getTargetId()); List<Integer> srcEdges = edges.get(c.getSrcId()); List<Integer> targetEdges = edges.get(c.getTargetId()); if (srcEdges == null) { srcEdges = new ArrayList<Integer>(); edges.put(c.getSrcId(), srcEdges); } if (targetEdges == null) { targetEdges = new ArrayList<Integer>(); edges.put(c.getTargetId(), targetEdges); } srcEdges.add(c.getTargetId()); targetEdges.add(c.getSrcId()); } } CliqueIdentification clique = new CliqueIdentification(); Set<Set<Integer>> cliques = clique.simpleCluster(nodes, edges); HashMap<Integer, List<Integer>> umlsGroups = new HashMap<Integer, List<Integer>>(); HashSet<Integer> notInGroup = new HashSet<Integer>(); for (int id : umlsGroup.getObjIds().keySet()) { notInGroup.add(id); } notInGroup.removeAll(nodes); int id = 0; for (Set<Integer> c : cliques) { List<Integer> list = new ArrayList<Integer>(c); umlsGroups.put(id++, list); } for (Integer nig : notInGroup) { List<Integer> list = new ArrayList<Integer>(); umlsGroups.put(id++, list); list.add(nig); } return umlsGroups; }
public static HashMap<Integer, List<Integer>> groupSimilarUMLSConsByConnectedComponent( EncodedEntityStructure umlsGroup, Set<GenericProperty> preRanAtts, int size, FormRepository gi, float threshold) { EncodedAnnotationMapping set = new EncodedAnnotationMapping(); Set<Integer> srcs = new HashSet<Integer>(); srcs.add(umlsGroup.getStructureId()); MatchOperator mop = new MatchOperator( RegisteredMatcher.TRIGRAM_MATCHER, AggregationFunction.MAX, preRanAtts, preRanAtts, 0.4f); ExecutionTree tree = new ExecutionTree(); tree.addOperator(mop); try { log.debug(umlsGroup.getObjIds().size()); set = gi.getMatchManager().matchEncoded(umlsGroup, umlsGroup, tree, null); log.debug("matched group"); } catch (MatchingExecutionException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } HashSet<EntityAnnotation> toRemoveCorrs = new HashSet<EntityAnnotation>(); HashMap<Integer, Node> nodeMap = new HashMap<Integer, Node>(); boolean isChange = true; int counter = 0; for (EntityAnnotation c : set.getAnnotations()) { if (c.getSrcId() == c.getTargetId()) { toRemoveCorrs.add(c); } } for (EntityAnnotation c : toRemoveCorrs) { set.removeAnnotation(c.getSrcId(), c.getTargetId()); } while (isChange && counter < 10000) { counter++; if (set.getNumberOfAnnotations() != 0) { for (EntityAnnotation cor : set.getAnnotations()) { if (cor.getSrcId() != cor.getTargetId()) { Node n = nodeMap.get(cor.getSrcId()); if (n == null) { n = new Node(); n.ownId = cor.getSrcId(); n.minId = n.ownId; nodeMap.put(n.ownId, n); } Node n2 = nodeMap.get(cor.getTargetId()); if (n2 == null) { n2 = new Node(); n2.ownId = cor.getTargetId(); n2.minId = cor.getTargetId(); nodeMap.put(n2.ownId, n2); } int min = Math.min(n.minId, n2.minId); if (n2.minId == min && n.minId == min) { isChange = false; } else { isChange = true; } n.minId = min; n2.minId = min; } } } else { for (int id : umlsGroup.getObjIds().keySet()) { Node n = new Node(); n.ownId = id; n.minId = n.ownId; nodeMap.put(id, n); } isChange = false; } } HashMap<Integer, List<Integer>> umlsGroups = new HashMap<Integer, List<Integer>>(); HashSet<Integer> notInGroup = new HashSet<Integer>(); for (int id : umlsGroup.getObjIds().keySet()) { notInGroup.add(id); } notInGroup.removeAll(nodeMap.keySet()); for (Entry<Integer, Node> e : nodeMap.entrySet()) { List<Integer> list = umlsGroups.get(e.getValue().minId); if (list == null) { list = new ArrayList<Integer>(); umlsGroups.put(e.getValue().minId, list); } list.add(e.getKey()); } for (Integer nig : notInGroup) { List<Integer> list = new ArrayList<Integer>(); umlsGroups.put(nig, list); list.add(nig); } return umlsGroups; }
public static HashMap<Integer, List<Integer>> groupSimilarUMLSConsByCW( EncodedEntityStructure umlsGroup, Set<GenericProperty> preRanAtts, int size, FormRepository gi) { EncodedAnnotationMapping set = new EncodedAnnotationMapping(); Set<Integer> srcs = new HashSet<Integer>(); srcs.add(umlsGroup.getStructureId()); Int2FloatMap idfMap = TFIDFTokenWeightGenerator.getInstance().generateIDFValues(srcs, size); Map<String, Object> externalObjects = new HashMap<String, Object>(); externalObjects.put(TFIDFMatcher.IDF_MAP_SOURCE, idfMap); externalObjects.put(TFIDFMatcher.IDF_MAP_TARGET, idfMap); externalObjects.put(TFIDFMatcher.TFIDF_SOURCE_SEPARATED, false); long time = System.currentTimeMillis(); MatchOperator mop = new MatchOperator( RegisteredMatcher.TFIDF_MATCHER, AggregationFunction.MAX, preRanAtts, preRanAtts, 0.3f); mop.setGlobalObjects(externalObjects); ExecutionTree tree = new ExecutionTree(); tree.addOperator(mop); try { set = gi.getMatchManager().matchEncoded(umlsGroup, umlsGroup, tree, null); } catch (MatchingExecutionException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } Map<Integer, Integer> positionMapping = new HashMap<Integer, Integer>(); Map<Integer, Integer> revMapping = new HashMap<Integer, Integer>(); HashSet<EntityAnnotation> toRemoveCorrs = new HashSet<EntityAnnotation>(); int id = 0; for (EntityAnnotation c : set.getAnnotations()) { if (c.getSrcId() == c.getTargetId()) { toRemoveCorrs.add(c); } else { if (!positionMapping.containsKey(c.getSrcId())) { revMapping.put(id, c.getSrcId()); positionMapping.put(c.getSrcId(), id++); } if (!positionMapping.containsKey(c.getTargetId())) { revMapping.put(id, c.getTargetId()); positionMapping.put(c.getTargetId(), id++); } } } Matrix adMatrix = new ArrayMatrix(positionMapping.size(), positionMapping.size()); for (EntityAnnotation c : set.getAnnotations()) { if (positionMapping.containsKey(c.getSrcId()) && positionMapping.containsKey(c.getTargetId())) { int posX = positionMapping.get(c.getSrcId()); int posY = positionMapping.get(c.getTargetId()); if (c.getSrcId() == c.getTargetId()) { adMatrix.set(posX, posY, 0); } else { adMatrix.set(posX, posY, c.getSim()); } } } ChineseWhispers w = new ChineseWhispers(1000); Assignments assig = w.cluster(adMatrix, null); HashMap<Integer, List<Integer>> umlsGroups = new HashMap<Integer, List<Integer>>(); HashSet<Integer> notInGroup = new HashSet<Integer>(); for (int node : umlsGroup.getObjIds().keySet()) { notInGroup.add(node); } notInGroup.removeAll(positionMapping.keySet()); id = 0; for (Set<Integer> c : assig.clusters()) { List<Integer> list = new ArrayList<Integer>(); for (int i : c) { list.add(revMapping.get(i)); } umlsGroups.put(id++, list); } for (Integer nig : notInGroup) { List<Integer> list = new ArrayList<Integer>(); umlsGroups.put(id++, list); list.add(nig); } return umlsGroups; }