Beispiel #1
0
  private static void initGlossary(IGlossary glossary, IInflector inflector, TaxonGroup taxonGroup)
      throws IOException {
    OTOClient otoClient = new OTOClient("http://biosemantics.arizona.edu:8080/OTO");
    GlossaryDownload glossaryDownload = new GlossaryDownload();
    String glossaryVersion = "latest";
    otoClient.open();
    Future<GlossaryDownload> futureGlossaryDownload =
        otoClient.getGlossaryDownload(taxonGroup.getDisplayName(), glossaryVersion);

    try {
      glossaryDownload = futureGlossaryDownload.get();
    } catch (Exception e) {
      otoClient.close();
      e.printStackTrace();
    }
    otoClient.close();

    // add the syn set of the glossary
    HashSet<Term> gsyns = new HashSet<Term>();
    for (TermSynonym termSyn : glossaryDownload.getTermSynonyms()) {

      // if(termSyn.getCategory().compareTo("structure")==0){
      if (termSyn.getCategory().matches("structure|taxon_name|substance")) {
        // take care of singular and plural forms
        String syns = "";
        String synp = "";
        String terms = "";
        String termp = "";
        if (inflector.isPlural(
            termSyn
                .getSynonym()
                .replaceAll(
                    "_",
                    "-"))) { // must convert _ to -, as matching entity phrases will be converted
                             // from leg iii to leg-iii in the sentence.
          synp = termSyn.getSynonym().replaceAll("_", "-");
          syns = inflector.getSingular(synp);
        } else {
          syns = termSyn.getSynonym().replaceAll("_", "-");
          synp = inflector.getPlural(syns);
        }

        if (inflector.isPlural(termSyn.getTerm().replaceAll("_", "-"))) {
          termp = termSyn.getTerm().replaceAll("_", "-");
          terms = inflector.getSingular(termp);
        } else {
          terms = termSyn.getTerm().replaceAll("_", "-");
          termp = inflector.getPlural(terms);
        }
        glossary.addSynonym(syns, termSyn.getCategory(), terms);
        glossary.addSynonym(synp, termSyn.getCategory(), termp);
        gsyns.add(new Term(syns, termSyn.getCategory()));
        gsyns.add(new Term(synp, termSyn.getCategory()));
      } else {
        // glossary.addSynonym(termSyn.getSynonym().replaceAll("_",  "-"), "arrangement",
        // termSyn.getTerm());
        glossary.addSynonym(
            termSyn.getSynonym().replaceAll("_", "-"), termSyn.getCategory(), termSyn.getTerm());
        gsyns.add(new Term(termSyn.getSynonym().replaceAll("_", "-"), termSyn.getCategory()));
        // gsyns.add(new Term(termSyn.getSynonym().replaceAll("_",  "-"), "arrangement"));
      }
    }

    // the glossary, excluding gsyns
    for (TermCategory termCategory : glossaryDownload.getTermCategories()) {
      if (!gsyns.contains(
          new Term(termCategory.getTerm().replaceAll("_", "-"), termCategory.getCategory())))
        glossary.addEntry(
            termCategory.getTerm().replaceAll("_", "-"),
            termCategory.getCategory()); // primocane_foliage =>primocane-foliage Hong 3/2014
    }

    List<Synonym> synonyms = new LinkedList<Synonym>();
    CSVReader reader =
        new CSVReader(
            new FileReader(
                "C:\\Users\\rodenhausen\\Desktop\\test-enhance\\"
                    + "Gordon_complexity_term_review\\category_mainterm_synonymterm-task-Gordon_complexity.csv"));
    List<String[]> lines = reader.readAll();
    int i = 0;
    Set<String> hasSynonym = new HashSet<String>();
    for (String[] line : lines) {
      synonyms.add(new Synonym(String.valueOf(i), line[1], line[0], line[2]));
      hasSynonym.add(line[1]);
    }

    reader =
        new CSVReader(
            new FileReader(
                "C:\\Users\\rodenhausen\\Desktop\\test-enhance\\"
                    + "Gordon_complexity_term_review\\category_term-task-Gordon_complexity.csv"));
    lines = reader.readAll();
    List<Decision> decisions = new LinkedList<Decision>();
    i = 0;
    for (String[] line : lines) {
      decisions.add(
          new Decision(String.valueOf(i), line[1], line[0], hasSynonym.contains(line[1]), ""));
    }

    Download download = new Download(true, decisions, synonyms);

    // add syn set of term_category
    HashSet<Term> dsyns = new HashSet<Term>();
    if (download != null) {
      for (Synonym termSyn : download.getSynonyms()) {
        // Hong TODO need to add category info to synonym entry in OTOLite
        // if(termSyn.getCategory().compareTo("structure")==0){
        if (termSyn.getCategory().matches("structure|taxon_name|substance")) {
          // take care of singular and plural forms
          String syns = "";
          String synp = "";
          String terms = "";
          String termp = "";
          if (inflector.isPlural(termSyn.getSynonym().replaceAll("_", "-"))) {
            synp = termSyn.getSynonym().replaceAll("_", "-");
            syns = inflector.getSingular(synp);
          } else {
            syns = termSyn.getSynonym().replaceAll("_", "-");
            synp = inflector.getPlural(syns);
          }

          if (inflector.isPlural(termSyn.getTerm().replaceAll("_", "-"))) {
            termp = termSyn.getTerm().replaceAll("_", "-");
            terms = inflector.getSingular(termp);
          } else {
            terms = termSyn.getTerm().replaceAll("_", "-");
            termp = inflector.getPlural(terms);
          }
          // glossary.addSynonym(syns, termSyn.getCategory(), terms);
          // glossary.addSynonym(synp, termSyn.getCategory(), termp);
          // dsyns.add(new Term(syns, termSyn.getCategory());
          // dsyns.add(new Term(synp, termSyn.getCategory());
          glossary.addSynonym(syns, termSyn.getCategory(), terms);
          glossary.addSynonym(synp, termSyn.getCategory(), termp);
          dsyns.add(new Term(syns, termSyn.getCategory()));
          dsyns.add(new Term(synp, termSyn.getCategory()));
        } else { // forking_1 and forking are syns 5/5/14 hong test, shouldn't _1 have already been
                 // removed?
          glossary.addSynonym(
              termSyn.getSynonym().replaceAll("_", "-"), termSyn.getCategory(), termSyn.getTerm());
          dsyns.add(new Term(termSyn.getSynonym().replaceAll("_", "-"), termSyn.getCategory()));
        }
      }

      // term_category from OTO, excluding dsyns
      for (Decision decision : download.getDecisions()) {
        if (!dsyns.contains(
            new Term(
                decision.getTerm().replaceAll("_", "-"),
                decision.getCategory()))) // calyx_tube => calyx-tube
        glossary.addEntry(decision.getTerm().replaceAll("_", "-"), decision.getCategory());
      }
    }
  }
Beispiel #2
0
  public static void main(String[] args) throws IOException {
    /*for (String arg : args) {
      // option #1: By sentence.
      DocumentPreprocessor dp = new DocumentPreprocessor(arg);
      for (List<HasWord> sentence : dp) {
        System.out.println(sentence);
      }
      // option #2: By token
      PTBTokenizer<CoreLabel> ptbt = new PTBTokenizer<>(new FileReader(arg),
              new CoreLabelTokenFactory(), "");
      while (ptbt.hasNext()) {
        CoreLabel label = ptbt.next();
        System.out.println(label);
      }
    }*/

    ITokenizer tokenizer = new WhitespaceTokenizer(); /*ITokenizer() {		
			@Override
			public List<Token> tokenize(String text) {
				List<Token> result = new LinkedList<Token>();
				
				return null;
			}
		};*/
    TaxonGroup taxonGroup = TaxonGroup.PLANT;
    WordNetPOSKnowledgeBase wordNetPOSKnowledgeBase =
        new WordNetPOSKnowledgeBase(Configuration.wordNetDirectory, false);
    SingularPluralProvider singularPluralProvider = new SingularPluralProvider();
    IInflector inflector =
        new SomeInflector(
            wordNetPOSKnowledgeBase,
            singularPluralProvider.getSingulars(),
            singularPluralProvider.getPlurals());
    Map<String, String> renames = new HashMap<String, String>();
    renames.put("count", "quantity");
    renames.put("atypical_count", "atypical_quantity");
    renames.put("color", "coloration");
    IGlossary glossary = new InMemoryGlossary();
    initGlossary(glossary, inflector, taxonGroup);
    Set<String> lifeStyles = glossary.getWordsInCategory("life_style");
    lifeStyles.addAll(glossary.getWordsInCategory("growth_form"));
    Set<String> durations = glossary.getWordsInCategory("duration");
    String negWords = "no|not|never";
    String advModifiers = "at least|at first|at times";
    String stopWords =
        "a|about|above|across|after|along|also|although|amp|an|and|are|as|at|be|because|become|becomes|becoming|been|before|being|"
            + "beneath|between|beyond|but|by|ca|can|could|did|do|does|doing|done|for|from|had|has|have|hence|here|how|however|if|in|into|inside|inward|is|it|its|"
            + "may|might|more|most|near|of|off|on|onto|or|out|outside|outward|over|should|so|than|that|the|then|there|these|this|those|throughout|"
            + "to|toward|towards|up|upward|was|were|what|when|where|which|why|with|within|without|would";
    String units =
        "(?:(?:pm|cm|mm|dm|ft|m|meters|meter|micro_m|micro-m|microns|micron|unes|µm|μm|um|centimeters|centimeter|millimeters|millimeter|transdiameters|transdiameter)[23]?)"; // squared or cubed
    ICharacterKnowledgeBase characterKnowledgeBase =
        new GlossaryBasedCharacterKnowledgeBase(
            glossary, negWords, advModifiers, stopWords, units, inflector);
    Set<String> possessionTerms = getWordSet("with|has|have|having|possess|possessing|consist_of");

    CSVReader reader =
        new CSVReader(
            new FileReader(
                "C:\\Users\\rodenhausen\\Desktop\\test-enhance\\"
                    + "Gordon_complexity_term_review\\category_mainterm_synonymterm-task-Gordon_complexity.csv"));
    List<String[]> lines = reader.readAll();
    int i = 0;
    final Map<String, SynonymSet> synonymSetsMap = new HashMap<String, SynonymSet>();
    for (String[] line : lines) {
      String preferredTerm = line[1];
      String synonym = line[2];
      if (!synonymSetsMap.containsKey(preferredTerm))
        synonymSetsMap.put(preferredTerm, new SynonymSet(preferredTerm, new HashSet<String>()));
      synonymSetsMap.get(preferredTerm).getSynonyms().add(synonym);
    }

    /*List<KnowsSynonyms> hasBiologicalEntitySynonymsList = new LinkedList<KnowsSynonyms>();
    List<KnowsSynonyms> hasCharacterSynonymsList = new LinkedList<KnowsSynonyms>();
    hasBiologicalEntitySynonymsList.add(new CSVKnowsSynonyms() {
    	@Override
    	public Set<SynonymSet> getSynonyms(String term) {
    		Set<SynonymSet> result = new HashSet<SynonymSet>();
    		for(SynonymSet synonymSet : synonymSetsMap.values()) {
    			if(synonymSet.getPreferredTerm().equals(term) || synonymSet.getSynonyms().contains(term))
    				result.add(synonymSet);
    		}
    		if(result.isEmpty())
    			result.add(new SynonymSet(term, new HashSet<String>()));
    		return result;
    	}
    });*/

    Run run = new Run();
    // AbstractTransformer transformer = new RemoveSynonyms(hasBiologicalEntitySynonymsList,
    // hasBiologicalEntitySynonymsList);
    // AbstractTransformer transformer = new CreateRelationFromCharacterConstraint(new
    // KeyWordBasedKnowsCharacterConstraintType(wordNetPOSKnowledgeBase), inflector);
    // AbstractTransformer transformer = new MoveRelationToBiologicalEntityConstraint();//new
    // KeyWordBasedKnowsCharacterConstraintType(wordNetPOSKnowledgeBase), inflector);
    // AbstractTransformer transformer = new
    // MoveNegationOrAdverbBiologicalEntityConstraint(wordNetPOSKnowledgeBase);
    /*AbstractTransformer transformer = new RemoveNonSpecificBiologicalEntitiesByRelations(new KnowsPartOf() {
    	@Override
    	public boolean isPartOf(String part, String parent) {
    		if(part.equals("apex") && parent.equals("leaf")) {
    			return true;
    		}
    		if(part.equals("base") && parent.equals("fruit")) {
    			return true;
    		}
    		if(part.equals("base") && parent.equals("petal")) {
    			return true;
    		}
    		return false;
    	}
    }, tokenizer, new CollapseBiologicalEntityToName());*/

    // AbstractTransformer transformer1 = new MoveCharacterToStructureConstraint();
    // AbstractTransformer transformer2 = new ReplaceNegationCharacterByNegationOrAbsence();

    /*AbstractTransformer transformer = new MoveModifierCharactersToBiologicalEntityConstraint(tokenizer, new KnowsEntityExistence() {
    	@Override
    	public boolean isExistsEntity(String name) {
    		if(name.equals("red leaf")) {
    			return true;
    		}
    		return false;
    	}
    });*/

    CSVKnowsSynonyms csvKnowsSynonyms = new CSVKnowsSynonyms("synonyms.csv", inflector);
    RemoveNonSpecificBiologicalEntitiesByRelations transformer1 =
        new RemoveNonSpecificBiologicalEntitiesByRelations(
            new CSVKnowsPartOf("part-of.csv", csvKnowsSynonyms, inflector),
            csvKnowsSynonyms,
            tokenizer,
            new CollapseBiologicalEntityToName());
    RemoveNonSpecificBiologicalEntitiesByBackwardConnectors transformer2 =
        new RemoveNonSpecificBiologicalEntitiesByBackwardConnectors(
            new CSVKnowsPartOf("part-of.csv", csvKnowsSynonyms, inflector),
            csvKnowsSynonyms,
            tokenizer,
            new CollapseBiologicalEntityToName());
    RemoveNonSpecificBiologicalEntitiesByForwardConnectors transformer3 =
        new RemoveNonSpecificBiologicalEntitiesByForwardConnectors(
            new CSVKnowsPartOf("part-of.csv", csvKnowsSynonyms, inflector),
            csvKnowsSynonyms,
            tokenizer,
            new CollapseBiologicalEntityToName());
    RemoveNonSpecificBiologicalEntitiesByPassedParents transformer4 =
        new RemoveNonSpecificBiologicalEntitiesByPassedParents(
            new CSVKnowsPartOf("part-of.csv", csvKnowsSynonyms, inflector),
            csvKnowsSynonyms,
            tokenizer,
            new CollapseBiologicalEntityToName(),
            inflector);
    // RemoveNonSpecificBiologicalEntitiesByCollections removeByCollections = new
    // RemoveNonSpecificBiologicalEntitiesByCollections(
    //		new CSVKnowsPartOf(csvKnowsSynonyms, inflector), csvKnowsSynonyms, new
    // CSVKnowsClassHierarchy(inflector),
    //		tokenizer, new CollapseBiologicalEntityToName(), inflector);

    run.addTransformer(new SimpleRemoveSynonyms(csvKnowsSynonyms));
    run.addTransformer(transformer1);
    run.addTransformer(transformer2);
    run.addTransformer(transformer3);
    run.addTransformer(transformer4);
    // run.addTransformer(removeByCollections);

    // run.addTransformer(transformer1);
    // run.addTransformer(transformer2);
    /*
    AbstractTransformer transformer = new CollapseCharacterToValue();
    run.addTransformer(new RemoveOrphanRelations());
    run.addTransformer(new RemoveDuplicateValues());
    run.addTransformer(new CollapseBiologicalEntityToName());
    run.addTransformer(new CollapseCharacterToValue());
    run.addTransformer(new CollapseBiologicalEntities());
    run.addTransformer(new CollapseCharacters());
    */
    /*
    run.addTransformer(new SplitCompoundBiologicalEntity(inflector));
    run.addTransformer(new SplitCompoundBiologicalEntitiesCharacters(inflector));
    run.addTransformer(new RemoveUselessWholeOrganism());
    run.addTransformer(new RemoveUselessCharacterConstraint());
    run.addTransformer(new RenameCharacter(renames));
    run.addTransformer(new MoveCharacterToStructureConstraint());
    run.addTransformer(new MoveNegationCharacterToBiologicalEntityConstraint());
    run.addTransformer(new MoveNegationOrAdverbBiologicalEntityConstraint(wordNetPOSKnowledgeBase));
    run.addTransformer(new MoveCharactersToAlternativeParent());
    run.addTransformer(new ReplaceTaxonNameByWholeOrganism());
    run.addTransformer(new CreateOrPopulateWholeOrganism(lifeStyles, "growth_form"));
    run.addTransformer(new CreateOrPopulateWholeOrganism(durations, "duration"));
    run.addTransformer(new StandardizeQuantityPresence());
    run.addTransformer(new StandardizeCount());
    run.addTransformer(new SortBiologicalEntityNameWithDistanceCharacter());
    run.addTransformer(new OrderBiologicalEntityConstraint());
    run.addTransformer(new StandardizeStructureName(characterKnowledgeBase, possessionTerms));
    run.addTransformer(new StandardizeTerminology(characterKnowledgeBase));

    run.addTransformer(new RemoveOrphanRelations());
    run.addTransformer(new RemoveDuplicateValues());
    run.addTransformer(new CollapseBiologicalEntityToName());
    run.addTransformer(new CollapseCharacterToValue());
    run.addTransformer(new CollapseBiologicalEntities());
    run.addTransformer(new CollapseCharacters());


    */

    // run.run(new File("C:\\Users\\rodenhausen\\Desktop\\test-enhance\\selection_parsed2"), new
    // File("C:\\Users\\rodenhausen\\Desktop\\test-enhance\\selection_parsed2_out_" +
    // transformer.getClass().getSimpleName()));
    run.run(new File("in"), new File("out"));
  }