public static void main(String[] args) throws IOException { String serializedClassifier = "classifiers/english.all.3class.distsim.crf.ser.gz"; if (args.length > 0) { serializedClassifier = args[0]; } AbstractSequenceClassifier<CoreLabel> classifier = CRFClassifier.getClassifierNoExceptions(serializedClassifier); /* For either a file to annotate or for the hardcoded text example, this demo file shows two ways to process the output, for teaching purposes. For the file, it shows both how to run NER on a String and how to run it on a whole file. For the hard-coded String, it shows how to run it on a single sentence, and how to do this and produce an inline XML output format. */ if (args.length > 1) { String fileContents = IOUtils.slurpFile(args[1]); List<List<CoreLabel>> out = classifier.classify(fileContents); for (List<CoreLabel> sentence : out) { for (CoreLabel word : sentence) { System.out.print(word.word() + '/' + word.get(AnswerAnnotation.class) + ' '); } System.out.println(); } out = classifier.classifyFile(args[1]); for (List<CoreLabel> sentence : out) { for (CoreLabel word : sentence) { System.out.print(word.word() + '/' + word.get(AnswerAnnotation.class) + ' '); } System.out.println(); } } else { String s1 = "Good afternoon Rajat Raina, how are you today?"; String s2 = "I go to school at Stanford University, which is located in California."; System.out.println(classifier.classifyToString(s1)); System.out.println(classifier.classifyWithInlineXML(s2)); System.out.println(classifier.classifyToString(s2, "xml", true)); int i = 0; for (List<CoreLabel> lcl : classifier.classify(s2)) { for (CoreLabel cl : lcl) { System.out.println(i++ + ":"); System.out.println(cl); } } } }
// Method to process the named entities public void processNamedEntity() { String serializedClassifier = "classifiers/english.all.3class.distsim.crf.ser.gz"; // 3 class (PERSON, LOCATION and // ORGANISATION) classifier // String serializedClassifier = "classifiers/english.conll.4class.distsim.crf.ser.gz"; AbstractSequenceClassifier<CoreLabel> classifier = CRFClassifier.getClassifierNoExceptions(serializedClassifier); FileHandler fw = new FileHandler(); org.apache.log4j.PropertyConfigurator.configure(fw.filenameLog4JConfig); Path path = Paths.get(fw.filename); // path for input file try (BufferedReader reader = Files.newBufferedReader(path, Charset.defaultCharset())) { fw.setFile(fw.fileResultsWithNER); // delete contents to file before starting the process String line = null; while ((line = reader.readLine()) != null) { String classifiedData = classifier.classifyWithInlineXML(line); fw.writeToFile("", classifiedData, fw.fileResultsWithNER, true); } } catch (IOException e) { e.printStackTrace(); } }
public static Set<String> analyzeThisFile(String path) { String serializedClassifier = "classifiers/english.conll.4class.distsim.crf.ser.gz"; Set<String> annotationsNE = new HashSet<String>(); try { BufferedReader br = new BufferedReader(new FileReader(path)); AbstractSequenceClassifier<CoreLabel> classifier = CRFClassifier.getClassifier(serializedClassifier); String line = ""; while ((line = br.readLine()) != null) { List<Triple<String, Integer, Integer>> triples = classifier.classifyToCharacterOffsets(line); for (Triple<String, Integer, Integer> trip : triples) { annotationsNE.add(line.substring(trip.second(), trip.third())); } } System.out.println("Named Entity TROVATE"); for (String ne : annotationsNE) System.out.println(ne); } catch (FileNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ClassCastException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ClassNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } return annotationsNE; }
public static void main(String[] args) throws Exception { // String serializedClassifier = "classifiers/english.all.3class.distsim.crf.ser.gz"; String serializedClassifier = "classifiers/english.muc.7class.distsim.crf.ser.gz"; if (args.length > 0) { serializedClassifier = args[0]; } AbstractSequenceClassifier<CoreLabel> classifier = CRFClassifier.getClassifier(serializedClassifier); /* For either a file to annotate or for the hardcoded text example, this demo file shows several ways to process the input, for teaching purposes. */ if (args.length > 1) { /* For the file, it shows (1) how to run NER on a String, (2) how to get the entities in the String with character offsets, and (3) how to run NER on a whole file (without loading it into a String). */ String fileContents = IOUtils.slurpFile(args[1]); List<List<CoreLabel>> out = classifier.classify(fileContents); for (List<CoreLabel> sentence : out) { for (CoreLabel word : sentence) { System.out.print( word.word() + '/' + word.get(CoreAnnotations.AnswerAnnotation.class) + ' '); } System.out.println(); } System.out.println("---"); out = classifier.classifyFile(args[1]); for (List<CoreLabel> sentence : out) { for (CoreLabel word : sentence) { System.out.print( word.word() + '/' + word.get(CoreAnnotations.AnswerAnnotation.class) + ' '); } System.out.println(); } System.out.println("---"); List<Triple<String, Integer, Integer>> list = classifier.classifyToCharacterOffsets(fileContents); for (Triple<String, Integer, Integer> item : list) { // print entity/or non-entity - their nearby tokens System.out.println( item.first() + ": " + fileContents.substring(item.second(), item.third())); } System.out.println("---"); System.out.println("Ten best entity labelings"); DocumentReaderAndWriter<CoreLabel> readerAndWriter = classifier.makePlainTextReaderAndWriter(); classifier.classifyAndWriteAnswersKBest(args[1], 10, readerAndWriter); System.out.println("---"); System.out.println("Per-token marginalized probabilities"); classifier.printProbs(args[1], readerAndWriter); // -- This code prints out the first order (token pair) clique probabilities. // -- But that output is a bit overwhelming, so we leave it commented out by default. // System.out.println("---"); // System.out.println("First Order Clique Probabilities"); // ((CRFClassifier) classifier).printFirstOrderProbs(args[1], readerAndWriter); } else { /* For the hard-coded String, it shows how to run it on a single sentence, and how to do this and produce several formats, including slash tags and an inline XML output format. It also shows the full contents of the {@code CoreLabel}s that are constructed by the classifier. And it shows getting out the probabilities of different assignments and an n-best list of classifications with probabilities. */ String[] example = { "Good afternoon Rajat Raina, how are you today? I go to Washington DC on September 19. And Tomorrow.", "I go to school at Stanford University, which is located in California." }; for (String str : example) { System.out.println(classifier.classifyToString(str)); } System.out.println("---"); // ***sentence-by-sentence for (String str : example) { // This one puts in spaces and newlines between tokens, so just print not println. System.out.print(classifier.classifyToString(str, "slashTags", false)); } System.out.println("---"); // ***print: entities + Classes + remaining text in the text for (String str : example) { // This one is best for dealing with the output as a TSV (tab-separated column) file. // The first column gives entities, the second their classes, and the third the remaining // text in a document System.out.print(classifier.classifyToString(str, "tabbedEntities", false)); } System.out.println("---"); for (String str : example) { System.out.println(classifier.classifyWithInlineXML(str)); } System.out.println("---"); for (String str : example) { System.out.println(classifier.classifyToString(str, "xml", true)); } System.out.println("---"); for (String str : example) { System.out.print(classifier.classifyToString(str, "tsv", false)); } System.out.println("---"); // This gets out entities with character offsets System.out.print("character offsets"); int j = 0; for (String str : example) { j++; List<Triple<String, Integer, Integer>> triples = classifier.classifyToCharacterOffsets(str); for (Triple<String, Integer, Integer> trip : triples) { System.out.printf( "%s over character offsets [%d, %d) in sentence %d.%n", trip.first(), trip.second(), trip.third, j); } } System.out.println("---"); // This prints out all the details of what is stored for each token int i = 0; for (String str : example) { for (List<CoreLabel> lcl : classifier.classify(str)) { for (CoreLabel cl : lcl) { System.out.print(i++ + ": "); System.out.println(cl.toShorterString()); } } } System.out.println("---"); } }