Java TokenTypeAttribute示例

示例#1

文件： HashtagTokenCombinerTest.java 项目： EricCen/commons

 private void verify(List<String> tokens, boolean[] isHashtag) {
   for (int i = 0; i < tokens.size(); i++) {
     assertTrue(stream.incrementToken());
     assertEquals(tokens.get(i), termAttr.getTermString());
     assertEquals(isHashtag[i], TokenType.HASHTAG.equals(typeAttr.getType()));
   }
   assertFalse(stream.incrementToken());
 }

示例#2

文件： EmoticonTokenCombinerTest.java 项目： nsanch/commons

 private void verify(List<String> tokens, TokenType... types) {
   for (int i = 0; i < tokens.size(); i++) {
     assertTrue(stream.incrementToken());
     assertEquals(tokens.get(i), termAttr.getTermString());
     assertEquals(types[i], typeAttr.getType());
   }
   assertFalse(stream.incrementToken());
 }