Ejemplo n.º 1
0
 /** simple test for longestOnly option */
 public void testLongestOnly() throws IOException {
   MockTokenizer tokenizer = whitespaceMockTokenizer("lucene is awesome");
   tokenizer.setEnableChecks(true);
   HunspellStemFilter filter = new HunspellStemFilter(tokenizer, dictionary, true, true);
   assertTokenStreamContents(
       filter, new String[] {"lucene", "is", "awesome"}, new int[] {1, 1, 1});
 }
Ejemplo n.º 2
0
  /** Simple test for KeywordAttribute */
  public void testKeywordAttribute() throws IOException {
    MockTokenizer tokenizer = whitespaceMockTokenizer("lucene is awesome");
    tokenizer.setEnableChecks(true);
    HunspellStemFilter filter = new HunspellStemFilter(tokenizer, dictionary);
    assertTokenStreamContents(
        filter, new String[] {"lucene", "lucen", "is", "awesome"}, new int[] {1, 0, 1, 1});

    // assert with keyword marker
    tokenizer = whitespaceMockTokenizer("lucene is awesome");
    CharArraySet set = new CharArraySet(Arrays.asList("Lucene"), true);
    filter = new HunspellStemFilter(new SetKeywordMarkerFilter(tokenizer, set), dictionary);
    assertTokenStreamContents(
        filter, new String[] {"lucene", "is", "awesome"}, new int[] {1, 1, 1});
  }