public TokenIndex getTokenIndex(String str) {
TokenIndex ret = new TokenIndex();
try {
Tokenizer tokenizer =
new JapaneseTokenizer(
new StringReader(str),
null,
true,
org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode.SEARCH);
TokenStream stream = new JapaneseBaseFormFilter(tokenizer);
// stream = new JapanesePartOfSpeechStopFilter(true, stream, stoptags);
stream = new CJKWidthFilter(stream);
// stream = new StopFilter(matchVersion, stream, stopwords);
stream = new JapaneseKatakanaStemFilter(stream);
// stream = new LowerCaseFilter(matchVersion, stream);
OffsetAttribute offsetAttribute = stream.addAttribute(OffsetAttribute.class);
CharTermAttribute charTermAttribute = stream.addAttribute(CharTermAttribute.class);
while (stream.incrementToken()) {
int startOffset = offsetAttribute.startOffset();
int endOffset = offsetAttribute.endOffset();
String token = charTermAttribute.toString();
ret.add(startOffset, endOffset);
// System.out.println(token.str+" \t\tS="+token.start+" E="+token.end);
}
} catch (java.io.IOException e) {
System.err.println(e);
}
return ret;
}
