/**
* A rule-based tokenizer used to prepare a sentence for NE extraction.
*
* @param text text to tokenize
* @return array of tokens
*/
public static String[] tokenize(String text) {
Span[] spans = NameFinder.tokenizeToSpans(text);
return NameFinder.spansToStrings(spans, text);
}
