public void testMultilingualInput() throws Exception {
List<TermFreq> input = LookupBenchmarkTest.readTop50KWiki();
FSTCompletionLookup lookup = new FSTCompletionLookup();
lookup.build(new TermFreqArrayIterator(input));
for (TermFreq tf : input) {
assertNotNull(
"Not found: " + tf.term.toString(),
lookup.get(_TestUtil.bytesToCharSequence(tf.term, random())));
assertEquals(
tf.term.utf8ToString(),
lookup
.lookup(_TestUtil.bytesToCharSequence(tf.term, random()), true, 1)
.get(0)
.key
.toString());
}
List<LookupResult> result = lookup.lookup(stringToCharSequence("wit"), true, 5);
assertEquals(5, result.size());
assertTrue(result.get(0).key.toString().equals("wit")); // exact match.
assertTrue(result.get(1).key.toString().equals("with")); // highest count.
}
public void testRandom() throws Exception {
List<TermFreq> freqs = new ArrayList<TermFreq>();
Random rnd = random();
for (int i = 0; i < 2500 + rnd.nextInt(2500); i++) {
int weight = rnd.nextInt(100);
freqs.add(new TermFreq("" + rnd.nextLong(), weight));
}
FSTCompletionLookup lookup = new FSTCompletionLookup();
lookup.build(new TermFreqArrayIterator(freqs.toArray(new TermFreq[freqs.size()])));
for (TermFreq tf : freqs) {
final String term = tf.term.utf8ToString();
for (int i = 1; i < term.length(); i++) {
String prefix = term.substring(0, i);
for (LookupResult lr : lookup.lookup(stringToCharSequence(prefix), true, 10)) {
assertTrue(lr.key.toString().startsWith(prefix));
}
}
}
}