public void testMakeTitleDocument() {
IndexId iid = IndexId.get("en-titles");
String text =
"Some very simple text used for testing\n== Heading 1 ==\nParagraph\n[[Category:Category1]]";
int references = 100;
int redirectTargetNamespace = -1;
ArrayList<Redirect> redirects = new ArrayList<Redirect>();
redirects.add(new Redirect(0, "Redirect", 2));
ArrayList<RelatedTitle> rel = new ArrayList<RelatedTitle>();
rel.add(new RelatedTitle(new Title(0, "Related test"), 50));
Hashtable<String, Integer> anchors = new Hashtable<String, Integer>();
anchors.put("Anchor", 20);
Date date = new Date();
Article article =
new Article(
10,
0,
"Test page",
text,
null,
references,
redirectTargetNamespace,
0,
redirects,
rel,
anchors,
date);
analyzer = Analyzers.getIndexerAnalyzer(new FieldBuilder(iid));
highlightAnalyzer = Analyzers.getHighlightAnalyzer(iid, false);
try {
doc =
WikiIndexModifier.makeTitleDocument(
article,
analyzer,
highlightAnalyzer,
iid,
"wiki",
"enwiki",
false,
new HashSet<String>());
assertEquals("1 [test] 1 [page] 255 [redirect]", tokens("alttitle"));
assertEquals("wiki:10", value("pageid"));
assertEquals("1 [wiki:0:Test page]", tokens("key"));
assertEquals("wiki", value("suffix"));
assertEquals("1 [enwiki]", tokens("dbname"));
assertEquals("0", value("namespace"));
assertEquals("Test page", value("title"));
} catch (IOException e) {
fail();
}
}
public void testSpellcheck() {
IndexId iid = IndexId.get("enwiki");
String text =
"Some very [[simple]] text used for testing, used for testing of something\n== Heading 1 ==\nParagraph\n[[Category:Category1]]";
int references = 100;
int redirectTargetNamespace = -1;
ArrayList<Redirect> redirects = new ArrayList<Redirect>();
redirects.add(new Redirect(0, "Redirect", 2));
ArrayList<RelatedTitle> rel = new ArrayList<RelatedTitle>();
rel.add(new RelatedTitle(new Title(0, "Related test"), 50));
Hashtable<String, Integer> anchors = new Hashtable<String, Integer>();
anchors.put("Anchor", 20);
Date date = new Date();
HashSet<String> stopWords = new HashSet<String>();
stopWords.add("of");
stopWords.add("for");
Article article =
new Article(
10,
0,
"Test for page",
text,
null,
references,
redirectTargetNamespace,
0,
redirects,
rel,
anchors,
date);
FieldBuilder fb =
new FieldBuilder(
iid,
FieldBuilder.Case.IGNORE_CASE,
FieldBuilder.Stemmer.NO_STEMMER,
FieldBuilder.Options.SPELL_CHECK);
fb.getBuilder().getFilters().setStopWords(stopWords);
analyzer = Analyzers.getSpellCheckAnalyzer(iid, stopWords);
try {
doc = WikiIndexModifier.makeDocument(article, fb, iid, stopWords, analyzer, true);
assertEquals(
"1 [test] 1 [for] 1 [page] 1 [test_for_page] 1 [some] 1 [page_some] 1 [very] 1 [some_very] 1 [simple] 1 [very_simple] 1 [text] 1 [simple_text] 1 [used] 1 [text_used] 1 [testing] 1 [used_for_testing] 1 [testing_used] 1 [of] 1 [something] 1 [testing_of_something] 1 [heading] 1 [something_heading] 1 [1] 1 [heading_1] 1 [paragraph] 1 [1_paragraph] 1 [category1] 1 [paragraph_category1] 1 [category1_test] 1 [anchor] 1 [page_anchor] 1 [redirect] 1 [anchor_redirect]",
tokens("contents"));
assertEquals(
"1 [test] 1 [for] 1 [test_for] 1 [page] 1 [for_page] 1 [test_for_page]", tokens("title"));
StringList sl = new StringList(value("spellcheck_context"));
assertEquals("[1, page, for, test, simple, heading]", sl.toCollection().toString());
} catch (IOException e) {
fail(e.getMessage());
}
}
public void testMakeDocuments() {
IndexId iid = IndexId.get("enwiki");
String text =
"Some very [[simple]] text used for testing\n== Heading 1 ==\nParagraph\n[[Category:Category1]]";
int references = 100;
int redirectTargetNamespace = -1;
ArrayList<Redirect> redirects = new ArrayList<Redirect>();
redirects.add(new Redirect(0, "Redirect", 2));
ArrayList<RelatedTitle> rel = new ArrayList<RelatedTitle>();
rel.add(new RelatedTitle(new Title(0, "Related test"), 50));
Hashtable<String, Integer> anchors = new Hashtable<String, Integer>();
anchors.put("Anchor", 20);
Date date = new Date();
Article article =
new Article(
10,
0,
"Test page",
text,
null,
references,
redirectTargetNamespace,
0,
redirects,
rel,
anchors,
date);
analyzer = Analyzers.getIndexerAnalyzer(new FieldBuilder(iid));
try {
doc =
WikiIndexModifier.makeDocument(
article, new FieldBuilder(iid), iid, new HashSet<String>(), analyzer);
assertEquals(
"1 [test] 1 [page] 3 [some] 1 [very] 1 [simple] 1 [text] 1 [used] 0 [use] 1 [for] 1 [testing] 0 [test] 500 [heading] 1 [1] 1 [paragraph] 1 [category1] 10 [test] 1 [page] 10 [anchor] 10 [redirect]",
tokens("contents"));
assertEquals("1 [test] 1 [page]", tokens("title"));
assertEquals("1 [test] 1 [page] 255 [anchor] 256 [redirect]", tokens("alttitle"));
assertEquals("1 [related] 1 [test]", tokens("related"));
assertEquals("1 [heading] 1 [1]", tokens("sections"));
assertEquals("1 [category1]", tokens("category"));
assertEquals("10", value("key"));
assertEquals("103", value("rank"));
assertEquals("1 [0:test page]", tokens("prefix"));
assertEquals("1 [egap] 1 [tset]", tokens("reverse_title"));
} catch (IOException e) {
fail();
}
}
public static void testTokenizer() throws Exception {
ArrayList<String> hits = new ArrayList<String>();
hits.add("0:Douglas Adams");
hits.add("0:The Hitchhiker's Guide to the Galaxy");
hits.add("0:2001");
hits.add("0:Boot");
IndexId iid = IndexId.get("wikilucene.nspart1.sub1");
FieldBuilder.BuilderSet bs = new FieldBuilder(iid).getBuilder();
Analyzer analyzer = Analyzers.getSearcherAnalyzer(iid);
ArrayList<String> stopWords = StopWords.getCached(iid);
WikiQueryParser parser =
new WikiQueryParser(
bs.getFields().contents(),
new NamespaceFilter(0),
analyzer,
bs,
NamespacePolicy.IGNORE,
stopWords);
Query q = parser.parse("miniseries");
HashSet<Term> termSet = new HashSet<Term>();
q.extractTerms(termSet);
Iterator<Term> it = termSet.iterator();
while (it.hasNext()) {
if (!(it.next().field().equals("contents"))) it.remove();
}
Term[] terms = termSet.toArray(new Term[] {});
IndexSearcher searcher = SearcherCache.getInstance().getLocalSearcher(iid);
int[] df = searcher.docFreqs(terms);
Highlight.highlight(
hits,
iid,
terms,
df,
searcher.maxDoc(),
parser.getWordsClean(),
StopWords.getPredefinedSet(iid),
false,
null,
false,
false);
}
public void testMakeHighlightDocuments() {
IndexId iid = IndexId.get("enwiki");
String text =
"Some very [[simple]] text used for testing\n== Heading 1 ==\nParagraph\n[[Category:Category1]]";
int references = 100;
int redirectTargetNamespace = -1;
ArrayList<Redirect> redirects = new ArrayList<Redirect>();
redirects.add(new Redirect(0, "Redirect", 2));
ArrayList<RelatedTitle> rel = new ArrayList<RelatedTitle>();
rel.add(new RelatedTitle(new Title(0, "Related test"), 50));
Hashtable<String, Integer> anchors = new Hashtable<String, Integer>();
anchors.put("Anchor", 20);
Date date = new Date();
Article article =
new Article(
10,
0,
"Test page",
text,
null,
references,
redirectTargetNamespace,
0,
redirects,
rel,
anchors,
date);
analyzer = Analyzers.getHighlightAnalyzer(iid, false);
try {
doc = WikiIndexModifier.makeHighlightDocument(article, new FieldBuilder(iid), iid);
assertEquals("1 [10]", tokens("pageid"));
assertEquals("1 [0:Test page]", tokens("key"));
} catch (IOException e) {
fail();
}
}