@Test
public void testVectorHighlighter() throws Exception {
Directory dir = new RAMDirectory();
IndexWriter indexWriter =
new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
Document document = new Document();
document.add(new TextField("_id", "1", Field.Store.YES));
document.add(
new Field(
"content",
"the big bad dog",
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.WITH_POSITIONS_OFFSETS));
indexWriter.addDocument(document);
IndexReader reader = DirectoryReader.open(indexWriter, true);
IndexSearcher searcher = new IndexSearcher(reader);
TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);
assertThat(topDocs.totalHits, equalTo(1));
XFastVectorHighlighter highlighter = new XFastVectorHighlighter();
String fragment =
highlighter.getBestFragment(
highlighter.getFieldQuery(new TermQuery(new Term("content", "bad"))),
reader,
topDocs.scoreDocs[0].doc,
"content",
30);
assertThat(fragment, notNullValue());
assertThat(fragment, equalTo("the big <b>bad</b> dog"));
}
// Tests whether merging of docs that have different
// omitTermFreqAndPositions for the same field works
public void testMixedMerge() throws Exception {
Directory ram = newDirectory();
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter writer =
new IndexWriter(
ram,
newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)
.setMaxBufferedDocs(3)
.setMergePolicy(newLogMergePolicy(2)));
Document d = new Document();
// this field will have Tf
Field f1 = newField("f1", "This field has term freqs", normalType);
d.add(f1);
// this field will NOT have Tf
Field f2 = newField("f2", "This field has NO Tf in all docs", omitType);
d.add(f2);
for (int i = 0; i < 30; i++) writer.addDocument(d);
// now we add another document which has term freq for field f2 and not for f1 and verify if the
// SegmentMerger
// keep things constant
d = new Document();
// Reverese
f1 = newField("f1", "This field has term freqs", omitType);
d.add(f1);
f2 = newField("f2", "This field has NO Tf in all docs", normalType);
d.add(f2);
for (int i = 0; i < 30; i++) writer.addDocument(d);
// force merge
writer.forceMerge(1);
// flush
writer.close();
SegmentReader reader = getOnlySegmentReader(DirectoryReader.open(ram));
FieldInfos fi = reader.getFieldInfos();
assertEquals(
"OmitTermFreqAndPositions field bit should be set.",
IndexOptions.DOCS_ONLY,
fi.fieldInfo("f1").getIndexOptions());
assertEquals(
"OmitTermFreqAndPositions field bit should be set.",
IndexOptions.DOCS_ONLY,
fi.fieldInfo("f2").getIndexOptions());
reader.close();
ram.close();
}
private void doTest(Random random, PrintWriter out, boolean useCompoundFiles, int MAX_DOCS)
throws Exception {
Directory directory = newDirectory();
Analyzer analyzer = new MockAnalyzer(random);
IndexWriterConfig conf = newIndexWriterConfig(analyzer);
final MergePolicy mp = conf.getMergePolicy();
mp.setNoCFSRatio(useCompoundFiles ? 1.0 : 0.0);
IndexWriter writer = new IndexWriter(directory, conf);
if (VERBOSE) {
System.out.println("TEST: now build index MAX_DOCS=" + MAX_DOCS);
}
for (int j = 0; j < MAX_DOCS; j++) {
Document d = new Document();
d.add(newTextField(PRIORITY_FIELD, HIGH_PRIORITY, Field.Store.YES));
d.add(newTextField(ID_FIELD, Integer.toString(j), Field.Store.YES));
writer.addDocument(d);
}
writer.close();
// try a search without OR
IndexReader reader = DirectoryReader.open(directory);
IndexSearcher searcher = newSearcher(reader);
Query query = new TermQuery(new Term(PRIORITY_FIELD, HIGH_PRIORITY));
out.println("Query: " + query.toString(PRIORITY_FIELD));
if (VERBOSE) {
System.out.println("TEST: search query=" + query);
}
final Sort sort = new Sort(SortField.FIELD_SCORE, new SortField(ID_FIELD, SortField.Type.INT));
ScoreDoc[] hits = searcher.search(query, null, MAX_DOCS, sort).scoreDocs;
printHits(out, hits, searcher);
checkHits(hits, MAX_DOCS, searcher);
// try a new search with OR
searcher = newSearcher(reader);
hits = null;
BooleanQuery booleanQuery = new BooleanQuery();
booleanQuery.add(
new TermQuery(new Term(PRIORITY_FIELD, HIGH_PRIORITY)), BooleanClause.Occur.SHOULD);
booleanQuery.add(
new TermQuery(new Term(PRIORITY_FIELD, MED_PRIORITY)), BooleanClause.Occur.SHOULD);
out.println("Query: " + booleanQuery.toString(PRIORITY_FIELD));
hits = searcher.search(booleanQuery, null, MAX_DOCS, sort).scoreDocs;
printHits(out, hits, searcher);
checkHits(hits, MAX_DOCS, searcher);
reader.close();
directory.close();
}
/** {@inheritDoc} */
public void pageRemoved(WikiPage page) {
IndexWriter writer = null;
try {
Directory luceneDir = new SimpleFSDirectory(new File(m_luceneDirectory), null);
writer = getIndexWriter(luceneDir);
Query query = new TermQuery(new Term(LUCENE_ID, page.getName()));
writer.deleteDocuments(query);
} catch (Exception e) {
log.error("Unable to remove page '" + page.getName() + "' from Lucene index", e);
} finally {
close(writer);
}
}
private Directory getDir2(Random random) throws IOException {
Directory dir2 = newDirectory();
IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig(new MockAnalyzer(random)));
Document d3 = new Document();
d3.add(newTextField("f3", "v1", Field.Store.YES));
d3.add(newTextField("f4", "v1", Field.Store.YES));
w2.addDocument(d3);
Document d4 = new Document();
d4.add(newTextField("f3", "v2", Field.Store.YES));
d4.add(newTextField("f4", "v2", Field.Store.YES));
w2.addDocument(d4);
w2.close();
return dir2;
}
private Directory getDir1(Random random) throws IOException {
Directory dir1 = newDirectory();
IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig(new MockAnalyzer(random)));
Document d1 = new Document();
d1.add(newTextField("f1", "v1", Field.Store.YES));
d1.add(newTextField("f2", "v1", Field.Store.YES));
w1.addDocument(d1);
Document d2 = new Document();
d2.add(newTextField("f1", "v2", Field.Store.YES));
d2.add(newTextField("f2", "v2", Field.Store.YES));
w1.addDocument(d2);
w1.close();
return dir1;
}
public void sort(File directory) throws IOException {
LOG.info("IndexSorter: starting.");
Date start = new Date();
int termIndexInterval = getConf().getInt("indexer.termIndexInterval", 128);
IndexReader reader = IndexReader.open(new File(directory, "index"));
Searcher searcher =
new IndexSearcher(new File(directory, "index").getAbsolutePath()); // TODO MC
SortingReader sorter = new SortingReader(reader, newToOld(reader, searcher)); // TODO MC
IndexWriter writer = new IndexWriter(new File(directory, "index-sorted"), null, true);
writer.setTermIndexInterval(termIndexInterval);
writer.setUseCompoundFile(false);
writer.addIndexes(new IndexReader[] {sorter});
writer.close();
Date end = new Date();
LOG.info("IndexSorter: done, " + (end.getTime() - start.getTime()) + " total milliseconds");
}
void close(IndexWriter writer) {
try {
if (writer != null) {
writer.close(true);
}
} catch (IOException e) {
log.error(e);
}
}
// Fields 1-4 indexed together:
private IndexSearcher single(Random random) throws IOException {
dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random)));
Document d1 = new Document();
d1.add(newTextField("f1", "v1", Field.Store.YES));
d1.add(newTextField("f2", "v1", Field.Store.YES));
d1.add(newTextField("f3", "v1", Field.Store.YES));
d1.add(newTextField("f4", "v1", Field.Store.YES));
w.addDocument(d1);
Document d2 = new Document();
d2.add(newTextField("f1", "v2", Field.Store.YES));
d2.add(newTextField("f2", "v2", Field.Store.YES));
d2.add(newTextField("f3", "v2", Field.Store.YES));
d2.add(newTextField("f4", "v2", Field.Store.YES));
w.addDocument(d2);
w.close();
DirectoryReader ir = DirectoryReader.open(dir);
return newSearcher(ir);
}
/** Closes the index writer, returning <tt>false</tt> if it failed to close. */
public static boolean safeClose(IndexWriter writer) {
if (writer == null) {
return true;
}
try {
writer.close();
return true;
} catch (IOException e) {
return false;
}
}
public NewIndexer(String indexDir2) throws IOException {
// create the index
if (indexWriter2 == null) {
indexWriter2 =
new IndexWriter(
FSDirectory.open(new File("resultsList")),
new IndexWriterConfig(
Version.LUCENE_36,
new EnglishAnalyzer(Version.LUCENE_36, StandardAnalyzer.STOP_WORDS_SET)));
indexWriter2.deleteAll();
}
}
public void testIncompatibleIndexes() throws IOException {
// two documents:
Directory dir1 = getDir1(random());
// one document only:
Directory dir2 = newDirectory();
IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig(new MockAnalyzer(random())));
Document d3 = new Document();
d3.add(newTextField("f3", "v1", Field.Store.YES));
w2.addDocument(d3);
w2.close();
LeafReader ir1 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir1));
LeafReader ir2 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir2));
try {
new ParallelLeafReader(ir1, ir2);
fail("didn't get exptected exception: indexes don't have same number of documents");
} catch (IllegalArgumentException e) {
// expected exception
}
try {
new ParallelLeafReader(
random().nextBoolean(), new LeafReader[] {ir1, ir2}, new LeafReader[] {ir1, ir2});
fail("didn't get expected exception: indexes don't have same number of documents");
} catch (IllegalArgumentException e) {
// expected exception
}
// check RefCounts
assertEquals(1, ir1.getRefCount());
assertEquals(1, ir2.getRefCount());
ir1.close();
ir2.close();
dir1.close();
dir2.close();
}
// add items to index
public void newIndex(NewIndexItem newIndexItem) throws IOException {
// deleting the item, if already exists
// indexWriter.deleteDocuments(new Term(IndexItem.SEARCHLABEL,
// indexItem.getSearchLabel().toString()));
Document doc = new Document();
doc.add(
new Field(
NewIndexItem.QUERYNUMBER,
newIndexItem.getNewQueryNumber().toString(),
Field.Store.YES,
Field.Index.ANALYZED));
doc.add(
new Field(
NewIndexItem.TERMID,
newIndexItem.getNewTermID().toString(),
Field.Store.YES,
Field.Index.ANALYZED));
doc.add(
new Field(
NewIndexItem.LABEL,
newIndexItem.getNewLabel().toString(),
Field.Store.YES,
Field.Index.ANALYZED));
doc.add(
new Field(
NewIndexItem.SEARCHLABEL,
newIndexItem.getNewSearchLabel().toString(),
Field.Store.YES,
Field.Index.ANALYZED));
doc.add(
new Field(
NewIndexItem.QUERYTERM,
newIndexItem.getNewQueryTerm().toString(),
Field.Store.YES,
Field.Index.ANALYZED));
doc.add(
new Field(
NewIndexItem.NUMOFRESULTS,
newIndexItem.getNewNumOfResults().toString(),
Field.Store.YES,
Field.Index.ANALYZED));
// add the document to the index
indexWriter2.addDocument(doc);
}
// Verifies no *.prx exists when all fields omit term freq:
public void testNoPrxFile() throws Throwable {
Directory ram = newDirectory();
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter writer =
new IndexWriter(
ram,
newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)
.setMaxBufferedDocs(3)
.setMergePolicy(newLogMergePolicy()));
LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
lmp.setMergeFactor(2);
lmp.setUseCompoundFile(false);
Document d = new Document();
Field f1 = newField("f1", "This field has term freqs", omitType);
d.add(f1);
for (int i = 0; i < 30; i++) writer.addDocument(d);
writer.commit();
assertNoPrx(ram);
// now add some documents with positions, and check
// there is no prox after full merge
d = new Document();
f1 = newTextField("f1", "This field has positions", Field.Store.NO);
d.add(f1);
for (int i = 0; i < 30; i++) writer.addDocument(d);
// force merge
writer.forceMerge(1);
// flush
writer.close();
assertNoPrx(ram);
ram.close();
}
/**
* Assert that the {@code scoreType} operates as expected and parents are found in the expected
* order.
*
* <p>This will use the test index's parent/child types to create parents with multiple children.
* Each child will have a randomly generated scored stored in {@link #CHILD_SCORE_NAME}, which is
* used to score based on the {@code scoreType} by using a {@link MockScorer} to determine the
* expected scores.
*
* @param scoreType The score type to use within the query to score parents relative to their
* children.
* @throws IOException if any unexpected error occurs
*/
private void assertScoreType(ScoreType scoreType) throws IOException {
SearchContext context = SearchContext.current();
Directory directory = newDirectory();
IndexWriter writer =
new IndexWriter(directory, newIndexWriterConfig(new MockAnalyzer(random())));
// calculates the expected score per parent
MockScorer scorer = new MockScorer(scoreType);
scorer.scores = new FloatArrayList(10);
// number of parents to generate
int parentDocs = scaledRandomIntBetween(2, 10);
// unique child ID
int childDocId = 0;
// Parent ID to expected score
Map<String, Float> parentScores = new TreeMap<>();
// Add a few random parents to ensure that the children's score is appropriately taken into
// account
for (int parentDocId = 0; parentDocId < parentDocs; ++parentDocId) {
String parent = Integer.toString(parentDocId);
// Create the parent
Document parentDocument = new Document();
parentDocument.add(
new StringField(UidFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.YES));
parentDocument.add(new StringField(IdFieldMapper.NAME, parent, Field.Store.YES));
parentDocument.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO));
// add the parent to the index
writer.addDocument(parentDocument);
int numChildDocs = scaledRandomIntBetween(1, 10);
// forget any parent's previous scores
scorer.scores.clear();
// associate children with the parent
for (int i = 0; i < numChildDocs; ++i) {
int childScore = random().nextInt(128);
Document childDocument = new Document();
childDocument.add(
new StringField(
UidFieldMapper.NAME,
Uid.createUid("child", Integer.toString(childDocId++)),
Field.Store.NO));
childDocument.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO));
// parent association:
childDocument.add(
new StringField(
ParentFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.NO));
childDocument.add(new DoubleField(CHILD_SCORE_NAME, childScore, Field.Store.NO));
// remember the score to be calculated
scorer.scores.add(childScore);
// add the associated child to the index
writer.addDocument(childDocument);
}
// this score that should be returned for this parent
parentScores.put(parent, scorer.score());
}
writer.commit();
IndexReader reader = DirectoryReader.open(writer, true);
IndexSearcher searcher = new IndexSearcher(reader);
// setup to read the parent/child map
Engine.Searcher engineSearcher =
new Engine.Searcher(ChildrenQueryTests.class.getSimpleName(), searcher);
((TestSearchContext) context).setSearcher(new ContextIndexSearcher(context, engineSearcher));
// child query that returns the score as the value of "childScore" for each child document, with
// the parent's score determined by the score type
QueryBuilder childQueryBuilder =
functionScoreQuery(typeFilter("child"))
.add(new FieldValueFactorFunctionBuilder(CHILD_SCORE_NAME));
QueryBuilder queryBuilder =
hasChildQuery("child", childQueryBuilder)
.scoreType(scoreType.name().toLowerCase(Locale.ENGLISH))
.setShortCircuitCutoff(parentDocs);
// Perform the search for the documents using the selected score type
TopDocs docs = searcher.search(parseQuery(queryBuilder), parentDocs);
assertThat("Expected all parents", docs.totalHits, is(parentDocs));
// score should be descending (just a sanity check)
float topScore = docs.scoreDocs[0].score;
// ensure each score is returned as expected
for (int i = 0; i < parentDocs; ++i) {
ScoreDoc scoreDoc = docs.scoreDocs[i];
// get the ID from the document to get its expected score; remove it so we cannot double-count
// it
float score = parentScores.remove(reader.document(scoreDoc.doc).get(IdFieldMapper.NAME));
// expect exact match
assertThat("Unexpected score", scoreDoc.score, is(score));
assertThat("Not descending", score, lessThanOrEqualTo(topScore));
// it had better keep descending
topScore = score;
}
reader.close();
writer.close();
directory.close();
}
// Test scores with one field with Term Freqs and one without, otherwise with equal content
public void testBasic() throws Exception {
Directory dir = newDirectory();
Analyzer analyzer = new MockAnalyzer(random());
IndexWriter writer =
new IndexWriter(
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)
.setMaxBufferedDocs(2)
.setSimilarity(new SimpleSimilarity())
.setMergePolicy(newLogMergePolicy(2)));
StringBuilder sb = new StringBuilder(265);
String term = "term";
for (int i = 0; i < 30; i++) {
Document d = new Document();
sb.append(term).append(" ");
String content = sb.toString();
Field noTf = newField("noTf", content + (i % 2 == 0 ? "" : " notf"), omitType);
d.add(noTf);
Field tf = newField("tf", content + (i % 2 == 0 ? " tf" : ""), normalType);
d.add(tf);
writer.addDocument(d);
// System.out.println(d);
}
writer.forceMerge(1);
// flush
writer.close();
/*
* Verify the index
*/
IndexReader reader = DirectoryReader.open(dir);
IndexSearcher searcher = newSearcher(reader);
searcher.setSimilarity(new SimpleSimilarity());
Term a = new Term("noTf", term);
Term b = new Term("tf", term);
Term c = new Term("noTf", "notf");
Term d = new Term("tf", "tf");
TermQuery q1 = new TermQuery(a);
TermQuery q2 = new TermQuery(b);
TermQuery q3 = new TermQuery(c);
TermQuery q4 = new TermQuery(d);
PhraseQuery pq = new PhraseQuery();
pq.add(a);
pq.add(c);
try {
searcher.search(pq, 10);
fail("did not hit expected exception");
} catch (Exception e) {
Throwable cause = e;
// If the searcher uses an executor service, the IAE is wrapped into other exceptions
while (cause.getCause() != null) {
cause = cause.getCause();
}
assertTrue("Expected an IAE, got " + cause, cause instanceof IllegalStateException);
}
searcher.search(
q1,
new CountingHitCollector() {
private Scorer scorer;
@Override
public final void setScorer(Scorer scorer) {
this.scorer = scorer;
}
@Override
public final void collect(int doc) throws IOException {
// System.out.println("Q1: Doc=" + doc + " score=" + score);
float score = scorer.score();
assertTrue("got score=" + score, score == 1.0f);
super.collect(doc);
}
});
// System.out.println(CountingHitCollector.getCount());
searcher.search(
q2,
new CountingHitCollector() {
private Scorer scorer;
@Override
public final void setScorer(Scorer scorer) {
this.scorer = scorer;
}
@Override
public final void collect(int doc) throws IOException {
// System.out.println("Q2: Doc=" + doc + " score=" + score);
float score = scorer.score();
assertEquals(1.0f + doc, score, 0.00001f);
super.collect(doc);
}
});
// System.out.println(CountingHitCollector.getCount());
searcher.search(
q3,
new CountingHitCollector() {
private Scorer scorer;
@Override
public final void setScorer(Scorer scorer) {
this.scorer = scorer;
}
@Override
public final void collect(int doc) throws IOException {
// System.out.println("Q1: Doc=" + doc + " score=" + score);
float score = scorer.score();
assertTrue(score == 1.0f);
assertFalse(doc % 2 == 0);
super.collect(doc);
}
});
// System.out.println(CountingHitCollector.getCount());
searcher.search(
q4,
new CountingHitCollector() {
private Scorer scorer;
@Override
public final void setScorer(Scorer scorer) {
this.scorer = scorer;
}
@Override
public final void collect(int doc) throws IOException {
float score = scorer.score();
// System.out.println("Q1: Doc=" + doc + " score=" + score);
assertTrue(score == 1.0f);
assertTrue(doc % 2 == 0);
super.collect(doc);
}
});
// System.out.println(CountingHitCollector.getCount());
BooleanQuery bq = new BooleanQuery();
bq.add(q1, Occur.MUST);
bq.add(q4, Occur.MUST);
searcher.search(
bq,
new CountingHitCollector() {
@Override
public final void collect(int doc) throws IOException {
// System.out.println("BQ: Doc=" + doc + " score=" + score);
super.collect(doc);
}
});
assertEquals(15, CountingHitCollector.getCount());
reader.close();
dir.close();
}
@Test
public void testVectorHighlighterPrefixQuery() throws Exception {
Directory dir = new RAMDirectory();
IndexWriter indexWriter =
new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
Document document = new Document();
document.add(new TextField("_id", "1", Field.Store.YES));
document.add(
new Field(
"content",
"the big bad dog",
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.WITH_POSITIONS_OFFSETS));
indexWriter.addDocument(document);
IndexReader reader = DirectoryReader.open(indexWriter, true);
IndexSearcher searcher = new IndexSearcher(reader);
TopDocs topDocs = searcher.search(new TermQuery(new Term("_id", "1")), 1);
assertThat(topDocs.totalHits, equalTo(1));
XFastVectorHighlighter highlighter = new XFastVectorHighlighter();
PrefixQuery prefixQuery = new PrefixQuery(new Term("content", "ba"));
assertThat(
prefixQuery.getRewriteMethod().getClass().getName(),
equalTo(PrefixQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT.getClass().getName()));
String fragment =
highlighter.getBestFragment(
highlighter.getFieldQuery(prefixQuery),
reader,
topDocs.scoreDocs[0].doc,
"content",
30);
assertThat(fragment, nullValue());
prefixQuery.setRewriteMethod(PrefixQuery.SCORING_BOOLEAN_QUERY_REWRITE);
Query rewriteQuery = prefixQuery.rewrite(reader);
fragment =
highlighter.getBestFragment(
highlighter.getFieldQuery(rewriteQuery),
reader,
topDocs.scoreDocs[0].doc,
"content",
30);
assertThat(fragment, notNullValue());
// now check with the custom field query
prefixQuery = new PrefixQuery(new Term("content", "ba"));
assertThat(
prefixQuery.getRewriteMethod().getClass().getName(),
equalTo(PrefixQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT.getClass().getName()));
fragment =
highlighter.getBestFragment(
new CustomFieldQuery(prefixQuery, reader, highlighter),
reader,
topDocs.scoreDocs[0].doc,
"content",
30);
assertThat(fragment, notNullValue());
}
// Closing the index
public void close() throws IOException {
indexWriter2.close();
}
/**
* Indexes page using the given IndexWriter.
*
* @param page WikiPage
* @param text Page text to index
* @param writer The Lucene IndexWriter to use for indexing
* @return the created index Document
* @throws IOException If there's an indexing problem
*/
protected Document luceneIndexPage(WikiPage page, String text, IndexWriter writer)
throws IOException {
if (log.isDebugEnabled()) log.debug("Indexing " + page.getName() + "...");
// make a new, empty document
Document doc = new Document();
if (text == null) return doc;
// Raw name is the keyword we'll use to refer to this document for updates.
Field field = new Field(LUCENE_ID, page.getName(), Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.add(field);
// Body text. It is stored in the doc for search contexts.
field =
new Field(
LUCENE_PAGE_CONTENTS, text, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO);
doc.add(field);
// Allow searching by page name. Both beautified and raw
String unTokenizedTitle =
StringUtils.replaceChars(
page.getName(), MarkupParser.PUNCTUATION_CHARS_ALLOWED, c_punctuationSpaces);
field =
new Field(
LUCENE_PAGE_NAME,
TextUtil.beautifyString(page.getName()) + " " + unTokenizedTitle,
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.NO);
doc.add(field);
// Allow searching by authorname
if (page.getAuthor() != null) {
field =
new Field(
LUCENE_AUTHOR,
page.getAuthor(),
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.NO);
doc.add(field);
}
// Now add the names of the attachments of this page
try {
Collection attachments = m_engine.getAttachmentManager().listAttachments(page);
String attachmentNames = "";
for (Iterator it = attachments.iterator(); it.hasNext(); ) {
Attachment att = (Attachment) it.next();
attachmentNames += att.getName() + ";";
}
field =
new Field(
LUCENE_ATTACHMENTS,
attachmentNames,
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.NO);
doc.add(field);
} catch (ProviderException e) {
// Unable to read attachments
log.error("Failed to get attachments for page", e);
}
writer.addDocument(doc);
return doc;
}