// FIXME: Should also use events!
public void deleteAttachment(Attachment att) throws ProviderException {
if (m_provider == null) return;
m_provider.deleteAttachment(att);
m_engine.getSearchManager().pageRemoved(att);
m_engine.getReferenceManager().clearPageEntries(att.getName());
}
/**
* Makes WikiText from a Collection.
*
* @param links Collection to make into WikiText.
* @param separator Separator string to use.
* @param numItems How many items to show.
* @return The WikiText
*/
protected String wikitizeCollection(Collection links, String separator, int numItems) {
if (links == null || links.isEmpty()) return "";
StringBuffer output = new StringBuffer();
Iterator it = links.iterator();
int count = 0;
//
// The output will be B Item[1] A S B Item[2] A S B Item[3] A
//
while (it.hasNext() && ((count < numItems) || (numItems == ALL_ITEMS))) {
String value = (String) it.next();
if (count > 0) {
output.append(m_after);
output.append(m_separator);
}
output.append(m_before);
// Make a Wiki markup link. See TranslatorReader.
output.append("[" + m_engine.beautifyTitle(value) + "|" + value + "]");
count++;
}
//
// Output final item - if there have been none, no "after" is printed
//
if (count > 0) output.append(m_after);
return output.toString();
}
/**
* Stores an attachment directly from a stream. If the attachment did not exist previously, this
* method will create it. If it did exist, it stores a new version.
*
* @param att Attachment to store this under.
* @param in InputStream from which the attachment contents will be read.
* @throws IOException If writing the attachment failed.
* @throws ProviderException If something else went wrong.
*/
public void storeAttachment(Attachment att, InputStream in)
throws IOException, ProviderException {
if (m_provider == null) {
return;
}
//
// Checks if the actual, real page exists without any modifications
// or aliases. We cannot store an attachment to a non-existant page.
//
if (!m_engine.getPageManager().pageExists(att.getParentName())) {
// the caller should catch the exception and use the exception text as an i18n key
throw new ProviderException("attach.parent.not.exist");
}
m_provider.putAttachmentData(att, in);
m_engine.getReferenceManager().updateReferences(att.getName(), new java.util.Vector());
WikiPage parent = new WikiPage(m_engine, att.getParentName());
m_engine.updateReferences(parent);
m_engine.getSearchManager().reindexPage(att);
}
/**
* Fetches the attachment content from the repository. Content is flat text that can be used for
* indexing/searching or display
*
* @param attachmentName Name of the attachment.
* @param version The version of the attachment.
* @return the content of the Attachment as a String.
*/
protected String getAttachmentContent(String attachmentName, int version) {
AttachmentManager mgr = m_engine.getAttachmentManager();
try {
Attachment att = mgr.getAttachmentInfo(attachmentName, version);
// FIXME: Find out why sometimes att is null
if (att != null) {
return getAttachmentContent(att);
}
} catch (ProviderException e) {
log.error("Attachment cannot be loaded", e);
}
// Something was wrong, no result is returned.
return null;
}
// FIXME: This API should be changed to return a List.
@SuppressWarnings("unchecked")
public Collection listAttachments(WikiPage wikipage) throws ProviderException {
if (m_provider == null) {
return new ArrayList();
}
Collection atts = m_provider.listAttachments(wikipage);
//
// This is just a sanity check; all of our providers return a Collection.
//
if (atts instanceof List) {
m_engine.getPageSorter().sortPages((List) atts);
}
return atts;
}
/** {@inheritDoc} */
public void initialize(WikiEngine engine, Properties props)
throws NoRequiredPropertyException, IOException {
m_engine = engine;
m_luceneDirectory = engine.getWorkDir() + File.separator + LUCENE_DIR;
int initialDelay =
TextUtil.getIntegerProperty(props, PROP_LUCENE_INITIALDELAY, LuceneUpdater.INITIAL_DELAY);
int indexDelay =
TextUtil.getIntegerProperty(props, PROP_LUCENE_INDEXDELAY, LuceneUpdater.INDEX_DELAY);
m_analyzerClass = TextUtil.getStringProperty(props, PROP_LUCENE_ANALYZER, m_analyzerClass);
// FIXME: Just to be simple for now, we will do full reindex
// only if no files are in lucene directory.
File dir = new File(m_luceneDirectory);
log.info("Lucene enabled, cache will be in: " + dir.getAbsolutePath());
try {
if (!dir.exists()) {
dir.mkdirs();
}
if (!dir.exists() || !dir.canWrite() || !dir.canRead()) {
log.error("Cannot write to Lucene directory, disabling Lucene: " + dir.getAbsolutePath());
throw new IOException("Invalid Lucene directory.");
}
String[] filelist = dir.list();
if (filelist == null) {
throw new IOException(
"Invalid Lucene directory: cannot produce listing: " + dir.getAbsolutePath());
}
} catch (IOException e) {
log.error("Problem while creating Lucene index - not using Lucene.", e);
}
// Start the Lucene update thread, which waits first
// for a little while before starting to go through
// the Lucene "pages that need updating".
LuceneUpdater updater = new LuceneUpdater(m_engine, this, initialDelay, indexDelay);
updater.start();
}
/**
* Makes HTML with common parameters.
*
* @param context The WikiContext
* @param wikitext The wikitext to render
* @return HTML
* @since 1.6.4
*/
protected String makeHTML(WikiContext context, String wikitext) {
String result = "";
RenderingManager mgr = m_engine.getRenderingManager();
try {
MarkupParser parser = mgr.getParser(context, wikitext);
parser.addLinkTransmutator(new CutMutator(m_maxwidth));
parser.enableImageInlining(false);
WikiDocument doc = parser.parse();
result = mgr.getHTML(context, doc);
} catch (IOException e) {
log.error("Failed to convert page data to HTML", e);
}
return result;
}
/**
* Adds a page-text pair to the lucene update queue. Safe to call always
*
* @param page WikiPage to add to the update queue.
*/
public void reindexPage(WikiPage page) {
if (page != null) {
String text;
// TODO: Think if this was better done in the thread itself?
if (page instanceof Attachment) {
text = getAttachmentContent((Attachment) page);
} else {
text = m_engine.getPureText(page);
}
if (text != null) {
// Add work item to m_updates queue.
Object[] pair = new Object[2];
pair[0] = page;
pair[1] = text;
m_updates.add(pair);
log.debug("Scheduling page " + page.getName() + " for index update");
}
}
}
/**
* @param att Attachment to get content for. Filename extension is used to determine the type of
* the attachment.
* @return String representing the content of the file. FIXME This is a very simple implementation
* of some text-based attachment, mainly used for testing. This should be replaced /moved to
* Attachment search providers or some other 'pluggable' wat to search attachments
*/
protected String getAttachmentContent(Attachment att) {
AttachmentManager mgr = m_engine.getAttachmentManager();
// FIXME: Add attachment plugin structure
String filename = att.getFileName();
boolean searchSuffix = false;
for (String suffix : SEARCHABLE_FILE_SUFFIXES) {
if (filename.endsWith(suffix)) {
searchSuffix = true;
}
}
if (searchSuffix) {
InputStream attStream;
try {
attStream = mgr.getAttachmentStream(att);
StringWriter sout = new StringWriter();
FileUtil.copyContents(new InputStreamReader(attStream), sout);
attStream.close();
sout.close();
return sout.toString();
} catch (ProviderException e) {
log.error("Attachment cannot be loaded", e);
return null;
} catch (IOException e) {
log.error("Attachment cannot be loaded", e);
return null;
}
}
return null;
}
public String execute(WikiContext context, Map params) throws PluginException {
WikiEngine engine = context.getEngine();
WikiPage page = context.getPage();
if (context.getVariable(VAR_ALREADY_PROCESSING) != null) return "Table of Contents";
StringBuffer sb = new StringBuffer();
sb.append("<div class=\"toc\">\n");
sb.append("<div class=\"collapsebox\">\n");
String title = (String) params.get(PARAM_TITLE);
if (title != null) {
sb.append("<h4>" + TextUtil.replaceEntities(title) + "</h4>\n");
} else {
sb.append("<h4>Table of Contents</h4>\n");
}
// should we use an ordered list?
m_usingNumberedList = false;
if (params.containsKey(PARAM_NUMBERED)) {
String numbered = (String) params.get(PARAM_NUMBERED);
if (numbered.equalsIgnoreCase("true")) {
m_usingNumberedList = true;
} else if (numbered.equalsIgnoreCase("yes")) {
m_usingNumberedList = true;
}
}
// if we are using a numbered list, get the rest of the parameters (if any) ...
if (m_usingNumberedList) {
int start = 0;
String startStr = (String) params.get(PARAM_START);
if ((startStr != null) && (startStr.matches("^\\d+$"))) {
start = Integer.parseInt(startStr);
}
if (start < 0) start = 0;
m_starting = start;
m_level1Index = start - 1;
if (m_level1Index < 0) m_level1Index = 0;
m_level2Index = 0;
m_level3Index = 0;
m_prefix = (String) params.get(PARAM_PREFIX);
if (m_prefix == null) m_prefix = "";
m_lastLevel = Heading.HEADING_LARGE;
}
try {
String wikiText = engine.getPureText(page);
context.setVariable(VAR_ALREADY_PROCESSING, "x");
JSPWikiMarkupParser parser = new JSPWikiMarkupParser(context, new StringReader(wikiText));
parser.addHeadingListener(this);
parser.parse();
sb.append("<ul>\n" + m_buf.toString() + "</ul>\n");
} catch (IOException e) {
log.error("Could not construct table of contents", e);
throw new PluginException("Unable to construct table of contents (see logs)");
}
sb.append("</div>\n</div>\n");
return sb.toString();
}
/**
* Figures out the full attachment name from the context and attachment name.
*
* @param context The current WikiContext
* @param attachmentname The file name of the attachment.
* @param version A particular version.
* @return Attachment, or null, if no such attachment or version exists.
* @throws ProviderException If something goes wrong.
*/
public Attachment getAttachmentInfo(WikiContext context, String attachmentname, int version)
throws ProviderException {
if (m_provider == null) {
return null;
}
WikiPage currentPage = null;
if (context != null) {
currentPage = context.getPage();
}
//
// Figure out the parent page of this attachment. If we can't find it,
// we'll assume this refers directly to the attachment.
//
int cutpt = attachmentname.lastIndexOf('/');
if (cutpt != -1) {
String parentPage = attachmentname.substring(0, cutpt);
parentPage = MarkupParser.cleanLink(parentPage);
attachmentname = attachmentname.substring(cutpt + 1);
// If we for some reason have an empty parent page name;
// this can't be an attachment
if (parentPage.length() == 0) return null;
currentPage = m_engine.getPage(parentPage);
//
// Go check for legacy name
//
// FIXME: This should be resolved using CommandResolver,
// not this adhoc way. This also assumes that the
// legacy charset is a subset of the full allowed set.
if (currentPage == null) {
currentPage = m_engine.getPage(MarkupParser.wikifyLink(parentPage));
}
}
//
// If the page cannot be determined, we cannot possibly find the
// attachments.
//
if (currentPage == null || currentPage.getName().length() == 0) {
return null;
}
// System.out.println("Seeking info on "+currentPage+"::"+attachmentname);
//
// Finally, figure out whether this is a real attachment or a generated
// attachment.
//
Attachment att;
att = getDynamicAttachment(currentPage.getName() + "/" + attachmentname);
if (att == null) {
att = m_provider.getAttachmentInfo(currentPage, attachmentname, version);
}
return att;
}
/**
* Filters a collection according to the include and exclude parameters.
*
* @param c The collection to filter.
* @return A filtered collection.
*/
protected Collection filterCollection(Collection c) {
ArrayList<Object> result = new ArrayList<Object>();
PatternMatcher pm = new Perl5Matcher();
for (Iterator i = c.iterator(); i.hasNext(); ) {
String pageName = null;
Object objectje = i.next();
if (objectje instanceof WikiPage) {
pageName = ((WikiPage) objectje).getName();
} else {
pageName = (String) objectje;
}
//
// If include parameter exists, then by default we include only those
// pages in it (excluding the ones in the exclude pattern list).
//
// include='*' means the same as no include.
//
boolean includeThis = m_include == null;
if (m_include != null) {
for (int j = 0; j < m_include.length; j++) {
if (pm.matches(pageName, m_include[j])) {
includeThis = true;
break;
}
}
}
if (m_exclude != null) {
for (int j = 0; j < m_exclude.length; j++) {
if (pm.matches(pageName, m_exclude[j])) {
includeThis = false;
break; // The inner loop, continue on the next item
}
}
}
if (includeThis) {
if (objectje instanceof WikiPage) {
result.add(objectje);
} else {
result.add(pageName);
}
//
// if we want to show the last modified date of the most recently change page, we keep a
// "high watermark" here:
WikiPage page = null;
if (m_lastModified) {
page = m_engine.getPage(pageName);
if (page != null) {
Date lastModPage = page.getLastModified();
if (log.isDebugEnabled()) {
log.debug("lastModified Date of page " + pageName + " : " + m_dateLastModified);
}
if (lastModPage.after(m_dateLastModified)) {
m_dateLastModified = lastModPage;
}
}
}
}
}
return result;
}
/**
* Searches pages using a particular combination of flags.
*
* @param query The query to perform in Lucene query language
* @param flags A set of flags
* @return A Collection of SearchResult instances
* @throws ProviderException if there is a problem with the backend
*/
public Collection findPages(String query, int flags) throws ProviderException {
IndexSearcher searcher = null;
ArrayList<SearchResult> list = null;
Highlighter highlighter = null;
try {
String[] queryfields = {
LUCENE_PAGE_CONTENTS, LUCENE_PAGE_NAME, LUCENE_AUTHOR, LUCENE_ATTACHMENTS
};
QueryParser qp =
new MultiFieldQueryParser(Version.LUCENE_36, queryfields, getLuceneAnalyzer());
// QueryParser qp = new QueryParser( LUCENE_PAGE_CONTENTS, getLuceneAnalyzer() );
Query luceneQuery = qp.parse(query);
if ((flags & FLAG_CONTEXTS) != 0) {
highlighter =
new Highlighter(
new SimpleHTMLFormatter("<span class=\"searchmatch\">", "</span>"),
new SimpleHTMLEncoder(),
new QueryScorer(luceneQuery));
}
try {
File dir = new File(m_luceneDirectory);
Directory luceneDir = new SimpleFSDirectory(dir, null);
IndexReader reader = IndexReader.open(luceneDir);
searcher = new IndexSearcher(reader);
} catch (Exception ex) {
log.info("Lucene not yet ready; indexing not started", ex);
return null;
}
ScoreDoc[] hits = searcher.search(luceneQuery, MAX_SEARCH_HITS).scoreDocs;
list = new ArrayList<SearchResult>(hits.length);
for (int curr = 0; curr < hits.length; curr++) {
int docID = hits[curr].doc;
Document doc = searcher.doc(docID);
String pageName = doc.get(LUCENE_ID);
WikiPage page = m_engine.getPage(pageName, WikiPageProvider.LATEST_VERSION);
if (page != null) {
if (page instanceof Attachment) {
// Currently attachments don't look nice on the search-results page
// When the search-results are cleaned up this can be enabled again.
}
int score = (int) (hits[curr].score * 100);
// Get highlighted search contexts
String text = doc.get(LUCENE_PAGE_CONTENTS);
String[] fragments = new String[0];
if (text != null && highlighter != null) {
TokenStream tokenStream =
getLuceneAnalyzer().tokenStream(LUCENE_PAGE_CONTENTS, new StringReader(text));
fragments = highlighter.getBestFragments(tokenStream, text, MAX_FRAGMENTS);
}
SearchResult result = new SearchResultImpl(page, score, fragments);
list.add(result);
} else {
log.error(
"Lucene found a result page '"
+ pageName
+ "' that could not be loaded, removing from Lucene cache");
pageRemoved(new WikiPage(m_engine, pageName));
}
}
} catch (IOException e) {
log.error("Failed during lucene search", e);
} catch (ParseException e) {
log.info("Broken query; cannot parse query ", e);
throw new ProviderException(
"You have entered a query Lucene cannot process: " + e.getMessage());
} catch (InvalidTokenOffsetsException e) {
log.error("Tokens are incompatible with provided text ", e);
} finally {
if (searcher != null) {
try {
searcher.close();
} catch (IOException e) {
log.error(e);
}
}
}
return list;
}
/**
* Indexes page using the given IndexWriter.
*
* @param page WikiPage
* @param text Page text to index
* @param writer The Lucene IndexWriter to use for indexing
* @return the created index Document
* @throws IOException If there's an indexing problem
*/
protected Document luceneIndexPage(WikiPage page, String text, IndexWriter writer)
throws IOException {
if (log.isDebugEnabled()) log.debug("Indexing " + page.getName() + "...");
// make a new, empty document
Document doc = new Document();
if (text == null) return doc;
// Raw name is the keyword we'll use to refer to this document for updates.
Field field = new Field(LUCENE_ID, page.getName(), Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.add(field);
// Body text. It is stored in the doc for search contexts.
field =
new Field(
LUCENE_PAGE_CONTENTS, text, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO);
doc.add(field);
// Allow searching by page name. Both beautified and raw
String unTokenizedTitle =
StringUtils.replaceChars(
page.getName(), MarkupParser.PUNCTUATION_CHARS_ALLOWED, c_punctuationSpaces);
field =
new Field(
LUCENE_PAGE_NAME,
TextUtil.beautifyString(page.getName()) + " " + unTokenizedTitle,
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.NO);
doc.add(field);
// Allow searching by authorname
if (page.getAuthor() != null) {
field =
new Field(
LUCENE_AUTHOR,
page.getAuthor(),
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.NO);
doc.add(field);
}
// Now add the names of the attachments of this page
try {
Collection attachments = m_engine.getAttachmentManager().listAttachments(page);
String attachmentNames = "";
for (Iterator it = attachments.iterator(); it.hasNext(); ) {
Attachment att = (Attachment) it.next();
attachmentNames += att.getName() + ";";
}
field =
new Field(
LUCENE_ATTACHMENTS,
attachmentNames,
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.NO);
doc.add(field);
} catch (ProviderException e) {
// Unable to read attachments
log.error("Failed to get attachments for page", e);
}
writer.addDocument(doc);
return doc;
}
/**
* Performs a full Lucene reindex, if necessary.
*
* @throws IOException If there's a problem during indexing
*/
protected void doFullLuceneReindex() throws IOException {
File dir = new File(m_luceneDirectory);
String[] filelist = dir.list();
if (filelist == null) {
throw new IOException(
"Invalid Lucene directory: cannot produce listing: " + dir.getAbsolutePath());
}
try {
if (filelist.length == 0) {
//
// No files? Reindex!
//
Date start = new Date();
IndexWriter writer = null;
log.info("Starting Lucene reindexing, this can take a couple minutes...");
Directory luceneDir = new SimpleFSDirectory(dir, null);
try {
writer = getIndexWriter(luceneDir);
Collection allPages = m_engine.getPageManager().getAllPages();
for (Iterator iterator = allPages.iterator(); iterator.hasNext(); ) {
WikiPage page = (WikiPage) iterator.next();
try {
String text =
m_engine
.getPageManager()
.getPageText(page.getName(), WikiProvider.LATEST_VERSION);
luceneIndexPage(page, text, writer);
} catch (IOException e) {
log.warn("Unable to index page " + page.getName() + ", continuing to next ", e);
}
}
Collection allAttachments = m_engine.getAttachmentManager().getAllAttachments();
for (Iterator iterator = allAttachments.iterator(); iterator.hasNext(); ) {
Attachment att = (Attachment) iterator.next();
try {
String text = getAttachmentContent(att.getName(), WikiProvider.LATEST_VERSION);
luceneIndexPage(att, text, writer);
} catch (IOException e) {
log.warn("Unable to index attachment " + att.getName() + ", continuing to next", e);
}
}
} finally {
close(writer);
}
Date end = new Date();
log.info(
"Full Lucene index finished in "
+ (end.getTime() - start.getTime())
+ " milliseconds.");
} else {
log.info("Files found in Lucene directory, not reindexing.");
}
} catch (NoClassDefFoundError e) {
log.info("Lucene libraries do not exist - not using Lucene.");
} catch (IOException e) {
log.error("Problem while creating Lucene index - not using Lucene.", e);
} catch (ProviderException e) {
log.error("Problem reading pages while creating Lucene index (JSPWiki won't start.)", e);
throw new IllegalArgumentException("unable to create Lucene index");
} catch (Exception e) {
log.error("Unable to start lucene", e);
}
}