/**
* Updates the lucene index for a single page.
*
* @param page The WikiPage to check
* @param text The page text to index.
*/
protected synchronized void updateLuceneIndex(WikiPage page, String text) {
IndexWriter writer = null;
log.debug("Updating Lucene index for page '" + page.getName() + "'...");
Directory luceneDir = null;
try {
pageRemoved(page);
// Now add back the new version.
luceneDir = new SimpleFSDirectory(new File(m_luceneDirectory), null);
writer = getIndexWriter(luceneDir);
luceneIndexPage(page, text, writer);
} catch (IOException e) {
log.error("Unable to update page '" + page.getName() + "' from Lucene index", e);
// reindexPage( page );
} catch (Exception e) {
log.error("Unexpected Lucene exception - please check configuration!", e);
// reindexPage( page );
} finally {
close(writer);
}
log.debug("Done updating Lucene index for page '" + page.getName() + "'.");
}
/**
* Compare two pages (WikiPage version). Compares them by name first. If the same name, compares
* their versions.
*
* @param page1 the first page
* @param page2 the second page
* @return see java.util.Comparator
* @see java.util.Comparator#compare(java.lang.Object, java.lang.Object)
*/
public int compare(WikiPage page1, WikiPage page2) {
if (page1 == page2) return 0; // the same object
int res = m_comparator.compare(page1.getName(), page2.getName());
if (res == 0) res = page1.getVersion() - page2.getVersion();
return res;
}
/** {@inheritDoc} */
public void pageRemoved(WikiPage page) {
IndexWriter writer = null;
try {
Directory luceneDir = new SimpleFSDirectory(new File(m_luceneDirectory), null);
writer = getIndexWriter(luceneDir);
Query query = new TermQuery(new Term(LUCENE_ID, page.getName()));
writer.deleteDocuments(query);
} catch (Exception e) {
log.error("Unable to remove page '" + page.getName() + "' from Lucene index", e);
} finally {
close(writer);
}
}
private String doRunAndGetErrorLog(String content) throws Exception {
WikiPage testPage = crawler.addPage(root, PathParser.parse("TestPage"), content);
request.setResource(testPage.getName());
Response response = responder.makeResponse(context, request);
MockResponseSender sender = new MockResponseSender();
sender.doSending(response);
String results = sender.sentData();
assertHasRegexp("ErrorLog", results);
WikiPage errorLog = errorLogsParentPage.getChildPage(testPage.getName());
return errorLog.getData().getContent();
}
private WikiPage getPropertiesContentFromPage(WikiPage page) throws Exception {
request = new MockRequest();
request.setResource(page.getName());
responder = new PropertiesResponder();
SimpleResponse response =
(SimpleResponse) responder.makeResponse(new FitNesseContext(root), request);
content = response.getContent();
return page;
}
@Test
public void DirectoryOfHtmlFilesIsExternalSuitePageChild() throws Exception {
fileSystem.makeFile("./somepath/ExternalSuite/subsuite/myfile.html", "stuff");
ExternalSuitePage page =
(ExternalSuitePage) pageRepository.makeChildPage("ExternalSuite", rootPage);
WikiPage child = pageRepository.findChildren(page).get(0);
assertEquals(ExternalSuitePage.class, child.getClass());
assertEquals("SubsuitE", child.getName());
}
@Test
public void testSimpleRun() throws Exception {
doSimpleRun(passFixtureTable());
assertSubString(testPage.getName(), results);
assertSubString("Test Results", results);
assertSubString("class", results);
assertNotSubString("ClassNotFoundException", results);
}
private Element createXmlFromPage(Document document, WikiPage page) {
Element pageElement = document.createElement("page");
XmlUtil.addTextNode(document, pageElement, "name", page.getName());
addLastModifiedTag(page, document, pageElement);
addXmlFromChildren(page, document, pageElement);
return pageElement;
}
public List getNormalChildren() throws Exception {
List children = realPage.getChildren();
List symChildren = new LinkedList();
for (Iterator iterator = children.iterator(); iterator.hasNext(); ) {
WikiPage child = (WikiPage) iterator.next();
if (!(child instanceof SymbolicPage))
symChildren.add(new SymbolicPage(child.getName(), child, this));
}
return symChildren;
}
/** {@inheritDoc} */
public Attachment getAttachmentInfo(WikiPage page, String name, int version)
throws ProviderException {
Attachment att = new Attachment(page.getName(), name);
File dir = findAttachmentDir(att);
if (!dir.exists()) {
// log.debug("Attachment dir not found - thus no attachment can exist.");
return null;
}
if (version == WikiProvider.LATEST_VERSION) {
version = findLatestVersion(att);
}
att.setVersion(version);
// Should attachment be cachable by the client (browser)?
if (m_disableCache != null) {
Matcher matcher = m_disableCache.matcher(name);
if (matcher.matches()) {
att.setCacheable(false);
}
}
// System.out.println("Fetching info on version "+version);
try {
Properties props = getPageProperties(att);
att.setAuthor(props.getProperty(version + ".author"));
String changeNote = props.getProperty(version + ".changenote");
if (changeNote != null) {
att.setChangeNote(changeNote);
}
File f = findFile(dir, att);
att.setSize(f.length());
att.setLastModified(new Date(f.lastModified()));
} catch (FileNotFoundException e) {
log.error("Can't get attachment properties for " + att, e);
return null;
} catch (IOException e) {
log.error("Can't read page properties", e);
throw new ProviderException("Cannot read page properties: " + e.getMessage());
}
// FIXME: Check for existence of this particular version.
return att;
}
@Test
public void testGetChidren() throws Exception {
crawler.addPage(root, PathParser.parse("AaAa"), "A content");
crawler.addPage(root, PathParser.parse("BbBb"), "B content");
crawler.addPage(root, PathParser.parse("CcCc"), "C content");
new File(defaultPath + "/root/someOtherDir").mkdir();
List<WikiPage> children = root.getChildren();
assertEquals(3, children.size());
for (WikiPage child : children) {
String name = child.getName();
boolean isOk = "AaAa".equals(name) || "BbBb".equals(name) || "CcCc".equals(name);
assertTrue("WikiPAge is not a valid one: " + name, isOk);
}
}
@Test
public void testHasExitValueHeader() throws Exception {
WikiPage testPage =
crawler.addPage(
root, PathParser.parse("TestPage"), classpathWidgets() + passFixtureTable());
request.setResource(testPage.getName());
Response response = responder.makeResponse(context, request);
MockResponseSender sender = new MockResponseSender();
sender.doSending(response);
String results = sender.sentData();
assertSubString("Exit-Code: 0", results);
}
@Test
public void testFixtureThatCrashes() throws Exception {
responder.setFastTest(false);
WikiPage testPage =
crawler.addPage(
root, PathParser.parse("TestPage"), classpathWidgets() + crashFixtureTable());
request.setResource(testPage.getName());
Response response = responder.makeResponse(context, request);
MockResponseSender sender = new MockResponseSender();
sender.doSending(response);
String results = sender.sentData();
assertSubString("ErrorLog", results);
}
@Test
public void testEmptyTestPage() throws Exception {
PageData data = root.getData();
data.setContent(classpathWidgets());
root.commit(data);
testPage = crawler.addPage(root, PathParser.parse("EmptyTestPage"));
request.setResource(testPage.getName());
response = responder.makeResponse(context, request);
sender = new MockResponseSender();
sender.doSending(response);
sender.sentData();
WikiPagePath errorLogPath = PathParser.parse("ErrorLogs.EmptyTestPage");
WikiPage errorLogPage = crawler.getPage(root, errorLogPath);
String errorLogContent = errorLogPage.getData().getContent();
assertNotSubString("Exception", errorLogContent);
}
private void doSimpleRunWithTags(String fixtureTable, String tags) throws Exception {
String simpleRunPageName = "TestPage";
testPage =
crawler.addPage(
root, PathParser.parse(simpleRunPageName), classpathWidgets() + fixtureTable);
if (tags != null) {
PageData pageData = testPage.getData();
pageData.setAttribute(PageData.PropertySUITES, tags);
testPage.commit(pageData);
}
request.setResource(testPage.getName());
responder.turnOffChunkingForTests();
response = responder.makeResponse(context, request);
sender = new MockResponseSender();
sender.doSending(response);
results = sender.sentData();
}
/**
* Adds a page-text pair to the lucene update queue. Safe to call always
*
* @param page WikiPage to add to the update queue.
*/
public void reindexPage(WikiPage page) {
if (page != null) {
String text;
// TODO: Think if this was better done in the thread itself?
if (page instanceof Attachment) {
text = getAttachmentContent((Attachment) page);
} else {
text = m_engine.getPureText(page);
}
if (text != null) {
// Add work item to m_updates queue.
Object[] pair = new Object[2];
pair[0] = page;
pair[1] = text;
m_updates.add(pair);
log.debug("Scheduling page " + page.getName() + " for index update");
}
}
}
public void processPage(WikiPage page) throws Exception {
traversedPages.add(page.getName());
}
/**
* Figures out the full attachment name from the context and attachment name.
*
* @param context The current WikiContext
* @param attachmentname The file name of the attachment.
* @param version A particular version.
* @return Attachment, or null, if no such attachment or version exists.
* @throws ProviderException If something goes wrong.
*/
public Attachment getAttachmentInfo(WikiContext context, String attachmentname, int version)
throws ProviderException {
if (m_provider == null) {
return null;
}
WikiPage currentPage = null;
if (context != null) {
currentPage = context.getPage();
}
//
// Figure out the parent page of this attachment. If we can't find it,
// we'll assume this refers directly to the attachment.
//
int cutpt = attachmentname.lastIndexOf('/');
if (cutpt != -1) {
String parentPage = attachmentname.substring(0, cutpt);
parentPage = MarkupParser.cleanLink(parentPage);
attachmentname = attachmentname.substring(cutpt + 1);
// If we for some reason have an empty parent page name;
// this can't be an attachment
if (parentPage.length() == 0) return null;
currentPage = m_engine.getPage(parentPage);
//
// Go check for legacy name
//
// FIXME: This should be resolved using CommandResolver,
// not this adhoc way. This also assumes that the
// legacy charset is a subset of the full allowed set.
if (currentPage == null) {
currentPage = m_engine.getPage(MarkupParser.wikifyLink(parentPage));
}
}
//
// If the page cannot be determined, we cannot possibly find the
// attachments.
//
if (currentPage == null || currentPage.getName().length() == 0) {
return null;
}
// System.out.println("Seeking info on "+currentPage+"::"+attachmentname);
//
// Finally, figure out whether this is a real attachment or a generated
// attachment.
//
Attachment att;
att = getDynamicAttachment(currentPage.getName() + "/" + attachmentname);
if (att == null) {
att = m_provider.getAttachmentInfo(currentPage, attachmentname, version);
}
return att;
}
/** {@inheritDoc} */
public Collection listAttachments(WikiPage page) throws ProviderException {
Collection<Attachment> result = new ArrayList<Attachment>();
File dir = findPageDir(page.getName());
if (dir != null) {
String[] attachments = dir.list();
if (attachments != null) {
//
// We now have a list of all potential attachments in
// the directory.
//
for (int i = 0; i < attachments.length; i++) {
File f = new File(dir, attachments[i]);
if (f.isDirectory()) {
String attachmentName = unmangleName(attachments[i]);
//
// Is it a new-stylea attachment directory? If yes,
// we'll just deduce the name. If not, however,
// we'll check if there's a suitable property file
// in the directory.
//
if (attachmentName.endsWith(ATTDIR_EXTENSION)) {
attachmentName =
attachmentName.substring(0, attachmentName.length() - ATTDIR_EXTENSION.length());
} else {
File propFile = new File(f, PROPERTY_FILE);
if (!propFile.exists()) {
//
// This is not obviously a JSPWiki attachment,
// so let's just skip it.
//
continue;
}
}
Attachment att = getAttachmentInfo(page, attachmentName, WikiProvider.LATEST_VERSION);
//
// Sanity check - shouldn't really be happening, unless
// you mess with the repository directly.
//
if (att == null) {
throw new ProviderException(
"Attachment disappeared while reading information:"
+ " if you did not touch the repository, there is a serious bug somewhere. "
+ "Attachment = "
+ attachments[i]
+ ", decoded = "
+ attachmentName);
}
result.add(att);
}
}
}
}
return result;
}
/**
* Performs a full Lucene reindex, if necessary.
*
* @throws IOException If there's a problem during indexing
*/
protected void doFullLuceneReindex() throws IOException {
File dir = new File(m_luceneDirectory);
String[] filelist = dir.list();
if (filelist == null) {
throw new IOException(
"Invalid Lucene directory: cannot produce listing: " + dir.getAbsolutePath());
}
try {
if (filelist.length == 0) {
//
// No files? Reindex!
//
Date start = new Date();
IndexWriter writer = null;
log.info("Starting Lucene reindexing, this can take a couple minutes...");
Directory luceneDir = new SimpleFSDirectory(dir, null);
try {
writer = getIndexWriter(luceneDir);
Collection allPages = m_engine.getPageManager().getAllPages();
for (Iterator iterator = allPages.iterator(); iterator.hasNext(); ) {
WikiPage page = (WikiPage) iterator.next();
try {
String text =
m_engine
.getPageManager()
.getPageText(page.getName(), WikiProvider.LATEST_VERSION);
luceneIndexPage(page, text, writer);
} catch (IOException e) {
log.warn("Unable to index page " + page.getName() + ", continuing to next ", e);
}
}
Collection allAttachments = m_engine.getAttachmentManager().getAllAttachments();
for (Iterator iterator = allAttachments.iterator(); iterator.hasNext(); ) {
Attachment att = (Attachment) iterator.next();
try {
String text = getAttachmentContent(att.getName(), WikiProvider.LATEST_VERSION);
luceneIndexPage(att, text, writer);
} catch (IOException e) {
log.warn("Unable to index attachment " + att.getName() + ", continuing to next", e);
}
}
} finally {
close(writer);
}
Date end = new Date();
log.info(
"Full Lucene index finished in "
+ (end.getTime() - start.getTime())
+ " milliseconds.");
} else {
log.info("Files found in Lucene directory, not reindexing.");
}
} catch (NoClassDefFoundError e) {
log.info("Lucene libraries do not exist - not using Lucene.");
} catch (IOException e) {
log.error("Problem while creating Lucene index - not using Lucene.", e);
} catch (ProviderException e) {
log.error("Problem reading pages while creating Lucene index (JSPWiki won't start.)", e);
throw new IllegalArgumentException("unable to create Lucene index");
} catch (Exception e) {
log.error("Unable to start lucene", e);
}
}
public void testAddChildPageWithMissingParent() throws Exception {
WikiPage page = crawler.addPage(root, PathParser.parse("WikiMail.BadSubject0123"), "");
assertNotNull(page);
assertEquals("BadSubject0123", page.getName());
assertEquals(PathParser.parse("WikiMail.BadSubject0123"), crawler.getFullPath(page));
}
/**
* Indexes page using the given IndexWriter.
*
* @param page WikiPage
* @param text Page text to index
* @param writer The Lucene IndexWriter to use for indexing
* @return the created index Document
* @throws IOException If there's an indexing problem
*/
protected Document luceneIndexPage(WikiPage page, String text, IndexWriter writer)
throws IOException {
if (log.isDebugEnabled()) log.debug("Indexing " + page.getName() + "...");
// make a new, empty document
Document doc = new Document();
if (text == null) return doc;
// Raw name is the keyword we'll use to refer to this document for updates.
Field field = new Field(LUCENE_ID, page.getName(), Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.add(field);
// Body text. It is stored in the doc for search contexts.
field =
new Field(
LUCENE_PAGE_CONTENTS, text, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO);
doc.add(field);
// Allow searching by page name. Both beautified and raw
String unTokenizedTitle =
StringUtils.replaceChars(
page.getName(), MarkupParser.PUNCTUATION_CHARS_ALLOWED, c_punctuationSpaces);
field =
new Field(
LUCENE_PAGE_NAME,
TextUtil.beautifyString(page.getName()) + " " + unTokenizedTitle,
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.NO);
doc.add(field);
// Allow searching by authorname
if (page.getAuthor() != null) {
field =
new Field(
LUCENE_AUTHOR,
page.getAuthor(),
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.NO);
doc.add(field);
}
// Now add the names of the attachments of this page
try {
Collection attachments = m_engine.getAttachmentManager().listAttachments(page);
String attachmentNames = "";
for (Iterator it = attachments.iterator(); it.hasNext(); ) {
Attachment att = (Attachment) it.next();
attachmentNames += att.getName() + ";";
}
field =
new Field(
LUCENE_ATTACHMENTS,
attachmentNames,
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.NO);
doc.add(field);
} catch (ProviderException e) {
// Unable to read attachments
log.error("Failed to get attachments for page", e);
}
writer.addDocument(doc);
return doc;
}
