/** * Updates the lucene index for a single page. * * @param page The WikiPage to check * @param text The page text to index. */ protected synchronized void updateLuceneIndex(WikiPage page, String text) { IndexWriter writer = null; log.debug("Updating Lucene index for page '" + page.getName() + "'..."); Directory luceneDir = null; try { pageRemoved(page); // Now add back the new version. luceneDir = new SimpleFSDirectory(new File(m_luceneDirectory), null); writer = getIndexWriter(luceneDir); luceneIndexPage(page, text, writer); } catch (IOException e) { log.error("Unable to update page '" + page.getName() + "' from Lucene index", e); // reindexPage( page ); } catch (Exception e) { log.error("Unexpected Lucene exception - please check configuration!", e); // reindexPage( page ); } finally { close(writer); } log.debug("Done updating Lucene index for page '" + page.getName() + "'."); }
/** * Compare two pages (WikiPage version). Compares them by name first. If the same name, compares * their versions. * * @param page1 the first page * @param page2 the second page * @return see java.util.Comparator * @see java.util.Comparator#compare(java.lang.Object, java.lang.Object) */ public int compare(WikiPage page1, WikiPage page2) { if (page1 == page2) return 0; // the same object int res = m_comparator.compare(page1.getName(), page2.getName()); if (res == 0) res = page1.getVersion() - page2.getVersion(); return res; }
/** {@inheritDoc} */ public void pageRemoved(WikiPage page) { IndexWriter writer = null; try { Directory luceneDir = new SimpleFSDirectory(new File(m_luceneDirectory), null); writer = getIndexWriter(luceneDir); Query query = new TermQuery(new Term(LUCENE_ID, page.getName())); writer.deleteDocuments(query); } catch (Exception e) { log.error("Unable to remove page '" + page.getName() + "' from Lucene index", e); } finally { close(writer); } }
private String doRunAndGetErrorLog(String content) throws Exception { WikiPage testPage = crawler.addPage(root, PathParser.parse("TestPage"), content); request.setResource(testPage.getName()); Response response = responder.makeResponse(context, request); MockResponseSender sender = new MockResponseSender(); sender.doSending(response); String results = sender.sentData(); assertHasRegexp("ErrorLog", results); WikiPage errorLog = errorLogsParentPage.getChildPage(testPage.getName()); return errorLog.getData().getContent(); }
private WikiPage getPropertiesContentFromPage(WikiPage page) throws Exception { request = new MockRequest(); request.setResource(page.getName()); responder = new PropertiesResponder(); SimpleResponse response = (SimpleResponse) responder.makeResponse(new FitNesseContext(root), request); content = response.getContent(); return page; }
@Test public void DirectoryOfHtmlFilesIsExternalSuitePageChild() throws Exception { fileSystem.makeFile("./somepath/ExternalSuite/subsuite/myfile.html", "stuff"); ExternalSuitePage page = (ExternalSuitePage) pageRepository.makeChildPage("ExternalSuite", rootPage); WikiPage child = pageRepository.findChildren(page).get(0); assertEquals(ExternalSuitePage.class, child.getClass()); assertEquals("SubsuitE", child.getName()); }
@Test public void testSimpleRun() throws Exception { doSimpleRun(passFixtureTable()); assertSubString(testPage.getName(), results); assertSubString("Test Results", results); assertSubString("class", results); assertNotSubString("ClassNotFoundException", results); }
private Element createXmlFromPage(Document document, WikiPage page) { Element pageElement = document.createElement("page"); XmlUtil.addTextNode(document, pageElement, "name", page.getName()); addLastModifiedTag(page, document, pageElement); addXmlFromChildren(page, document, pageElement); return pageElement; }
public List getNormalChildren() throws Exception { List children = realPage.getChildren(); List symChildren = new LinkedList(); for (Iterator iterator = children.iterator(); iterator.hasNext(); ) { WikiPage child = (WikiPage) iterator.next(); if (!(child instanceof SymbolicPage)) symChildren.add(new SymbolicPage(child.getName(), child, this)); } return symChildren; }
/** {@inheritDoc} */ public Attachment getAttachmentInfo(WikiPage page, String name, int version) throws ProviderException { Attachment att = new Attachment(page.getName(), name); File dir = findAttachmentDir(att); if (!dir.exists()) { // log.debug("Attachment dir not found - thus no attachment can exist."); return null; } if (version == WikiProvider.LATEST_VERSION) { version = findLatestVersion(att); } att.setVersion(version); // Should attachment be cachable by the client (browser)? if (m_disableCache != null) { Matcher matcher = m_disableCache.matcher(name); if (matcher.matches()) { att.setCacheable(false); } } // System.out.println("Fetching info on version "+version); try { Properties props = getPageProperties(att); att.setAuthor(props.getProperty(version + ".author")); String changeNote = props.getProperty(version + ".changenote"); if (changeNote != null) { att.setChangeNote(changeNote); } File f = findFile(dir, att); att.setSize(f.length()); att.setLastModified(new Date(f.lastModified())); } catch (FileNotFoundException e) { log.error("Can't get attachment properties for " + att, e); return null; } catch (IOException e) { log.error("Can't read page properties", e); throw new ProviderException("Cannot read page properties: " + e.getMessage()); } // FIXME: Check for existence of this particular version. return att; }
@Test public void testGetChidren() throws Exception { crawler.addPage(root, PathParser.parse("AaAa"), "A content"); crawler.addPage(root, PathParser.parse("BbBb"), "B content"); crawler.addPage(root, PathParser.parse("CcCc"), "C content"); new File(defaultPath + "/root/someOtherDir").mkdir(); List<WikiPage> children = root.getChildren(); assertEquals(3, children.size()); for (WikiPage child : children) { String name = child.getName(); boolean isOk = "AaAa".equals(name) || "BbBb".equals(name) || "CcCc".equals(name); assertTrue("WikiPAge is not a valid one: " + name, isOk); } }
@Test public void testHasExitValueHeader() throws Exception { WikiPage testPage = crawler.addPage( root, PathParser.parse("TestPage"), classpathWidgets() + passFixtureTable()); request.setResource(testPage.getName()); Response response = responder.makeResponse(context, request); MockResponseSender sender = new MockResponseSender(); sender.doSending(response); String results = sender.sentData(); assertSubString("Exit-Code: 0", results); }
@Test public void testFixtureThatCrashes() throws Exception { responder.setFastTest(false); WikiPage testPage = crawler.addPage( root, PathParser.parse("TestPage"), classpathWidgets() + crashFixtureTable()); request.setResource(testPage.getName()); Response response = responder.makeResponse(context, request); MockResponseSender sender = new MockResponseSender(); sender.doSending(response); String results = sender.sentData(); assertSubString("ErrorLog", results); }
@Test public void testEmptyTestPage() throws Exception { PageData data = root.getData(); data.setContent(classpathWidgets()); root.commit(data); testPage = crawler.addPage(root, PathParser.parse("EmptyTestPage")); request.setResource(testPage.getName()); response = responder.makeResponse(context, request); sender = new MockResponseSender(); sender.doSending(response); sender.sentData(); WikiPagePath errorLogPath = PathParser.parse("ErrorLogs.EmptyTestPage"); WikiPage errorLogPage = crawler.getPage(root, errorLogPath); String errorLogContent = errorLogPage.getData().getContent(); assertNotSubString("Exception", errorLogContent); }
private void doSimpleRunWithTags(String fixtureTable, String tags) throws Exception { String simpleRunPageName = "TestPage"; testPage = crawler.addPage( root, PathParser.parse(simpleRunPageName), classpathWidgets() + fixtureTable); if (tags != null) { PageData pageData = testPage.getData(); pageData.setAttribute(PageData.PropertySUITES, tags); testPage.commit(pageData); } request.setResource(testPage.getName()); responder.turnOffChunkingForTests(); response = responder.makeResponse(context, request); sender = new MockResponseSender(); sender.doSending(response); results = sender.sentData(); }
/** * Adds a page-text pair to the lucene update queue. Safe to call always * * @param page WikiPage to add to the update queue. */ public void reindexPage(WikiPage page) { if (page != null) { String text; // TODO: Think if this was better done in the thread itself? if (page instanceof Attachment) { text = getAttachmentContent((Attachment) page); } else { text = m_engine.getPureText(page); } if (text != null) { // Add work item to m_updates queue. Object[] pair = new Object[2]; pair[0] = page; pair[1] = text; m_updates.add(pair); log.debug("Scheduling page " + page.getName() + " for index update"); } } }
public void processPage(WikiPage page) throws Exception { traversedPages.add(page.getName()); }
/** * Figures out the full attachment name from the context and attachment name. * * @param context The current WikiContext * @param attachmentname The file name of the attachment. * @param version A particular version. * @return Attachment, or null, if no such attachment or version exists. * @throws ProviderException If something goes wrong. */ public Attachment getAttachmentInfo(WikiContext context, String attachmentname, int version) throws ProviderException { if (m_provider == null) { return null; } WikiPage currentPage = null; if (context != null) { currentPage = context.getPage(); } // // Figure out the parent page of this attachment. If we can't find it, // we'll assume this refers directly to the attachment. // int cutpt = attachmentname.lastIndexOf('/'); if (cutpt != -1) { String parentPage = attachmentname.substring(0, cutpt); parentPage = MarkupParser.cleanLink(parentPage); attachmentname = attachmentname.substring(cutpt + 1); // If we for some reason have an empty parent page name; // this can't be an attachment if (parentPage.length() == 0) return null; currentPage = m_engine.getPage(parentPage); // // Go check for legacy name // // FIXME: This should be resolved using CommandResolver, // not this adhoc way. This also assumes that the // legacy charset is a subset of the full allowed set. if (currentPage == null) { currentPage = m_engine.getPage(MarkupParser.wikifyLink(parentPage)); } } // // If the page cannot be determined, we cannot possibly find the // attachments. // if (currentPage == null || currentPage.getName().length() == 0) { return null; } // System.out.println("Seeking info on "+currentPage+"::"+attachmentname); // // Finally, figure out whether this is a real attachment or a generated // attachment. // Attachment att; att = getDynamicAttachment(currentPage.getName() + "/" + attachmentname); if (att == null) { att = m_provider.getAttachmentInfo(currentPage, attachmentname, version); } return att; }
/** {@inheritDoc} */ public Collection listAttachments(WikiPage page) throws ProviderException { Collection<Attachment> result = new ArrayList<Attachment>(); File dir = findPageDir(page.getName()); if (dir != null) { String[] attachments = dir.list(); if (attachments != null) { // // We now have a list of all potential attachments in // the directory. // for (int i = 0; i < attachments.length; i++) { File f = new File(dir, attachments[i]); if (f.isDirectory()) { String attachmentName = unmangleName(attachments[i]); // // Is it a new-stylea attachment directory? If yes, // we'll just deduce the name. If not, however, // we'll check if there's a suitable property file // in the directory. // if (attachmentName.endsWith(ATTDIR_EXTENSION)) { attachmentName = attachmentName.substring(0, attachmentName.length() - ATTDIR_EXTENSION.length()); } else { File propFile = new File(f, PROPERTY_FILE); if (!propFile.exists()) { // // This is not obviously a JSPWiki attachment, // so let's just skip it. // continue; } } Attachment att = getAttachmentInfo(page, attachmentName, WikiProvider.LATEST_VERSION); // // Sanity check - shouldn't really be happening, unless // you mess with the repository directly. // if (att == null) { throw new ProviderException( "Attachment disappeared while reading information:" + " if you did not touch the repository, there is a serious bug somewhere. " + "Attachment = " + attachments[i] + ", decoded = " + attachmentName); } result.add(att); } } } } return result; }
/** * Performs a full Lucene reindex, if necessary. * * @throws IOException If there's a problem during indexing */ protected void doFullLuceneReindex() throws IOException { File dir = new File(m_luceneDirectory); String[] filelist = dir.list(); if (filelist == null) { throw new IOException( "Invalid Lucene directory: cannot produce listing: " + dir.getAbsolutePath()); } try { if (filelist.length == 0) { // // No files? Reindex! // Date start = new Date(); IndexWriter writer = null; log.info("Starting Lucene reindexing, this can take a couple minutes..."); Directory luceneDir = new SimpleFSDirectory(dir, null); try { writer = getIndexWriter(luceneDir); Collection allPages = m_engine.getPageManager().getAllPages(); for (Iterator iterator = allPages.iterator(); iterator.hasNext(); ) { WikiPage page = (WikiPage) iterator.next(); try { String text = m_engine .getPageManager() .getPageText(page.getName(), WikiProvider.LATEST_VERSION); luceneIndexPage(page, text, writer); } catch (IOException e) { log.warn("Unable to index page " + page.getName() + ", continuing to next ", e); } } Collection allAttachments = m_engine.getAttachmentManager().getAllAttachments(); for (Iterator iterator = allAttachments.iterator(); iterator.hasNext(); ) { Attachment att = (Attachment) iterator.next(); try { String text = getAttachmentContent(att.getName(), WikiProvider.LATEST_VERSION); luceneIndexPage(att, text, writer); } catch (IOException e) { log.warn("Unable to index attachment " + att.getName() + ", continuing to next", e); } } } finally { close(writer); } Date end = new Date(); log.info( "Full Lucene index finished in " + (end.getTime() - start.getTime()) + " milliseconds."); } else { log.info("Files found in Lucene directory, not reindexing."); } } catch (NoClassDefFoundError e) { log.info("Lucene libraries do not exist - not using Lucene."); } catch (IOException e) { log.error("Problem while creating Lucene index - not using Lucene.", e); } catch (ProviderException e) { log.error("Problem reading pages while creating Lucene index (JSPWiki won't start.)", e); throw new IllegalArgumentException("unable to create Lucene index"); } catch (Exception e) { log.error("Unable to start lucene", e); } }
public void testAddChildPageWithMissingParent() throws Exception { WikiPage page = crawler.addPage(root, PathParser.parse("WikiMail.BadSubject0123"), ""); assertNotNull(page); assertEquals("BadSubject0123", page.getName()); assertEquals(PathParser.parse("WikiMail.BadSubject0123"), crawler.getFullPath(page)); }
/** * Indexes page using the given IndexWriter. * * @param page WikiPage * @param text Page text to index * @param writer The Lucene IndexWriter to use for indexing * @return the created index Document * @throws IOException If there's an indexing problem */ protected Document luceneIndexPage(WikiPage page, String text, IndexWriter writer) throws IOException { if (log.isDebugEnabled()) log.debug("Indexing " + page.getName() + "..."); // make a new, empty document Document doc = new Document(); if (text == null) return doc; // Raw name is the keyword we'll use to refer to this document for updates. Field field = new Field(LUCENE_ID, page.getName(), Field.Store.YES, Field.Index.NOT_ANALYZED); doc.add(field); // Body text. It is stored in the doc for search contexts. field = new Field( LUCENE_PAGE_CONTENTS, text, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO); doc.add(field); // Allow searching by page name. Both beautified and raw String unTokenizedTitle = StringUtils.replaceChars( page.getName(), MarkupParser.PUNCTUATION_CHARS_ALLOWED, c_punctuationSpaces); field = new Field( LUCENE_PAGE_NAME, TextUtil.beautifyString(page.getName()) + " " + unTokenizedTitle, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO); doc.add(field); // Allow searching by authorname if (page.getAuthor() != null) { field = new Field( LUCENE_AUTHOR, page.getAuthor(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO); doc.add(field); } // Now add the names of the attachments of this page try { Collection attachments = m_engine.getAttachmentManager().listAttachments(page); String attachmentNames = ""; for (Iterator it = attachments.iterator(); it.hasNext(); ) { Attachment att = (Attachment) it.next(); attachmentNames += att.getName() + ";"; } field = new Field( LUCENE_ATTACHMENTS, attachmentNames, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO); doc.add(field); } catch (ProviderException e) { // Unable to read attachments log.error("Failed to get attachments for page", e); } writer.addDocument(doc); return doc; }