/**
* Create a basic Lucene document to add to the index. This document is suitable to be parsed with
* the StandardAnalyzer.
*/
private Document createStandardDocument(Topic topic) {
String topicContent = topic.getTopicContent();
if (topicContent == null) {
topicContent = "";
}
Document doc = new Document();
// store the (not analyzed) topic name to use when deleting records from the index.
doc.add(
new Field(
FIELD_TOPIC_NAME, topic.getName(), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
// add the topic namespace (not analyzed) topic namespace to allow retrieval by namespace.
// this field is used internally in searches.
doc.add(
new Field(
FIELD_TOPIC_NAMESPACE,
topic.getNamespace().getId().toString(),
Field.Store.NO,
Field.Index.NOT_ANALYZED_NO_NORMS));
// analyze the topic name so that (for example) a search for "New York" will match "New York
// City"
Field nameField =
new Field(FIELD_TOPIC_NAME_ANALYZED, topic.getName(), Field.Store.NO, Field.Index.ANALYZED);
// make the topic name worth 3x as much as topic content in searches
nameField.setBoost(3.0f);
doc.add(nameField);
// analyze & store the topic content so that it is searchable and also usable for display in
// search result summaries
doc.add(new Field(FIELD_TOPIC_CONTENT, topicContent, Field.Store.YES, Field.Index.ANALYZED));
return doc;
}
/**
* Set up images separately - one image is created in both virtual wikis, the second image is set
* up in only the shared virtual wiki.
*/
private void setupImage(VirtualWiki virtualWiki, Topic topic)
throws DataAccessException, IOException, WikiException {
if (!topic.getName().toLowerCase().startsWith("image:")) {
throw new IllegalArgumentException(
"Cannot call JAMWikiUtilTest.setupImage for non-image topics");
}
TopicVersion topicVersion =
new TopicVersion(
null, "127.0.0.1", null, topic.getTopicContent(), topic.getTopicContent().length());
topic.setTopicType(TopicType.IMAGE);
topicVersion.setEditType(TopicVersion.EDIT_UPLOAD);
// hard code image details - Image:Test Image.jpg will be created for both the "en"
// and "test" virtual wikis, while Image:Test Image2.jpg will be created only for
// the "test" virtual wiki.
WikiFileVersion wikiFileVersion = new WikiFileVersion();
if (topic.getName().equals("Image:Test Image.jpg") && virtualWiki.getName().equals("en")) {
WikiBase.getDataHandler().writeTopic(topic, topicVersion, null, null);
ImageUtil.writeWikiFile(
topic,
wikiFileVersion,
null,
"127.0.0.1",
"test_image.jpg",
"/test_image.jpg",
"image/jpeg",
61136,
null);
} else if (topic.getName().equals("Image:Test Image.jpg")
&& virtualWiki.getName().equals("test")) {
WikiBase.getDataHandler().writeTopic(topic, topicVersion, null, null);
ImageUtil.writeWikiFile(
topic,
wikiFileVersion,
null,
"127.0.0.1",
"test_image_shared.jpg",
"/test_image_shared.jpg",
"image/jpeg",
61136,
null);
} else if (topic.getName().equals("Image:Test Image2.jpg")
&& virtualWiki.getName().equals("test")) {
WikiBase.getDataHandler().writeTopic(topic, topicVersion, null, null);
ImageUtil.writeWikiFile(
topic,
wikiFileVersion,
null,
"127.0.0.1",
"test_image2_shared.jpg",
"/test_image2_shared.jpg",
"image/jpeg",
61136,
null);
}
}
/**
* Given a topic, if that topic is a redirect find the target topic of the redirection.
*
* @param parent The topic being queried. If this topic is a redirect then the redirect target
* will be returned, otherwise the topic itself is returned.
* @param attempts The maximum number of child topics to follow. This parameter prevents infinite
* loops if topics redirect back to one another.
* @return If the parent topic is a redirect then this method returns the target topic that is
* being redirected to, otherwise the parent topic is returned.
* @throws DataAccessException Thrown if any error occurs while retrieving data.
*/
public static Topic findRedirectedTopic(Topic parent, int attempts) throws DataAccessException {
int count = attempts;
String target = parent.getRedirectTo();
if (parent.getTopicType() != TopicType.REDIRECT || StringUtils.isBlank(target)) {
logger.error("getRedirectTarget() called for non-redirect topic " + parent.getName());
return parent;
}
// avoid infinite redirection
count++;
if (count > 10) {
// TODO throw new WikiException(new WikiMessage("topic.redirect.infinite"));
return parent;
}
String virtualWiki = parent.getVirtualWiki();
WikiLink wikiLink = LinkUtil.parseWikiLink(virtualWiki, target);
if (wikiLink.getVirtualWiki() != null) {
virtualWiki = wikiLink.getVirtualWiki().getName();
}
// get the topic that is being redirected to
Topic child =
WikiBase.getDataHandler().lookupTopic(virtualWiki, wikiLink.getDestination(), false, null);
if (child == null) {
// child being redirected to doesn't exist, return parent
return parent;
}
if (StringUtils.isBlank(child.getRedirectTo())) {
// found a topic that is not a redirect, return
return child;
}
// child is a redirect, keep looking
return findRedirectedTopic(child, count);
}
/**
* Add a topic to the search index.
*
* @param topic The Topic object that is to be added to the index.
*/
public void addToIndex(Topic topic) {
try {
long start = System.currentTimeMillis();
IndexWriter writer = this.retrieveIndexWriter(topic.getVirtualWiki(), false);
this.addToIndex(writer, topic);
this.commit(writer, this.autoCommit);
if (logger.isDebugEnabled()) {
logger.debug(
"Add to search index for topic "
+ topic.getVirtualWiki()
+ " / "
+ topic.getName()
+ " in "
+ ((System.currentTimeMillis() - start) / 1000.000)
+ " s.");
}
} catch (Exception e) {
logger.error(
"Exception while adding topic " + topic.getVirtualWiki() + " / " + topic.getName(), e);
}
}
@Test
public void testImportFromFileWithTwoTopics() throws Throwable {
String virtualWiki = VIRTUAL_WIKI_EN;
List<String> results = this.importTestFile(FILE_TEST_TWO_TOPICS_WITH_HISTORY);
// validate that the first topic parsed
assertTrue("Parsed topic '" + TOPIC_NAME1 + "'", results.contains(TOPIC_NAME1));
Topic topic1 = WikiBase.getDataHandler().lookupTopic(virtualWiki, TOPIC_NAME1, false, null);
// validate that the parsed topic correctly set topic values
assertEquals("Topic name '" + TOPIC_NAME1 + "' set correctly", TOPIC_NAME1, topic1.getName());
assertTrue(
"Topic content set correctly",
topic1.getTopicContent().indexOf("Link to user page: [[User:Test User]]") != -1);
// validate that namespaces were converted from Mediawiki to JAMWiki correctly
assertTrue(
"Topic content namespaces updated correctly",
topic1.getTopicContent().indexOf("Link to user talk page: [[User comments: Test User]]")
!= -1);
// validate that the second topic parsed
assertTrue("Parsed topic '" + TOPIC_NAME2 + "'", results.contains(TOPIC_NAME2));
Topic topic2 = WikiBase.getDataHandler().lookupTopic(virtualWiki, TOPIC_NAME2, false, null);
// validate that the parsed topic correctly set topic values
assertEquals("Topic name '" + TOPIC_NAME2 + "' set correctly", TOPIC_NAME2, topic2.getName());
}
private void edit(HttpServletRequest request, ModelAndView next, WikiPageInfo pageInfo)
throws Exception {
String topicName = WikiUtil.getTopicFromRequest(request);
String virtualWiki = pageInfo.getVirtualWikiName();
Topic topic = loadTopic(virtualWiki, topicName);
// topic name might be updated by loadTopic
topicName = topic.getName();
Integer lastTopicVersionId = retrieveLastTopicVersionId(request, topic);
next.addObject("lastTopicVersionId", lastTopicVersionId);
String contents = (String) request.getParameter("contents");
if (isPreview(request)) {
preview(request, next, pageInfo);
} else if (isShowChanges(request)) {
showChanges(request, next, pageInfo, virtualWiki, topicName, lastTopicVersionId);
} else if (!StringUtils.isBlank(request.getParameter("topicVersionId"))) {
// editing an older version
Integer topicVersionId = Integer.valueOf(request.getParameter("topicVersionId"));
TopicVersion topicVersion = WikiBase.getDataHandler().lookupTopicVersion(topicVersionId);
if (topicVersion == null) {
throw new WikiException(new WikiMessage("common.exception.notopic"));
}
contents = topicVersion.getVersionContent();
if (!lastTopicVersionId.equals(topicVersionId)) {
next.addObject("topicVersionId", topicVersionId);
}
} else if (!StringUtils.isBlank(request.getParameter("section"))) {
// editing a section of a topic
int section = Integer.valueOf(request.getParameter("section"));
String[] sliceResults =
ParserUtil.parseSlice(
request.getContextPath(), request.getLocale(), virtualWiki, topicName, section);
contents = sliceResults[1];
String sectionName = sliceResults[0];
String editComment = "/* " + sectionName + " */ ";
next.addObject("editComment", editComment);
} else {
// editing a full new or existing topic
contents = (topic == null) ? "" : topic.getTopicContent();
}
this.loadEdit(request, next, pageInfo, contents, virtualWiki, topicName, true);
}
private void edit(HttpServletRequest request, ModelAndView next, WikiPageInfo pageInfo)
throws Exception {
String topicName = JAMWikiServlet.getTopicFromRequest(request);
String virtualWiki = JAMWikiServlet.getVirtualWikiFromURI(request);
Topic topic = loadTopic(virtualWiki, topicName);
// topic name might be updated by loadTopic
topicName = topic.getName();
int lastTopicVersionId = retrieveLastTopicVersionId(request, virtualWiki, topicName);
next.addObject("lastTopicVersionId", new Integer(lastTopicVersionId));
loadEdit(request, next, pageInfo, virtualWiki, topicName, true);
String contents = null;
if (isPreview(request)) {
preview(request, next, pageInfo);
return;
}
pageInfo.setAction(WikiPageInfo.ACTION_EDIT);
if (StringUtils.hasText(request.getParameter("topicVersionId"))) {
// editing an older version
int topicVersionId = Integer.parseInt(request.getParameter("topicVersionId"));
TopicVersion topicVersion =
WikiBase.getHandler().lookupTopicVersion(topicName, topicVersionId);
if (topicVersion == null) {
throw new WikiException(new WikiMessage("common.exception.notopic"));
}
contents = topicVersion.getVersionContent();
if (lastTopicVersionId != topicVersionId) {
next.addObject("topicVersionId", new Integer(topicVersionId));
}
} else if (StringUtils.hasText(request.getParameter("section"))) {
// editing a section of a topic
int section = (new Integer(request.getParameter("section"))).intValue();
ParserDocument parserDocument =
Utilities.parseSlice(request, virtualWiki, topicName, section);
contents = parserDocument.getContent();
} else {
// editing a full new or existing topic
contents = (topic == null) ? "" : topic.getTopicContent();
}
next.addObject("contents", contents);
}
/**
* Action used when viewing a topic.
*
* @param request The servlet request object.
* @param next The Spring ModelAndView object.
* @param topicName The topic being viewed. This value must be a valid topic that can be loaded as
* a org.jamwiki.model.Topic object.
*/
protected void viewTopic(
HttpServletRequest request,
ModelAndView next,
WikiMessage pageTitle,
Topic topic,
boolean sectionEdit)
throws Exception {
// FIXME - what should the default be for topics that don't exist?
String contents = "";
if (topic == null) {
throw new WikiException(new WikiMessage("common.exception.notopic"));
}
String virtualWiki = topic.getVirtualWiki();
String topicName = topic.getName();
String displayName = request.getRemoteAddr();
WikiUser user = Utilities.currentUser(request);
ParserInfo parserInfo = new ParserInfo(request.getContextPath(), request.getLocale());
parserInfo.setWikiUser(user);
parserInfo.setTopicName(topicName);
parserInfo.setUserIpAddress(request.getRemoteAddr());
parserInfo.setVirtualWiki(virtualWiki);
parserInfo.setAllowSectionEdit(sectionEdit);
contents = Utilities.parse(parserInfo, topic.getTopicContent(), topicName);
if (StringUtils.hasText(request.getParameter("highlight"))) {
// search servlet highlights search terms, so add that here
contents = AbstractSearchEngine.highlightHTML(contents, request.getParameter("highlight"));
}
topic.setTopicContent(contents);
if (topic.getTopicType() == Topic.TYPE_IMAGE) {
List fileVersions =
WikiBase.getHandler().getAllWikiFileVersions(virtualWiki, topicName, true);
next.addObject("fileVersions", fileVersions);
}
this.pageInfo.setSpecial(false);
this.pageInfo.setTopicName(topicName);
next.addObject(JAMWikiServlet.PARAMETER_TOPIC_OBJECT, topic);
this.pageInfo.setPageTitle(pageTitle);
}
/**
* Remove a topic from the search index.
*
* @param writer The IndexWriter to use when updating the search index.
* @param topic The topic object that is to be removed from the index.
*/
private void deleteFromIndex(IndexWriter writer, Topic topic) throws IOException {
writer.deleteDocuments(new Term(FIELD_TOPIC_NAME, topic.getName()));
this.resetIndexSearcher(topic.getVirtualWiki());
}
/** Functionality to handle the "Save" button being clicked. */
private void save(HttpServletRequest request, ModelAndView next, WikiPageInfo pageInfo)
throws Exception {
String topicName = WikiUtil.getTopicFromRequest(request);
String virtualWiki = pageInfo.getVirtualWikiName();
Topic topic = loadTopic(virtualWiki, topicName);
Topic lastTopic = WikiBase.getDataHandler().lookupTopic(virtualWiki, topicName, false, null);
if (lastTopic != null
&& !lastTopic.getCurrentVersionId().equals(retrieveLastTopicVersionId(request, topic))) {
// someone else has edited the topic more recently
resolve(request, next, pageInfo);
return;
}
String contents = request.getParameter("contents");
String sectionName = "";
if (!StringUtils.isBlank(request.getParameter("section"))) {
// load section of topic
int section = Integer.valueOf(request.getParameter("section"));
ParserOutput parserOutput = new ParserOutput();
String[] spliceResult =
ParserUtil.parseSplice(
parserOutput,
request.getContextPath(),
request.getLocale(),
virtualWiki,
topicName,
section,
contents);
contents = spliceResult[1];
sectionName = parserOutput.getSectionName();
}
if (contents == null) {
logger.warning("The topic " + topicName + " has no content");
throw new WikiException(new WikiMessage("edit.exception.nocontent", topicName));
}
// strip line feeds
contents = StringUtils.remove(contents, '\r');
String lastTopicContent =
(lastTopic != null) ? StringUtils.remove(lastTopic.getTopicContent(), '\r') : "";
if (lastTopic != null && StringUtils.equals(lastTopicContent, contents)) {
// topic hasn't changed. redirect to prevent user from refreshing and re-submitting
ServletUtil.redirect(next, virtualWiki, topic.getName());
return;
}
String editComment = request.getParameter("editComment");
if (handleSpam(request, next, topicName, contents, editComment)) {
this.loadEdit(request, next, pageInfo, contents, virtualWiki, topicName, false);
return;
}
// parse for signatures and other syntax that should not be saved in raw form
WikiUser user = ServletUtil.currentWikiUser();
ParserInput parserInput = new ParserInput();
parserInput.setContext(request.getContextPath());
parserInput.setLocale(request.getLocale());
parserInput.setWikiUser(user);
parserInput.setTopicName(topicName);
parserInput.setUserDisplay(ServletUtil.getIpAddress(request));
parserInput.setVirtualWiki(virtualWiki);
ParserOutput parserOutput = ParserUtil.parseMetadata(parserInput, contents);
// parse signatures and other values that need to be updated prior to saving
contents = ParserUtil.parseMinimal(parserInput, contents);
topic.setTopicContent(contents);
if (!StringUtils.isBlank(parserOutput.getRedirect())) {
// set up a redirect
topic.setRedirectTo(parserOutput.getRedirect());
topic.setTopicType(TopicType.REDIRECT);
} else if (topic.getTopicType() == TopicType.REDIRECT) {
// no longer a redirect
topic.setRedirectTo(null);
topic.setTopicType(TopicType.ARTICLE);
}
int charactersChanged = StringUtils.length(contents) - StringUtils.length(lastTopicContent);
TopicVersion topicVersion =
new TopicVersion(
user, ServletUtil.getIpAddress(request), editComment, contents, charactersChanged);
if (request.getParameter("minorEdit") != null) {
topicVersion.setEditType(TopicVersion.EDIT_MINOR);
}
WikiBase.getDataHandler()
.writeTopic(topic, topicVersion, parserOutput.getCategories(), parserOutput.getLinks());
// update watchlist
WikiUserDetailsImpl userDetails = ServletUtil.currentUserDetails();
if (!userDetails.hasRole(Role.ROLE_ANONYMOUS)) {
Watchlist watchlist = ServletUtil.currentWatchlist(request, virtualWiki);
boolean watchTopic = (request.getParameter("watchTopic") != null);
if (watchlist.containsTopic(topicName) != watchTopic) {
WikiBase.getDataHandler()
.writeWatchlistEntry(watchlist, virtualWiki, topicName, user.getUserId());
}
}
// redirect to prevent user from refreshing and re-submitting
String target = topic.getName();
if (!StringUtils.isBlank(sectionName)) {
target += "#" + sectionName;
}
ServletUtil.redirect(next, virtualWiki, target);
}
/**
* Utility method used when viewing a topic.
*
* @param request The current servlet request object.
* @param next The current Spring ModelAndView object.
* @param pageInfo The current WikiPageInfo object, which contains information needed for
* rendering the final JSP page.
* @param pageTitle The title of the page being rendered.
* @param topic The Topic object for the topic being displayed.
* @param sectionEdit Set to <code>true</code> if edit links should be displayed for each section
* of the topic.
* @throws Exception Thrown if any error occurs during processing.
*/
protected static void viewTopic(
HttpServletRequest request,
ModelAndView next,
WikiPageInfo pageInfo,
WikiMessage pageTitle,
Topic topic,
boolean sectionEdit)
throws Exception {
// FIXME - what should the default be for topics that don't exist?
if (topic == null) {
throw new WikiException(new WikiMessage("common.exception.notopic"));
}
WikiUtil.validateTopicName(topic.getName());
if (topic.getTopicType() == Topic.TYPE_REDIRECT
&& (request.getParameter("redirect") == null
|| !request.getParameter("redirect").equalsIgnoreCase("no"))) {
Topic child = Utilities.findRedirectedTopic(topic, 0);
if (!child.getName().equals(topic.getName())) {
pageInfo.setRedirectName(topic.getName());
pageTitle = new WikiMessage("topic.title", child.getName());
topic = child;
}
}
String virtualWiki = topic.getVirtualWiki();
String topicName = topic.getName();
WikiUser user = Utilities.currentUser();
if (sectionEdit && !ServletUtil.isEditable(virtualWiki, topicName, user)) {
sectionEdit = false;
}
ParserInput parserInput = new ParserInput();
parserInput.setContext(request.getContextPath());
parserInput.setLocale(request.getLocale());
parserInput.setWikiUser(user);
parserInput.setTopicName(topicName);
parserInput.setUserIpAddress(request.getRemoteAddr());
parserInput.setVirtualWiki(virtualWiki);
parserInput.setAllowSectionEdit(sectionEdit);
ParserDocument parserDocument = new ParserDocument();
String content = Utilities.parse(parserInput, parserDocument, topic.getTopicContent());
// FIXME - the null check should be unnecessary
if (parserDocument != null && parserDocument.getCategories().size() > 0) {
LinkedHashMap categories = new LinkedHashMap();
for (Iterator iterator = parserDocument.getCategories().keySet().iterator();
iterator.hasNext(); ) {
String key = (String) iterator.next();
String value =
key.substring(
NamespaceHandler.NAMESPACE_CATEGORY.length()
+ NamespaceHandler.NAMESPACE_SEPARATOR.length());
categories.put(key, value);
}
next.addObject("categories", categories);
}
topic.setTopicContent(content);
if (topic.getTopicType() == Topic.TYPE_CATEGORY) {
loadCategoryContent(next, virtualWiki, topic.getName());
}
if (topic.getTopicType() == Topic.TYPE_IMAGE || topic.getTopicType() == Topic.TYPE_FILE) {
Collection fileVersions =
WikiBase.getDataHandler().getAllWikiFileVersions(virtualWiki, topicName, true);
for (Iterator iterator = fileVersions.iterator(); iterator.hasNext(); ) {
// update version urls to include web root path
WikiFileVersion fileVersion = (WikiFileVersion) iterator.next();
String url =
FilenameUtils.normalize(
Environment.getValue(Environment.PROP_FILE_DIR_RELATIVE_PATH)
+ "/"
+ fileVersion.getUrl());
url = FilenameUtils.separatorsToUnix(url);
fileVersion.setUrl(url);
}
next.addObject("fileVersions", fileVersions);
if (topic.getTopicType() == Topic.TYPE_IMAGE) {
next.addObject("topicImage", new Boolean(true));
} else {
next.addObject("topicFile", new Boolean(true));
}
}
pageInfo.setSpecial(false);
pageInfo.setTopicName(topicName);
next.addObject(ServletUtil.PARAMETER_TOPIC_OBJECT, topic);
if (pageTitle != null) {
pageInfo.setPageTitle(pageTitle);
}
}
private void writePages(
Writer writer, String virtualWiki, List<String> topicNames, boolean excludeHistory)
throws DataAccessException, IOException, MigrationException {
// note that effort is being made to re-use temporary objects as this
// code can generate an OOM "GC overhead limit exceeded" with HUGE (500MB) topics
// since the garbage collector ends up being invoked excessively.
TopicVersion topicVersion;
Topic topic;
WikiUser user;
// choose 100,000 as an arbitrary max
Pagination pagination = new Pagination(100000, 0);
List<Integer> topicVersionIds;
Map<String, String> textAttributes = new HashMap<String, String>();
textAttributes.put("xml:space", "preserve");
for (String topicName : topicNames) {
topicVersionIds = new ArrayList<Integer>();
topic = WikiBase.getDataHandler().lookupTopic(virtualWiki, topicName, false);
if (topic == null) {
throw new MigrationException(
"Failure while exporting: topic " + topicName + " does not exist");
}
writer.append("\n<page>");
writer.append('\n');
XMLUtil.buildTag(writer, "title", topic.getName(), true);
writer.append('\n');
XMLUtil.buildTag(writer, "id", topic.getTopicId());
if (excludeHistory) {
// only include the most recent version
topicVersionIds.add(topic.getCurrentVersionId());
} else {
// FIXME - changes sorted newest-to-oldest, should be reverse
List<RecentChange> changes =
WikiBase.getDataHandler().getTopicHistory(topic, pagination, true);
for (int i = (changes.size() - 1); i >= 0; i--) {
topicVersionIds.add(changes.get(i).getTopicVersionId());
}
}
for (int topicVersionId : topicVersionIds) {
topicVersion = WikiBase.getDataHandler().lookupTopicVersion(topicVersionId);
writer.append("\n<revision>");
writer.append('\n');
XMLUtil.buildTag(writer, "id", topicVersion.getTopicVersionId());
writer.append('\n');
XMLUtil.buildTag(
writer, "timestamp", this.parseJAMWikiTimestamp(topicVersion.getEditDate()), true);
writer.append("\n<contributor>");
user =
(topicVersion.getAuthorId() != null)
? WikiBase.getDataHandler().lookupWikiUser(topicVersion.getAuthorId())
: null;
if (user != null) {
writer.append('\n');
XMLUtil.buildTag(writer, "username", user.getUsername(), true);
writer.append('\n');
XMLUtil.buildTag(writer, "id", user.getUserId());
} else if (Utilities.isIpAddress(topicVersion.getAuthorDisplay())) {
writer.append('\n');
XMLUtil.buildTag(writer, "ip", topicVersion.getAuthorDisplay(), true);
} else {
writer.append('\n');
XMLUtil.buildTag(writer, "username", topicVersion.getAuthorDisplay(), true);
}
writer.append("\n</contributor>");
writer.append('\n');
XMLUtil.buildTag(writer, "comment", topicVersion.getEditComment(), true);
writer.append('\n');
XMLUtil.buildTag(writer, "text", topicVersion.getVersionContent(), textAttributes, true);
writer.append("\n</revision>");
// explicitly null out temp variables to improve garbage collection and
// avoid OOM "GC overhead limit exceeded" errors on HUGE (500MB) topics
topicVersion = null;
user = null;
}
writer.append("\n</page>");
}
}