public void testParse() throws Exception {
final SyndFeedInput input = new SyndFeedInput();
final SyndFeed feed = input.build(new File(super.getTestFile("xml/custom-tags-example.xml")));
final List<SyndEntry> entries = feed.getEntries();
final SyndEntry entry = entries.get(0);
final CustomTags customTags = (CustomTags) entry.getModule(CustomTags.URI);
final Iterator<CustomTag> it = customTags.getValues().iterator();
while (it.hasNext()) {
final CustomTag tag = it.next();
LOG.debug("{}", tag);
if (tag.getName().equals("language_skills")) {
Assert.assertEquals("Fluent in English and German", tag.getValue());
}
if (tag.getName().equals("prior_experience_years")) {
Assert.assertEquals(new Integer(5), tag.getValue());
} else if (tag.getName().equals("start_date")) {
final Calendar cal = Calendar.getInstance();
cal.setTimeInMillis(0);
cal.set(2005, 10, 15, 0, 0, 0);
Assert.assertEquals(cal.getTime(), tag.getValue());
} else if (tag.getName().equals("test_url")) {
Assert.assertEquals(new URL("http://www.screaming-penguin.com"), tag.getValue());
} else if (tag.getName().equals("test_boolean")) {
Assert.assertEquals(new Boolean(true), tag.getValue());
} else if (tag.getName().equals("test_intUnit")) {
Assert.assertEquals(new IntUnit(25, "horses"), tag.getValue());
} else if (tag.getName().equals("test_floatUnit")) {
Assert.assertEquals(new FloatUnit((float) 2.5, "cows"), tag.getValue());
} else if (tag.getName().equals("test_location")) {
Assert.assertEquals(
new CustomTagImpl.Location("125 Main St, Sometown, GA"), tag.getValue());
} else if (tag.getName().equals("test_dateRange")) {
final Calendar cal = Calendar.getInstance();
cal.setTimeInMillis(0);
cal.set(2005, 06, 04, 20, 0, 0);
final Date start = cal.getTime();
cal.set(2005, 06, 04, 23, 0, 0);
final DateTimeRange dtr = new DateTimeRange(start, cal.getTime());
Assert.assertEquals(dtr, tag.getValue());
}
}
}
public void notifySubscribers(SyndEntry[] entries) {
for (SyndEntry entry : entries) {
for (String destination : subscribers) {
Main.getBot()
.sendMessage(
destination,
String.format(
"["
+ Colors.BOLD
+ "RSS - %s"
+ Colors.NORMAL
+ "]"
+ " ["
+ Colors.BOLD
+ "%s"
+ Colors.NORMAL
+ "]"
+ Colors.BLUE
+ Colors.BOLD
+ " %s"
+ Colors.NORMAL
+ ": "
+ Colors.DARK_GREEN
+ "%s"
+ Colors.NORMAL
+ " ("
+ Colors.OLIVE
+ Colors.BOLD
+ "%s"
+ Colors.NORMAL
+ ")",
title,
entry.getPublishedDate().toString(),
entry.getAuthor(),
entry.getTitle(),
entry.getLink()));
}
}
}
/** 全文输出 */
@At
@Ok("raw:xml")
public String rss() throws IOException, FeedException {
SyndFeed feed = new SyndFeedImpl();
feed.setFeedType("rss_2.0");
String urlbase = conf.get("website.urlbase", "https://nutz.cn");
feed.setLink(urlbase);
feed.setTitle(conf.get("website.title", "Nutz社区"));
feed.setDescription(conf.get("website.description", "一个有爱的社区"));
feed.setAuthor(conf.get("website.author", "wendal"));
feed.setEncoding("UTF-8");
feed.setLanguage("zh-cn");
List<SyndEntry> entries = new ArrayList<SyndEntry>();
SyndEntry entry;
SyndContent description;
List<Topic> list =
dao.query(Topic.class, Cnd.orderBy().desc("createTime"), dao.createPager(1, 10));
for (Topic topic : list) {
dao.fetchLinks(topic, "author");
entry = new SyndEntryImpl();
entry.setTitle(topic.getTitle());
entry.setLink(urlbase + "/yvr/t/" + topic.getId());
entry.setPublishedDate(topic.getCreateTime());
description = new SyndContentImpl();
description.setType("text/html");
description.setValue(Markdowns.toHtml(topic.getContent(), urlbase));
entry.setDescription(description);
entry.setAuthor(topic.getAuthor().getLoginname());
entries.add(entry);
}
feed.setEntries(entries);
if (list.size() > 0) {
feed.setPublishedDate(list.get(0).getCreateTime());
}
SyndFeedOutput output = new SyndFeedOutput();
return output.outputString(feed, true);
}
@SuppressWarnings("unchecked")
public void setPodcastFeedAttributes(Podcast podcast, boolean feedPropertyHasBeenSet)
throws IllegalArgumentException, FeedException, IOException {
SyndFeed syndFeed = null;
if (!feedPropertyHasBeenSet) {
syndFeed = syndFeedService.getSyndFeedForUrl(podcast.getUrl());
podcast.setPodcastFeed(syndFeed);
}
if (syndFeed != null) {
// set DESCRIPTION for podcast - used in search
if (syndFeed.getDescription() != null && !syndFeed.getDescription().equals("")) {
String description = syndFeed.getDescription();
// out of description remove tags if any exist and store also
// short description
String descWithoutTabs = description.replaceAll("\\<[^>]*>", "");
if (descWithoutTabs.length() > MAX_LENGTH_DESCRIPTION) {
podcast.setDescription(descWithoutTabs.substring(0, MAX_LENGTH_DESCRIPTION));
} else {
podcast.setDescription(descWithoutTabs);
}
}
// set TITLE - used in search
String podcastTitle = syndFeed.getTitle();
podcast.setTitle(podcastTitle);
// build the title that will appear in the URL when accessing a
// podcast from the main application
String titleInUrl = podcastTitle.trim().replaceAll("[^a-zA-Z0-9\\-\\s\\.]", "");
titleInUrl = titleInUrl.replaceAll("[\\-| |\\.]+", "-");
if (titleInUrl.length() > TITLE_IN_URL_MAX_LENGTH) {
podcast.setTitleInUrl(titleInUrl.substring(0, TITLE_IN_URL_MAX_LENGTH));
} else {
podcast.setTitleInUrl(titleInUrl);
}
// set author
podcast.setAuthor(syndFeed.getAuthor());
// set COPYRIGHT
podcast.setCopyright(syndFeed.getCopyright());
// set LINK
podcast.setLink(syndFeed.getLink());
// set url link of the podcast's image when selecting the podcast in
// the main application - mostly used through <a
// href="urlOfImageToDisplay"....
SyndImage podcastImage = syndFeed.getImage();
if (null != podcastImage) {
if (podcastImage.getUrl() != null) {
podcast.setUrlOfImageToDisplay(podcastImage.getUrl());
} else if (podcastImage.getLink() != null) {
podcast.setUrlOfImageToDisplay(podcastImage.getLink());
} else {
podcast.setUrlOfImageToDisplay(configBean.get("NO_IMAGE_LOCAL_URL"));
}
} else {
podcast.setUrlOfImageToDisplay(configBean.get("NO_IMAGE_LOCAL_URL"));
}
podcast.setPublicationDate(null); // default value is null, if cannot
// be set
// set url media link of the last episode - this is used when
// generating the ATOM and RSS feeds from the Start page for example
for (SyndEntry entry : (List<SyndEntry>) syndFeed.getEntries()) {
// get the list of enclosures
List<SyndEnclosure> enclosures = (List<SyndEnclosure>) entry.getEnclosures();
if (null != enclosures) {
// if in the enclosure list is a media type (either audio or
// video), this will set as the link of the episode
for (SyndEnclosure enclosure : enclosures) {
if (null != enclosure) {
podcast.setLastEpisodeMediaUrl(enclosure.getUrl());
break;
}
}
}
if (entry.getPublishedDate() == null) {
LOG.warn(
"PodURL["
+ podcast.getUrl()
+ "] - "
+ "COULD NOT SET publication date for podcast, default date 08.01.1983 will be used ");
} else {
podcast.setPublicationDate(entry.getPublishedDate());
}
// first episode in the list is last episode - normally (are
// there any exceptions?? TODO -investigate)
break;
}
}
}
public void setEpisodeAttributes(Episode episode, Podcast podcast, SyndEntry entry) {
// set DESCRIPTION for episode - used in search
if (null != entry.getDescription()) {
String episodeDesc = entry.getDescription().getValue();
// tags are removed from description
String descWithoutTabs = episodeDesc.replaceAll("\\<[^>]*>", "");
// carriage returns are removed from description - for player
String descWithoutEndOfLine = descWithoutTabs.replaceAll("\\n", "");
if (descWithoutEndOfLine.length() > MAX_LENGTH_DESCRIPTION) {
episode.setDescription(descWithoutEndOfLine.substring(0, MAX_LENGTH_DESCRIPTION));
} else {
episode.setDescription(descWithoutEndOfLine);
}
}
// set author
episode.setAuthor(entry.getAuthor());
// set title for episode - used in search
String episodeTitle = entry.getTitle();
if (episodeTitle != null) {
// removes quotes to display properly in player
episodeTitle = episodeTitle.replaceAll("\"", "");
if (episodeTitle.length() > MAX_PERMITTED_TITLE_LENGTH) {
episodeTitle = episodeTitle.substring(0, MAX_PERMITTED_TITLE_LENGTH);
}
episode.setTitle(episodeTitle);
String titleInUrl = episodeTitle.trim().replaceAll("[^a-zA-Z0-9\\-\\s\\.]", "");
titleInUrl = titleInUrl.replaceAll("[\\-| |\\.]+", "-");
if (titleInUrl.length() > TITLE_IN_URL_MAX_LENGTH) {
episode.setTitleInUrl(titleInUrl.substring(0, TITLE_IN_URL_MAX_LENGTH));
} else {
episode.setTitleInUrl(titleInUrl);
}
}
episode.setLink(entry.getLink());
// in the beginning inherit the media type from the podcast
episode.setMediaType(podcast.getMediaType());
// get the list of enclosures
@SuppressWarnings("unchecked")
List<SyndEnclosure> enclosures = (List<SyndEnclosure>) entry.getEnclosures();
List<String> audioMimeTypesList = Arrays.asList(audioMimeTypesArray);
List<String> videoMimeTypesList = Arrays.asList(videoMimeTypesArray);
// set media url for the episode - this will be played in the player
if (null != enclosures) {
// if in the enclosure list is a media type (either audio or video),
// this will set as the link of the episode
for (SyndEnclosure enclosure : enclosures) {
if (null != enclosure) {
episode.setMediaUrl(enclosure.getUrl());
if (enclosure.getLength() >= 0) episode.setLength(enclosure.getLength());
// when adding a new podcast media type is selected for the
// podcast based on an initial view, but it can be that is a
// mixed podcast so both audio
// and video should be considered and in that case PRIORITY
// has the type of the episode if any...
if (null != enclosure.getType()) {
episode.setEnclosureType(enclosure.getType().trim());
if (audioMimeTypesList.contains(enclosure.getType().trim())) {
episode.setMediaType(MediaType.Audio);
break;
}
if (videoMimeTypesList.contains(enclosure.getType().trim())) {
episode.setMediaType(MediaType.Video);
break;
}
}
}
}
} else {
episode.setMediaUrl("noMediaUrl");
}
if (episode.getMediaUrl() == null) {
episode.setMediaUrl("noMediaUrl");
}
if (episode.getMediaUrl() == null || episode.getMediaUrl().equals("noMediaUrl")) {
LOG.warn(
"PodcastId["
+ podcast.getPodcastId()
+ "] - "
+ "COULD NOT SET MEDIA URL - "
+ "epTitle["
+ entry.getTitle()
+ "]"
+ "feed["
+ podcast.getUrl()
+ "]");
}
// set link attribute
episode.setLink(entry.getLink());
episode.setPublicationDate(entry.getPublishedDate());
updatePodcastPublicationDateAndLastMediaUrl(episode, podcast);
if (episode.getPublicationDate() == null) {
LOG.warn(
"PodcastId["
+ podcast.getPodcastId()
+ "] - "
+ "COULD NOT SET publication date "
+ "epTitle["
+ entry.getTitle()
+ "]"
+ "feed["
+ podcast.getUrl()
+ "]");
}
}
private List<Outlink> parseFeed(String url, byte[] content, Metadata parentMetadata)
throws MalformedURLException {
List<Outlink> links = new ArrayList<>();
SyndFeed feed = null;
try (ByteArrayInputStream is = new ByteArrayInputStream(content)) {
SyndFeedInput input = new SyndFeedInput();
feed = input.build(new InputSource(is));
} catch (Exception e) {
LOG.error("Exception parsing feed from DOM {}", url);
return links;
}
URL sURL = new URL(url);
List<SyndEntry> entries = feed.getEntries();
for (SyndEntry entry : entries) {
String targetURL = entry.getLink();
// build an absolute URL
try {
targetURL = URLUtil.resolveURL(sURL, targetURL).toExternalForm();
} catch (MalformedURLException e) {
LOG.debug("MalformedURLException on {}", targetURL);
continue;
}
targetURL = urlFilters.filter(sURL, parentMetadata, targetURL);
if (StringUtils.isBlank(targetURL)) continue;
Outlink newLink = new Outlink(targetURL);
Metadata targetMD = metadataTransfer.getMetaForOutlink(targetURL, url, parentMetadata);
newLink.setMetadata(targetMD);
String title = entry.getTitle();
if (StringUtils.isNotBlank(title)) {
targetMD.setValue("feed.title", title.trim());
}
Date publishedDate = entry.getPublishedDate();
if (publishedDate != null) {
// filter based on the published date
if (filterHoursSincePub != -1) {
Calendar rightNow = Calendar.getInstance();
rightNow.add(Calendar.HOUR, -filterHoursSincePub);
if (publishedDate.before(rightNow.getTime())) {
LOG.info(
"{} has a published date {} which is more than {} hours old",
targetURL,
publishedDate.toString(),
filterHoursSincePub);
continue;
}
}
targetMD.setValue("feed.publishedDate", publishedDate.toString());
}
SyndContent description = entry.getDescription();
if (description != null && StringUtils.isNotBlank(description.getValue())) {
targetMD.setValue("feed.description", description.getValue());
}
links.add(newLink);
}
return links;
}