// returns whether this item needs to be added (passes filters) or not
private boolean runFilterOnPost(JDFeedMeFeed feed, JDFeedMePost post, String post_description) {
// see if we even need to run filters
if (!feed.getDoFilters()) return true;
// let's run our filters
if (feed.getFiltersearchtitle() || feed.getFiltersearchdesc()) {
// we need to check something, prepare the checker
FilterChecker filter = new FilterChecker(feed.getFilters());
// check title if needed
if (feed.getFiltersearchtitle()) {
if (filter.match(post.getTitle())) {
if (JDFeedMe.VERBOSE)
logger.info("JDFeedMe new item title: [" + post.getTitle() + "] passed filters");
return true;
}
}
// check description if needed
if (feed.getFiltersearchdesc()) {
if (filter.match(post_description)) {
if (JDFeedMe.VERBOSE)
logger.info("JDFeedMe new item description: [" + post_description + "] passed filters");
return true;
}
}
}
// if here then nothing passed
return false;
}
// returns true if downloaded something, false if didn't
private boolean downloadLinks(String linksText, JDFeedMeFeed feed, JDFeedMePost post) {
// make sure we have something to download
if (linksText.trim().length() == 0) return false;
boolean skip_grabber = feed.getSkiplinkgrabber();
boolean autostart = gui.getConfig().getStartdownloads();
// handle a direct download
if (skip_grabber) {
// get all the links from the text
ArrayList<DownloadLink> links = new DistributeData(linksText).findLinks();
// create a new package for the data
FilePackage fp = FilePackage.getInstance();
fp.setName(post.getTitle() + " [JDFeedMe]");
fp.addLinks(links);
// download the package
LinkGrabberController.getInstance().addLinks(links, skip_grabber, autostart);
// restart the downloads if needed
if (autostart) DownloadWatchDog.getInstance().startDownloads();
} else // throw into the link grabber using the old code
{
new DistributeData(linksText, skip_grabber).start();
}
return true;
}
public void downloadPost(JDFeedMeFeed feed, JDFeedMePost post, String post_description) {
String link_list_to_download = null;
// check if we have a valid files field
if ((link_list_to_download == null) && (post.hasValidFiles())) {
// just take the files field
link_list_to_download =
extractLinksFromHtml(post.getFiles(), feed.getHoster(), JDFeedMeFeed.HOSTER_EXCLUDE);
}
// no file fields, maybe we have a good link
if ((link_list_to_download == null) && (post.hasValidLink())) {
// try to follow this link
try {
Browser browser = new Browser();
browser.setFollowRedirects(true); // support redirects since some feeds have them
String response = browser.getPage(post.getLink());
link_list_to_download =
extractLinksFromHtml(response, feed.getHoster(), JDFeedMeFeed.HOSTER_EXCLUDE);
} catch (Exception e) {
logger.severe("JDFeedMe could not follow feed item link: " + post.getLink());
}
}
// no good link, try the description
if ((link_list_to_download == null)
&& (post_description != null)
&& (post_description.trim().length() > 0)) {
link_list_to_download =
extractLinksFromHtml(post_description, feed.getHoster(), JDFeedMeFeed.HOSTER_EXCLUDE);
}
// nothing, exit
if (link_list_to_download == null) {
post.setAdded(JDFeedMePost.ADDED_YES_NO_FILES);
return;
}
// JOptionPane.showMessageDialog(new JFrame(), "JDFeedMe says we need to download link:
// "+rssitem.link);
logger.info("JDFeedMe attempting to download: " + link_list_to_download);
// let's download the links.. finally..
boolean anything_downloaded = downloadLinks(link_list_to_download, feed, post);
if (anything_downloaded) {
post.setAdded(JDFeedMePost.ADDED_YES);
gui.notifyPostAddedInOtherFeed(post, feed);
} else post.setAdded(JDFeedMePost.ADDED_YES_NO_FILES);
}
private String parseFeed(JDFeedMeFeed feed, String timestamp, String content) throws Exception {
String new_timestamp = timestamp;
boolean found_new_posts = false;
// parse the rss xml
RssParser feed_parser = new RssParser(feed);
feed_parser.parseContent(content);
int feed_item_number = 0;
JDFeedMePost post = null;
while ((post = feed_parser.getPost()) != null) {
feed_item_number++;
// get the original description
String post_description = post.getDescription();
// originally we removed the description from the posts since posts are saved in xml and this
// could become large
// new feature: let's do save the description, but make it somewhat shorter (extract links
// from it)
post.setDescription(
extractLinksFromHtml(
post_description, JDFeedMeFeed.HOSTER_ANY_HOSTER, JDFeedMeFeed.HOSTER_EXCLUDE));
// handle the rss item
if (post.isValid()) {
boolean is_new = handlePost(feed, post, post_description, timestamp);
if (is_new) found_new_posts = true;
if (post.isTimestampNewer(new_timestamp)) new_timestamp = post.getTimestamp();
} else {
logger.severe(
"JDFeedMe rss item "
+ Integer.toString(feed_item_number)
+ " is invalid for feed: "
+ feed.getAddress());
}
}
// if found new posts, update the feed
if (found_new_posts) gui.setFeedNewposts(feed, true);
return new_timestamp;
}
// return true if the post is new, false if old
private boolean handlePost(
JDFeedMeFeed feed, JDFeedMePost post, String post_description, String timestamp) {
// make sure this rss item is indeed newer than what we have
if (post.isTimestampNewer(timestamp)) {
if (JDFeedMe.VERBOSE)
logger.info("JDFeedMe found new item with timestamp: [" + post.getTimestamp() + "]");
post.setNewpost(true);
// check for filters on this item (see if it passes download filters)
boolean need_to_add = runFilterOnPost(feed, post, post_description);
// if we don't need to add, we can return now
if (!need_to_add) {
if (JDFeedMe.VERBOSE)
logger.info(
"JDFeedMe new item title: ["
+ post.getTitle()
+ "] description: ["
+ post_description
+ "] did not pass filters");
post.setAdded(JDFeedMePost.ADDED_NO);
} else {
// start processing this rss item - we're going to download it
downloadPost(feed, post, post_description);
}
// update the post history
gui.addPostToFeed(post, feed);
return true;
} else {
if (JDFeedMe.VERBOSE)
logger.info("JDFeedMe ignoring item with old timestamp: [" + post.getTimestamp() + "]");
return false;
}
}
