public void downloadPost(JDFeedMeFeed feed, JDFeedMePost post, String post_description) {
String link_list_to_download = null;
// check if we have a valid files field
if ((link_list_to_download == null) && (post.hasValidFiles())) {
// just take the files field
link_list_to_download =
extractLinksFromHtml(post.getFiles(), feed.getHoster(), JDFeedMeFeed.HOSTER_EXCLUDE);
}
// no file fields, maybe we have a good link
if ((link_list_to_download == null) && (post.hasValidLink())) {
// try to follow this link
try {
Browser browser = new Browser();
browser.setFollowRedirects(true); // support redirects since some feeds have them
String response = browser.getPage(post.getLink());
link_list_to_download =
extractLinksFromHtml(response, feed.getHoster(), JDFeedMeFeed.HOSTER_EXCLUDE);
} catch (Exception e) {
logger.severe("JDFeedMe could not follow feed item link: " + post.getLink());
}
}
// no good link, try the description
if ((link_list_to_download == null)
&& (post_description != null)
&& (post_description.trim().length() > 0)) {
link_list_to_download =
extractLinksFromHtml(post_description, feed.getHoster(), JDFeedMeFeed.HOSTER_EXCLUDE);
}
// nothing, exit
if (link_list_to_download == null) {
post.setAdded(JDFeedMePost.ADDED_YES_NO_FILES);
return;
}
// JOptionPane.showMessageDialog(new JFrame(), "JDFeedMe says we need to download link:
// "+rssitem.link);
logger.info("JDFeedMe attempting to download: " + link_list_to_download);
// let's download the links.. finally..
boolean anything_downloaded = downloadLinks(link_list_to_download, feed, post);
if (anything_downloaded) {
post.setAdded(JDFeedMePost.ADDED_YES);
gui.notifyPostAddedInOtherFeed(post, feed);
} else post.setAdded(JDFeedMePost.ADDED_YES_NO_FILES);
}
// returns true if downloaded something, false if didn't
private boolean downloadLinks(String linksText, JDFeedMeFeed feed, JDFeedMePost post) {
// make sure we have something to download
if (linksText.trim().length() == 0) return false;
boolean skip_grabber = feed.getSkiplinkgrabber();
boolean autostart = gui.getConfig().getStartdownloads();
// handle a direct download
if (skip_grabber) {
// get all the links from the text
ArrayList<DownloadLink> links = new DistributeData(linksText).findLinks();
// create a new package for the data
FilePackage fp = FilePackage.getInstance();
fp.setName(post.getTitle() + " [JDFeedMe]");
fp.addLinks(links);
// download the package
LinkGrabberController.getInstance().addLinks(links, skip_grabber, autostart);
// restart the downloads if needed
if (autostart) DownloadWatchDog.getInstance().startDownloads();
} else // throw into the link grabber using the old code
{
new DistributeData(linksText, skip_grabber).start();
}
return true;
}
// returns whether this item needs to be added (passes filters) or not
private boolean runFilterOnPost(JDFeedMeFeed feed, JDFeedMePost post, String post_description) {
// see if we even need to run filters
if (!feed.getDoFilters()) return true;
// let's run our filters
if (feed.getFiltersearchtitle() || feed.getFiltersearchdesc()) {
// we need to check something, prepare the checker
FilterChecker filter = new FilterChecker(feed.getFilters());
// check title if needed
if (feed.getFiltersearchtitle()) {
if (filter.match(post.getTitle())) {
if (JDFeedMe.VERBOSE)
logger.info("JDFeedMe new item title: [" + post.getTitle() + "] passed filters");
return true;
}
}
// check description if needed
if (feed.getFiltersearchdesc()) {
if (filter.match(post_description)) {
if (JDFeedMe.VERBOSE)
logger.info("JDFeedMe new item description: [" + post_description + "] passed filters");
return true;
}
}
}
// if here then nothing passed
return false;
}
private String parseFeed(JDFeedMeFeed feed, String timestamp, String content) throws Exception {
String new_timestamp = timestamp;
boolean found_new_posts = false;
// parse the rss xml
RssParser feed_parser = new RssParser(feed);
feed_parser.parseContent(content);
int feed_item_number = 0;
JDFeedMePost post = null;
while ((post = feed_parser.getPost()) != null) {
feed_item_number++;
// get the original description
String post_description = post.getDescription();
// originally we removed the description from the posts since posts are saved in xml and this
// could become large
// new feature: let's do save the description, but make it somewhat shorter (extract links
// from it)
post.setDescription(
extractLinksFromHtml(
post_description, JDFeedMeFeed.HOSTER_ANY_HOSTER, JDFeedMeFeed.HOSTER_EXCLUDE));
// handle the rss item
if (post.isValid()) {
boolean is_new = handlePost(feed, post, post_description, timestamp);
if (is_new) found_new_posts = true;
if (post.isTimestampNewer(new_timestamp)) new_timestamp = post.getTimestamp();
} else {
logger.severe(
"JDFeedMe rss item "
+ Integer.toString(feed_item_number)
+ " is invalid for feed: "
+ feed.getAddress());
}
}
// if found new posts, update the feed
if (found_new_posts) gui.setFeedNewposts(feed, true);
return new_timestamp;
}
private void syncRss() {
// get the feed list
ArrayList<JDFeedMeFeed> feeds = gui.getFeeds();
// go over all the feeds
for (JDFeedMeFeed feed : feeds) {
if (!feed.isEnabled()) continue;
String feed_address = feed.getAddress();
if (feed_address.length() == 0) continue;
// sync each feed
String response = "not downloaded yet";
try {
// get the feed last update timestamp
String timestamp = null;
if ((feed.getTimestamp() != null) && (feed.getTimestamp().length() > 0))
timestamp = feed.getTimestamp();
logger.info("JDFeedMe syncing feed: " + feed_address + " [" + timestamp + "]");
gui.setFeedStatus(feed, JDFeedMeFeed.STATUS_RUNNING);
// get the feed content from the web
Browser browser = new Browser();
browser.setFollowRedirects(true); // support redirects since some feeds have them
response = browser.getPage(feed_address);
// parse the feed
String new_timestamp = parseFeed(feed, timestamp, response);
// set the new timestamp if needed
if ((new_timestamp != null) && (new_timestamp != timestamp)) {
gui.setFeedTimestamp(feed, new_timestamp);
}
// all ok
gui.setFeedStatus(feed, JDFeedMeFeed.STATUS_OK);
} catch (Exception e) {
// shorten the response so we can show it in the log
int max_reponse_length = 1000;
String response_short = response;
if (response_short != null) {
if (response_short.length() > max_reponse_length) {
response_short =
response_short.substring(0, max_reponse_length / 2)
+ "..."
+ response_short.substring(response_short.length() - max_reponse_length / 2);
}
}
e.printStackTrace();
logger.severe(
"JDFeedMe cannot sync feed: "
+ feed_address
+ " ("
+ e.toString()
+ "), feed response: "
+ response_short);
gui.setFeedStatus(feed, JDFeedMeFeed.STATUS_ERROR);
}
}
// save our xml with any updates from what we just downloaded
gui.saveFeeds(); // maybe do this only if we had updates
gui.savePosts(); // maybe do this only if we had updates
}
