// returns whether this item needs to be added (passes filters) or not private boolean runFilterOnPost(JDFeedMeFeed feed, JDFeedMePost post, String post_description) { // see if we even need to run filters if (!feed.getDoFilters()) return true; // let's run our filters if (feed.getFiltersearchtitle() || feed.getFiltersearchdesc()) { // we need to check something, prepare the checker FilterChecker filter = new FilterChecker(feed.getFilters()); // check title if needed if (feed.getFiltersearchtitle()) { if (filter.match(post.getTitle())) { if (JDFeedMe.VERBOSE) logger.info("JDFeedMe new item title: [" + post.getTitle() + "] passed filters"); return true; } } // check description if needed if (feed.getFiltersearchdesc()) { if (filter.match(post_description)) { if (JDFeedMe.VERBOSE) logger.info("JDFeedMe new item description: [" + post_description + "] passed filters"); return true; } } } // if here then nothing passed return false; }
// returns true if downloaded something, false if didn't private boolean downloadLinks(String linksText, JDFeedMeFeed feed, JDFeedMePost post) { // make sure we have something to download if (linksText.trim().length() == 0) return false; boolean skip_grabber = feed.getSkiplinkgrabber(); boolean autostart = gui.getConfig().getStartdownloads(); // handle a direct download if (skip_grabber) { // get all the links from the text ArrayList<DownloadLink> links = new DistributeData(linksText).findLinks(); // create a new package for the data FilePackage fp = FilePackage.getInstance(); fp.setName(post.getTitle() + " [JDFeedMe]"); fp.addLinks(links); // download the package LinkGrabberController.getInstance().addLinks(links, skip_grabber, autostart); // restart the downloads if needed if (autostart) DownloadWatchDog.getInstance().startDownloads(); } else // throw into the link grabber using the old code { new DistributeData(linksText, skip_grabber).start(); } return true; }
public void downloadPost(JDFeedMeFeed feed, JDFeedMePost post, String post_description) { String link_list_to_download = null; // check if we have a valid files field if ((link_list_to_download == null) && (post.hasValidFiles())) { // just take the files field link_list_to_download = extractLinksFromHtml(post.getFiles(), feed.getHoster(), JDFeedMeFeed.HOSTER_EXCLUDE); } // no file fields, maybe we have a good link if ((link_list_to_download == null) && (post.hasValidLink())) { // try to follow this link try { Browser browser = new Browser(); browser.setFollowRedirects(true); // support redirects since some feeds have them String response = browser.getPage(post.getLink()); link_list_to_download = extractLinksFromHtml(response, feed.getHoster(), JDFeedMeFeed.HOSTER_EXCLUDE); } catch (Exception e) { logger.severe("JDFeedMe could not follow feed item link: " + post.getLink()); } } // no good link, try the description if ((link_list_to_download == null) && (post_description != null) && (post_description.trim().length() > 0)) { link_list_to_download = extractLinksFromHtml(post_description, feed.getHoster(), JDFeedMeFeed.HOSTER_EXCLUDE); } // nothing, exit if (link_list_to_download == null) { post.setAdded(JDFeedMePost.ADDED_YES_NO_FILES); return; } // JOptionPane.showMessageDialog(new JFrame(), "JDFeedMe says we need to download link: // "+rssitem.link); logger.info("JDFeedMe attempting to download: " + link_list_to_download); // let's download the links.. finally.. boolean anything_downloaded = downloadLinks(link_list_to_download, feed, post); if (anything_downloaded) { post.setAdded(JDFeedMePost.ADDED_YES); gui.notifyPostAddedInOtherFeed(post, feed); } else post.setAdded(JDFeedMePost.ADDED_YES_NO_FILES); }
private String parseFeed(JDFeedMeFeed feed, String timestamp, String content) throws Exception { String new_timestamp = timestamp; boolean found_new_posts = false; // parse the rss xml RssParser feed_parser = new RssParser(feed); feed_parser.parseContent(content); int feed_item_number = 0; JDFeedMePost post = null; while ((post = feed_parser.getPost()) != null) { feed_item_number++; // get the original description String post_description = post.getDescription(); // originally we removed the description from the posts since posts are saved in xml and this // could become large // new feature: let's do save the description, but make it somewhat shorter (extract links // from it) post.setDescription( extractLinksFromHtml( post_description, JDFeedMeFeed.HOSTER_ANY_HOSTER, JDFeedMeFeed.HOSTER_EXCLUDE)); // handle the rss item if (post.isValid()) { boolean is_new = handlePost(feed, post, post_description, timestamp); if (is_new) found_new_posts = true; if (post.isTimestampNewer(new_timestamp)) new_timestamp = post.getTimestamp(); } else { logger.severe( "JDFeedMe rss item " + Integer.toString(feed_item_number) + " is invalid for feed: " + feed.getAddress()); } } // if found new posts, update the feed if (found_new_posts) gui.setFeedNewposts(feed, true); return new_timestamp; }
// return true if the post is new, false if old private boolean handlePost( JDFeedMeFeed feed, JDFeedMePost post, String post_description, String timestamp) { // make sure this rss item is indeed newer than what we have if (post.isTimestampNewer(timestamp)) { if (JDFeedMe.VERBOSE) logger.info("JDFeedMe found new item with timestamp: [" + post.getTimestamp() + "]"); post.setNewpost(true); // check for filters on this item (see if it passes download filters) boolean need_to_add = runFilterOnPost(feed, post, post_description); // if we don't need to add, we can return now if (!need_to_add) { if (JDFeedMe.VERBOSE) logger.info( "JDFeedMe new item title: [" + post.getTitle() + "] description: [" + post_description + "] did not pass filters"); post.setAdded(JDFeedMePost.ADDED_NO); } else { // start processing this rss item - we're going to download it downloadPost(feed, post, post_description); } // update the post history gui.addPostToFeed(post, feed); return true; } else { if (JDFeedMe.VERBOSE) logger.info("JDFeedMe ignoring item with old timestamp: [" + post.getTimestamp() + "]"); return false; } }