public FetchedFeed fetch(
      String feedUrl,
      boolean extractFeedUrlFromHtml,
      String lastModified,
      String eTag,
      Date lastPublishedDate,
      String lastContentHash)
      throws FeedException, ClientProtocolException, IOException, NotModifiedException {
    log.debug("Fetching feed {}", feedUrl);
    FetchedFeed fetchedFeed = null;

    HttpResult result = getter.getBinary(feedUrl, lastModified, eTag);
    if (extractFeedUrlFromHtml) {
      String extractedUrl = extractFeedUrl(StringUtils.newStringUtf8(result.getContent()), feedUrl);
      if (org.apache.commons.lang.StringUtils.isNotBlank(extractedUrl)) {
        result = getter.getBinary(extractedUrl, lastModified, eTag);
        feedUrl = extractedUrl;
      }
    }
    byte[] content = result.getContent();

    if (content == null) {
      throw new IOException("Feed content is empty.");
    }

    String hash = DigestUtils.sha1Hex(content);
    if (lastContentHash != null && hash != null && lastContentHash.equals(hash)) {
      log.debug("content hash not modified: {}", feedUrl);
      throw new NotModifiedException();
    }

    fetchedFeed = parser.parse(feedUrl, content);

    if (lastPublishedDate != null
        && fetchedFeed.getFeed().getLastPublishedDate() != null
        && lastPublishedDate.getTime() == fetchedFeed.getFeed().getLastPublishedDate().getTime()) {
      log.debug("publishedDate not modified: {}", feedUrl);
      throw new NotModifiedException();
    }

    Feed feed = fetchedFeed.getFeed();
    feed.setLastModifiedHeader(result.getLastModifiedSince());
    feed.setEtagHeader(FeedUtils.truncate(result.geteTag(), 255));
    feed.setLastContentHash(hash);
    fetchedFeed.setFetchDuration(result.getDuration());
    return fetchedFeed;
  }
Example #2
0
  private byte[] getImage(String url) {
    byte[] img = null;
    try {
      if (StringUtils.isNotBlank(url)) {
        int index = Math.max(url.length(), url.lastIndexOf("?"));
        url = url.substring(0, index);

        String iconUrl =
            "http://g.etfv.co/" + URLEncoder.encode(url, "UTF-8") + "?defaulticon=none";
        HttpResult result = getter.getBinary(iconUrl);
        if (result != null) {
          img = result.getContent();
        }
      }
    } catch (Exception e) {
      log.error(e.getMessage(), e);
    }
    if (img == null) {
      img = getDefaultIcon();
    }
    return img;
  }