public FetchedFeed fetch(
String feedUrl,
boolean extractFeedUrlFromHtml,
String lastModified,
String eTag,
Date lastPublishedDate,
String lastContentHash)
throws FeedException, ClientProtocolException, IOException, NotModifiedException {
log.debug("Fetching feed {}", feedUrl);
FetchedFeed fetchedFeed = null;
HttpResult result = getter.getBinary(feedUrl, lastModified, eTag);
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(StringUtils.newStringUtf8(result.getContent()), feedUrl);
if (org.apache.commons.lang.StringUtils.isNotBlank(extractedUrl)) {
result = getter.getBinary(extractedUrl, lastModified, eTag);
feedUrl = extractedUrl;
}
}
byte[] content = result.getContent();
if (content == null) {
throw new IOException("Feed content is empty.");
}
String hash = DigestUtils.sha1Hex(content);
if (lastContentHash != null && hash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException();
}
fetchedFeed = parser.parse(feedUrl, content);
if (lastPublishedDate != null
&& fetchedFeed.getFeed().getLastPublishedDate() != null
&& lastPublishedDate.getTime() == fetchedFeed.getFeed().getLastPublishedDate().getTime()) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException();
}
Feed feed = fetchedFeed.getFeed();
feed.setLastModifiedHeader(result.getLastModifiedSince());
feed.setEtagHeader(FeedUtils.truncate(result.geteTag(), 255));
feed.setLastContentHash(hash);
fetchedFeed.setFetchDuration(result.getDuration());
return fetchedFeed;
}
private byte[] getImage(String url) {
byte[] img = null;
try {
if (StringUtils.isNotBlank(url)) {
int index = Math.max(url.length(), url.lastIndexOf("?"));
url = url.substring(0, index);
String iconUrl =
"http://g.etfv.co/" + URLEncoder.encode(url, "UTF-8") + "?defaulticon=none";
HttpResult result = getter.getBinary(iconUrl);
if (result != null) {
img = result.getContent();
}
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
if (img == null) {
img = getDefaultIcon();
}
return img;
}