@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
if (checklistNotificationsUri == null) {
return;
}
boolean started =
!HibernateUtil.getSessionFactory().getCurrentSession().getTransaction().isActive();
if (started) {
HibernateUtil.getSessionFactory().getCurrentSession().getTransaction().begin();
}
try {
URI researchObjectUri = (URI) context.getMergedJobDataMap().get(RESEARCH_OBJECT_URI);
SyndFeedInput input = new SyndFeedInput();
URI requestedUri = createQueryUri(getTheLastFeedDate(researchObjectUri), researchObjectUri);
try {
context.setResult(input.build(new XmlReader(requestedUri.toURL())));
} catch (IllegalArgumentException | FeedException | IOException e) {
LOGGER.error("Can't get the feed " + requestedUri.toString());
}
} finally {
if (started) {
HibernateUtil.getSessionFactory().getCurrentSession().getTransaction().commit();
}
}
}
@Override
public List<GmailMessage> getUnreadMessages() {
final List<GmailMessage> messages = new ArrayList<GmailMessage>();
HttpGmailConnection c = null;
try {
// for ROME properties loader
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
c = getGmailConnection();
c.setUrl(gmailFeedUrl);
final URLConnection con = c.openConnection();
final SyndFeedInput feedInput = new SyndFeedInput();
final SyndFeed gmail = feedInput.build(new XmlReader(con));
for (final Object entry : gmail.getEntries()) {
if (entry instanceof SyndEntry) {
messages.add(new RssGmailMessage((SyndEntry) entry));
}
}
if (log.isDebugEnabled()) {
log.debug("Got " + messages.size() + " new messages.");
}
} catch (final Exception e) {
throw new GmailException("Failed getting unread messages", e);
}
return messages;
}
public static List<RssUrlBean> getRssUrlBeanListFromPage(int rssCompo_id, String url) {
List<RssUrlBean> rubList = new ArrayList<RssUrlBean>();
if (url.equals("")) return rubList;
try {
URL feedUrl = new URL(url);
// SyndFeedInput:从远程读到xml结构的内容转成SyndFeedImpl实例
SyndFeedInput input = new SyndFeedInput();
// rome按SyndFeed类型生成rss和atom的实例,
// SyndFeed是rss和atom实现类SyndFeedImpl的接口
SyndFeed syndFeed = input.build(new XmlReader(feedUrl));
List<SyndEntry> entryList = syndFeed.getEntries();
for (SyndEntry entry : entryList) {
RssUrlBean rub = new RssUrlBean();
rub.setRssCompo_id(rssCompo_id);
rub.setTitle(entry.getTitle());
rub.setLink(entry.getUri());
rub.setPublishedDate(CommonUtil.getStandardDate(entry.getPublishedDate().toLocaleString()));
rub.setDescription(entry.getDescription().getValue());
if (entry.getUpdatedDate() != null)
rub.setUpdatedDate(CommonUtil.getStandardDate(entry.getUpdatedDate().toLocaleString()));
rub.setAuthors(entry.getAuthor());
rubList.add(rub);
}
} catch (Exception ex) {
ex.printStackTrace();
}
return rubList;
}
public void testReadAtom1() throws FeedException {
SyndFeedInput input = new SyndFeedInput();
SyndFeed result =
input.build(
new InputSource(getClass().getResourceAsStream("/be/hikage/xml/rome/atom_1.0.xml")));
assertEquals("RSS Veille Techno", result.getTitle());
assertEquals("http://svn.cyg.be/", result.getLink());
assertEquals(1, result.getEntries().size());
}
/**
* Returns the feed for the given URL.
*
* @param url
* @return
* @throws IOException
* @throws FeedException
*/
private SyndFeed getFeed(String url) {
try {
URL feedSource = new URL(url);
SyndFeedInput input = new SyndFeedInput();
SyndFeed feed = input.build(new XmlReader(feedSource));
// LOG.info("Requested weather info " + url);
return feed;
} catch (Exception e) {
LOG.error("Error reading weather RSS stream: " + e.getMessage(), e);
}
return null;
}
@IPSJexlMethod(
description = "Returns a ROME SyndFeed instance for the given URL",
params = {@IPSJexlParam(name = "url", description = "the item GUID")})
public SyndFeed getFeed(String urlString)
throws IllegalArgumentException, FeedException, IOException {
URL feedUrl = new URL(urlString);
SyndFeedInput input = new SyndFeedInput();
log.debug("Requesting feed from " + feedUrl);
SyndFeed feed = input.build(new XmlReader(feedUrl));
return feed;
}
/**
* Read a SyndFeed from the
*
* @return
* @throws Exception
*/
public SyndFeed read() {
try {
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
InputStream inputStream = getContent();
Document doc = docBuilder.parse(inputStream);
doc.getDocumentElement().normalize();
SyndFeedInput input = new SyndFeedInput();
SyndFeed feed = input.build(doc);
return feed;
} catch (Exception e) {
LOG.error("Failed to read RSS feed");
throw new RuntimeException(e);
}
}
/** {@inheritDoc} */
public SyndFeed fetchFeed(
final String inFeedUrl,
final Map<String, String> inHttpHeaders,
final String inProxyHost,
final String inProxyPort,
final int inTimeout)
throws IOException, ParserConfigurationException, FeedException, SAXException {
DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
domFactory.setNamespaceAware(true);
Document doc =
documentFetcher.fetchDocument(
inFeedUrl, inHttpHeaders, inProxyHost, inProxyPort, inTimeout, domFactory);
SyndFeedInput input = new SyndFeedInput();
return input.build(doc);
}
public void run() {
try {
feed = input.build(new InputStreamReader(new URL(feedUrl).openStream()));
} catch (Exception e) {
e.printStackTrace();
}
}
@SuppressWarnings("unchecked")
public void test() throws Exception {
final SyndFeedInput input = new SyndFeedInput(true);
final SyndFeed feed = input.build(new File("c:\\temp\\google.xml"));
logger.debug("Successfully parsed the RSS feed");
logger.debug("Author = " + feed.getAuthors());
logger.debug("Categories = " + feed.getCategories());
final List<SyndEntry> entries = feed.getEntries();
for (final SyndEntry entry : entries) {
logger.debug("Title = " + StringEscapeUtils.unescapeHtml(entry.getTitle()));
logger.debug(
"Description = " + StringEscapeUtils.unescapeHtml(entry.getDescription().getValue()));
logger.debug(entry.getUri());
logger.debug("Updated date = " + entry.getUpdatedDate());
logger.debug("Published date = " + entry.getPublishedDate());
logger.debug("====================================================");
}
}
/** Test of parse method, of class com.totsp.xml.syndication.itunes.ITunesParser. */
public void testParse() throws Exception {
File feed = new File(this.getTestFile("/test/xml/leshow.xml"));
SyndFeedInput input = new SyndFeedInput();
SyndFeed syndfeed = input.build(new XmlReader(feed.toURL()));
Module module = syndfeed.getModule(AbstractITunesObject.URI);
FeedInformationImpl feedInfo = (FeedInformationImpl) module;
assertEquals("owner", "Harry Shearer", feedInfo.getOwnerName());
assertEquals("email", "", feedInfo.getOwnerEmailAddress());
assertEquals(
"image",
"http://a1.phobos.apple.com/Music/y2005/m06/d26/h21/mcdrrifv.jpg",
feedInfo.getImage().toExternalForm());
assertEquals("category", "Comedy", ((Category) feedInfo.getCategories().get(0)).getName());
assertEquals(
"summary",
"A weekly, hour-long romp through the worlds of media, politics, sports and show business, leavened with an eclectic mix of mysterious music, hosted by Harry Shearer.",
feedInfo.getSummary());
List entries = syndfeed.getEntries();
Iterator it = entries.iterator();
while (it.hasNext()) {
SyndEntry entry = (SyndEntry) it.next();
EntryInformationImpl entryInfo =
(EntryInformationImpl) entry.getModule(AbstractITunesObject.URI);
System.out.println(entryInfo);
}
feed = new File(this.getTestFile("/test/xml/rsr.xml"));
syndfeed = input.build(new XmlReader(feed.toURL()));
entries = syndfeed.getEntries();
it = entries.iterator();
while (it.hasNext()) {
SyndEntry entry = (SyndEntry) it.next();
EntryInformationImpl entryInfo =
(EntryInformationImpl) entry.getModule(AbstractITunesObject.URI);
System.out.println(entryInfo.getDuration());
}
}
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
logger.debug("HomeServlet.doGet(");
logger.debug("Retrieving yahoo news feed");
URL url = new URL("http://rss.news.yahoo.com/rss/tech");
SyndFeedInput syndFeedInput = new SyndFeedInput();
SyndFeed syndFeed = null;
try {
syndFeed = syndFeedInput.build(new XmlReader(url));
} catch (IllegalArgumentException e) {
logger.error("", e);
} catch (FeedException e) {
logger.error("", e);
}
logger.debug("Forwarding to home.jsp");
req.setAttribute("syndFeed", syndFeed);
homeJsp.forward(req, resp);
}
/**
* Import data from the input stream provided as argument into the KiWi database.
*
* @param is the input stream from which to read the data
* @param types the set of types to associate with each generated content item
* @param tags the set of content items to use as tags
* @param user the user to use as author of all imported data
*/
@Override
public int importData(
InputStream is,
String format,
Set<KiWiUriResource> types,
Set<ContentItem> tags,
User user,
Collection<ContentItem> output) {
SyndFeedInput input = new SyndFeedInput();
try {
SyndFeed feed = input.build(new XmlReader(is));
return importData(feed, types, tags, user, output);
} catch (FeedException ex) {
log.error("RSS/Atom feed could not be parsed", ex);
} catch (IOException ex) {
log.error("I/O error while building feed from input stream source", ex);
}
return 0;
}
/**
* Import data from the reader provided as argument into the KiWi database.
*
* @param reader the reader from which to read the data
* @param types the set of types to associate with each generated content item
* @param tags the set of content items to use as tags
* @param user the user to use as author of all imported data
*/
@Override
public int importData(
Reader reader,
String format,
final Set<KiWiUriResource> types,
final Set<ContentItem> tags,
final User user,
final Collection<ContentItem> output) {
SyndFeedInput input = new SyndFeedInput();
int count = 0;
try {
final SyndFeed feed = input.build(reader);
Identity.setSecurityEnabled(false);
count = importData(feed, types, tags, user, output);
} catch (FeedException ex) {
log.error("RSS/Atom feed could not be parsed", ex);
}
return count;
}
@SuppressWarnings("unchecked")
@Trigger("!buzz")
@Help("Fetches one of the latest posts from jeanmarcmorandini.com")
public List<String> getLatestBuzz() {
List<String> toReturn = new ArrayList<String>();
try {
URL url = new URL("http://www.jeanmarcmorandini.com/rss.php");
SyndFeedInput input = new SyndFeedInput();
SyndFeed rss = input.build(new XmlReader(url));
Iterator<SyndEntry> it = rss.getEntries().iterator();
String message = null;
while (it.hasNext()) {
SyndEntry item = it.next();
String guid = item.getUri();
RSSFeed buzz = dao.findByGUID(guid);
if (buzz == null) {
buzz = new RSSFeed();
buzz.setGuid(item.getUri());
dao.save(buzz);
String urlBitly = utilsService.bitly(item.getLink());
String content = Jsoup.parse(item.getDescription().getValue()).select("p").get(0).text();
message = IRCUtils.bold("EXCLU!") + " " + item.getTitle() + " - " + urlBitly;
toReturn.add(message);
toReturn.add(content);
break;
}
}
if (message == null) {
toReturn.add("Pas d'exclus pour le moment.");
}
} catch (Exception e) {
LOG.handle(e);
}
return toReturn;
}
@Override
public List<Article> read(URL url) {
try {
List<Article> articles = new ArrayList<>();
SyndFeedInput input = new SyndFeedInput();
SyndFeed syndFeed = input.build(new InputStreamReader(url.openStream()));
for (Object obj : syndFeed.getEntries()) {
if (!(obj instanceof SyndEntry)) {
continue;
}
SyndEntry syndEntry = (SyndEntry) obj;
articles.add(mapArticle(syndEntry));
}
return articles;
} catch (FeedException | IOException e) {
throw new RuntimeException("피드를 읽을 수 업습니다. " + e.getMessage(), e);
}
}
@SuppressWarnings("unchecked")
public static ArrayList<SyndEntry> getSyndEntrys(String link) {
ArrayList<SyndEntry> listOfSyndEntrys = new ArrayList<SyndEntry>();
try {
URL url = new URL(link);
HttpURLConnection httpcon = (HttpURLConnection) url.openConnection();
// Reading the feed
SyndFeedInput input = new SyndFeedInput();
SyndFeed feed = input.build(new XmlReader(httpcon));
List<SyndEntry> entries = feed.getEntries();
Iterator<SyndEntry> itEntries = entries.iterator();
SyndEntry entry;
while (itEntries.hasNext()) {
entry = itEntries.next();
listOfSyndEntrys.add(entry);
}
return listOfSyndEntrys;
} catch (Exception e) {
System.out.println(e.getMessage());
}
return listOfSyndEntrys;
}
private double getTemperatureFromRSS() {
double result = -1;
// Here we pretend to be the google bot to fake out User-Agent
// sniffing programs.
try {
tempFeedURL = new URL("http://www.hive13.org/isOpen/RSS.php?temp=0");
URLConnection conn = tempFeedURL.openConnection();
conn.setRequestProperty("User-Agent", BotProperties.getInstance().getUserAgentString());
// Create a feed off of the URL and get the latest news.
SyndFeedInput input = new SyndFeedInput();
SyndFeed feed = input.build(new XmlReader(conn));
// Get the feed's list of entries
@SuppressWarnings("unchecked")
List<SyndEntry> entryList = feed.getEntries();
Collections.sort(entryList, new SyndEntryComparator());
if (entryList.size() > 0) {
// We have found an RSS feed at least, is it formatted
// correctly?
result = Double.parseDouble(entryList.get(0).getTitle());
} else {
// We have not found an RSS feed for the hackerspace
// tempreature.
}
} catch (IOException e) {
e.printStackTrace();
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (FeedException e) {
e.printStackTrace();
}
return result;
}
public Feed(String urlString)
throws MalformedURLException, IOException, IllegalArgumentException, FeedException {
this.feedURL = new URL(urlString);
feedContent = input.build(new XmlReader(feedURL));
}
// Logs a new ATOM entry
public static synchronized void addATOMEntry(
String title, String link, String description, File atomFile, String context) {
try {
if (atomFile.exists()) {
// System.out.println("ATOM file found!");
/** Namespace URI for content:encoded elements */
String CONTENT_NS = "http://www.w3.org/2005/Atom";
/** Parses RSS or Atom to instantiate a SyndFeed. */
SyndFeedInput input = new SyndFeedInput();
/** Transforms SyndFeed to RSS or Atom XML. */
SyndFeedOutput output = new SyndFeedOutput();
// Load the feed, regardless of RSS or Atom type
SyndFeed feed = input.build(new XmlReader(atomFile));
// Set the output format of the feed
feed.setFeedType("atom_1.0");
List<SyndEntry> items = feed.getEntries();
int numItems = items.size();
if (numItems > 9) {
items.remove(0);
feed.setEntries(items);
}
SyndEntry newItem = new SyndEntryImpl();
newItem.setTitle(title);
newItem.setLink(link);
newItem.setUri(link);
SyndContent desc = new SyndContentImpl();
desc.setType("text/html");
desc.setValue(description);
newItem.setDescription(desc);
desc.setType("text/html");
newItem.setPublishedDate(new java.util.Date());
List<SyndCategory> categories = new ArrayList<SyndCategory>();
if (CommonConfiguration.getProperty("htmlTitle", context) != null) {
SyndCategory category2 = new SyndCategoryImpl();
category2.setName(CommonConfiguration.getProperty("htmlTitle", context));
categories.add(category2);
}
newItem.setCategories(categories);
if (CommonConfiguration.getProperty("htmlAuthor", context) != null) {
newItem.setAuthor(CommonConfiguration.getProperty("htmlAuthor", context));
}
items.add(newItem);
feed.setEntries(items);
feed.setPublishedDate(new java.util.Date());
FileWriter writer = new FileWriter(atomFile);
output.output(feed, writer);
writer.toString();
}
} catch (IOException ioe) {
System.out.println("ERROR: Could not find the ATOM file.");
ioe.printStackTrace();
} catch (Exception e) {
System.out.println("Unknown exception trying to add an entry to the ATOM file.");
e.printStackTrace();
}
}
package com.soobo.rss;
public Vector<SyndEntry> crawl() {
if (feedid == 1301) {
int z = 0;
z++;
}
SyndFeedInput input = new SyndFeedInput();
XmlReader reader = null;
Vector<SyndEntry> ret = new Vector<SyndEntry>();
Document doc;
try {
/*
* Document doc = Jsoup.parse(feedurl,10000);
* getLogger().info(doc.toString());
*/
SyndFeed feed = null;
try {
feed = input.build(reader = new XmlReader(feedurl));
} catch (Exception fe) {
try {
System.err.println("from url: " + feedurl);
fe.printStackTrace();
doc = Jsoup.parse(feedurl, 10000);
feed = input.build(new StringReader(doc.toString()));
getLogger().info("Could fix it woith jsoup");
} catch (ExceptionInInitializerError ed) {
System.err.println("from url: " + feedurl);
ed.printStackTrace();
try {
URL url = feedurl;
String feedpage = FeedCrawler.readPage(url);
feedpage =
feedpage
.replaceAll("\\&ldquo;", "\"")
.replaceAll("“", "\"")
.replaceAll("\\&rdquo;", "\"")
.replaceAll("”", "\"");
System.out.println(feedpage);
// feedpage=feedpage.replaceAll("\\&ldquo;", "\"");
// reader=new XmlReader(new InputSource(new
// StringReader(feedpage)).getCharacterStream());
feed = input.build(new InputSource(new StringReader(feedpage)));
getLogger().info("Could fix it with complicatedreader");
} catch (URISyntaxException e) {
// TODO Auto-generated catch block
System.err.println("from url: " + feedurl);
e.printStackTrace();
System.out.println("URL does not work: " + feedurl);
} catch (FeedException e) {
// TODO Auto-generated catch block
System.err.println("from url: " + feedurl);
e.printStackTrace();
System.out.println("URL does not work: " + feedurl);
} catch (Exception ex) {
System.err.println("from url: " + feedurl);
ex.printStackTrace();
System.out.println("URL does not work: " + feedurl);
}
}
}
for (Iterator<SyndEntry> i = feed.getEntries().iterator(); i.hasNext(); ) {
SyndEntry entry = (SyndEntry) i.next();
System.out.println("\t INFOR: Entry\t" + entry.getUri());
if (!entry.getUri().startsWith("http")) {
System.out.println("\t ERROR: \t" + entry.getLink());
System.out.println("\t DEBUG: \t" + feedurl);
System.out.println("\t FIX: \t");
if (domain.length() > 0) {
entry.setLink(domain + entry.getLink());
entry.setUri(domain + entry.getUri());
ret.add(entry);
}
} else ret.add(entry);
getLogger().info(entry.getTitle());
}
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
getLogger().info("Feed error: " + feedurl);
getLogger().info("try with jsoup");
System.err.println("from url: " + feedurl);
e.printStackTrace();
try {
doc = Jsoup.parse(feedurl, 10000);
} catch (IOException e1) {
// TODO Auto-generated catch block
System.err.println("from url: " + feedurl);
e1.printStackTrace();
}
} catch (FeedException fe) {
getLogger().info("Even jsoup did not work: " + feedurl);
System.err.println("from url: " + feedurl);
fe.printStackTrace();
} catch (IOException e) {
getLogger().info("Feed error: " + feedurl);
getLogger().info("try with jsoup");
// TODO Auto-generated catch block
System.err.println("from url: " + feedurl);
e.printStackTrace();
try {
doc = Jsoup.parse(feedurl, 10000);
} catch (IOException e2) {
// TODO Auto-generated catch block
System.err.println("from url: " + feedurl);
e2.printStackTrace();
try {
getLogger().info("try with jsoup");
doc = Jsoup.parse(feedurl, 10000);
getLogger().info(doc.toString());
} catch (IOException e1) {
// TODO Auto-generated catch block
getLogger().info("once again, jsoup cant do it");
System.err.println("from url: " + feedurl);
e1.printStackTrace();
}
}
} finally {
if (reader != null)
try {
reader.close();
} catch (IOException eg) {
getLogger().info("Feed error: " + feedurl);
// TODO Auto-generated catch block
System.err.println("from url: " + feedurl);
eg.printStackTrace();
}
}
return ret;
}
/**
* Generate an ebook from an RSS DOM Document.
*
* @param url The URL from where the Document was fetched (used only to set the author metadata)
* @param doc The DOM Document of the feed.
* @return An ebook.
* @throws IllegalArgumentException
* @throws FeedException
* @throws IOException
*/
private static Book createBookFromFeed(URL url, Document doc, List<Keyword> keywords)
throws IllegalArgumentException, FeedException, IOException {
Book book = new Book();
// start parsing our feed and have the above onItem methods called
SyndFeedInput input = new SyndFeedInput();
SyndFeed feed = input.build(doc);
System.out.println(feed);
// Set the title
book.getMetadata().addTitle(feed.getTitle());
// Add an Author
String author = feed.getAuthor();
if (author == null || "".equals(author.trim())) {
author = url.getHost();
}
book.getMetadata().addAuthor(new Author(author));
if (feed.getPublishedDate() != null) {
book.getMetadata().addDate(new nl.siegmann.epublib.domain.Date(feed.getPublishedDate()));
}
if (feed.getDescription() != null) {
book.getMetadata().addDescription(feed.getDescription());
}
if (feed.getCopyright() != null) {
book.getMetadata().getRights().add(feed.getCopyright());
}
// Set cover image - This has never worked.
// if (feed.getImage() != null) {
// System.out.println("There is an image for the feed");
// Promise<HttpResponse> futureImgResponse =
// WS.url(feed.getImage().getUrl()).getAsync();
// HttpResponse imgResponse = await(futureImgResponse);
// System.out.println("Content-type: " + imgResponse.getContentType());
// if (imgResponse.getContentType().startsWith("image/")) {
// String extension =
// imgResponse.getContentType().substring("image/".length());
// InputStream imageStream = imgResponse.getStream();
// book.getMetadata().setCoverImage(new Resource(imageStream, "cover." +
// extension));
// System.out.println("Using default cover");
// imageStream =
// VirtualFile.fromRelativePath("assets/cover.png").inputstream();
// if (imageStream != null) {
// System.out.println("Using default cover");
// book.getMetadata().setCoverImage(new Resource(imageStream,
// "cover.png"));
// } else {
// System.out.println("Could not load default cover");
// }
// }
// }
int entryNumber = 0;
List<SyndEntry> entries = feed.getEntries();
for (SyndEntry entry : entries) {
if (matchesKeyword(entry, keywords)) {
StringBuilder title = new StringBuilder(100);
if (entry.getTitle() != null) {
title.append(entry.getTitle());
}
if (entry.getAuthor() != null) {
title.append(" - ").append(entry.getAuthor());
}
StringBuilder content = new StringBuilder();
// Add title inside text
content.append("<h2>").append(title).append("</h2>");
if (entry.getDescription() != null) {
SyndContent syndContent = (SyndContent) entry.getDescription();
if (!syndContent.getType().contains("html")) {
content.append("<pre>\n");
}
content.append(syndContent.getValue());
if (!syndContent.getType().contains("html")) {
content.append("\n</pre>");
}
content.append("<hr/>");
}
if (entry.getContents().size() > 0) {
SyndContent syndContent = (SyndContent) entry.getContents().get(0);
if (!syndContent.getType().contains("html")) {
content.append("<pre>\n");
}
content.append(syndContent.getValue());
if (!syndContent.getType().contains("html")) {
content.append("\n</pre>");
}
}
String strContent = clean(content.toString());
// Add Chapter
try {
entryNumber++;
book.addSection(
title.toString(),
new Resource(new StringReader(strContent), "entry" + entryNumber + ".xhtml"));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
return book;
}
