public static List<RssUrlBean> getRssUrlBeanListFromPage(int rssCompo_id, String url) { List<RssUrlBean> rubList = new ArrayList<RssUrlBean>(); if (url.equals("")) return rubList; try { URL feedUrl = new URL(url); // SyndFeedInput:从远程读到xml结构的内容转成SyndFeedImpl实例 SyndFeedInput input = new SyndFeedInput(); // rome按SyndFeed类型生成rss和atom的实例, // SyndFeed是rss和atom实现类SyndFeedImpl的接口 SyndFeed syndFeed = input.build(new XmlReader(feedUrl)); List<SyndEntry> entryList = syndFeed.getEntries(); for (SyndEntry entry : entryList) { RssUrlBean rub = new RssUrlBean(); rub.setRssCompo_id(rssCompo_id); rub.setTitle(entry.getTitle()); rub.setLink(entry.getUri()); rub.setPublishedDate(CommonUtil.getStandardDate(entry.getPublishedDate().toLocaleString())); rub.setDescription(entry.getDescription().getValue()); if (entry.getUpdatedDate() != null) rub.setUpdatedDate(CommonUtil.getStandardDate(entry.getUpdatedDate().toLocaleString())); rub.setAuthors(entry.getAuthor()); rubList.add(rub); } } catch (Exception ex) { ex.printStackTrace(); } return rubList; }
public Item(String source, SyndEntry entry) { this( source, entry.getLink(), entry.getTitle(), entry.getDescription().getValue(), entry.getPublishedDate(), entry.getEnclosures()); }
/** * A helper function to print out the contents of an entry to log.trace * * @param entry */ public static void printEntry(SyndEntry entry) { if (!log.isTraceEnabled()) { return; } StringBuffer pr = new StringBuffer( "URI: " + entry.getUri() + "\n" + "Title: " + entry.getTitle() + "\n" + "\n" + "Date: " + entry.getPublishedDate() + "\n" + "Modified: " + entry.getUpdatedDate() + "\n"); pr.append("Creators: \n"); for (SyndPerson author : FeedHelper.getAuthors(entry)) { pr.append(" - " + author.getName() + "\n"); } pr.append("Links: \n"); for (SyndLink link : FeedHelper.getLinks(entry)) { pr.append(" - " + link.getTitle() + ": " + link.getHref() + "\n"); } SyndContent description = entry.getDescription(); if (description != null) { pr.append("\nDescription(" + description.getType() + "): " + description.getValue()); } pr.append("Contents: \n"); for (SyndContent content : FeedHelper.getContents(entry)) { pr.append(" Type: " + content.getType()); pr.append(" Body: " + content.getValue()); try { pr.append( " Body (Plain text): " + PlainTextExtractor.getPlainText(content.getType(), content.getValue())); } catch (ParserException e) { pr.append("Failed to parse content"); } } pr.append("Categories: \n"); for (SyndCategory category : FeedHelper.getCategories(entry)) { pr.append(category.getName() + "(" + category.getTaxonomyUri() + ")"); } log.trace(pr.toString()); }
@SuppressWarnings("unchecked") public void test() throws Exception { final SyndFeedInput input = new SyndFeedInput(true); final SyndFeed feed = input.build(new File("c:\\temp\\google.xml")); logger.debug("Successfully parsed the RSS feed"); logger.debug("Author = " + feed.getAuthors()); logger.debug("Categories = " + feed.getCategories()); final List<SyndEntry> entries = feed.getEntries(); for (final SyndEntry entry : entries) { logger.debug("Title = " + StringEscapeUtils.unescapeHtml(entry.getTitle())); logger.debug( "Description = " + StringEscapeUtils.unescapeHtml(entry.getDescription().getValue())); logger.debug(entry.getUri()); logger.debug("Updated date = " + entry.getUpdatedDate()); logger.debug("Published date = " + entry.getPublishedDate()); logger.debug("===================================================="); } }
private Article mapArticle(SyndEntry syndEntry) { StringBuilder sb = new StringBuilder(); for (Object obj : syndEntry.getContents()) { if (!(obj instanceof SyndContent)) { continue; } SyndContent syndContent = (SyndContent) obj; sb.append(syndContent.getValue()); } return Article.builder() .num(0) .title(syndEntry.getTitle()) .content(sb.toString()) .description(syndEntry.getDescription().getValue()) .author(syndEntry.getAuthor()) .image("") .writtenDate(syndEntry.getPublishedDate()) .build(); }
/** @see com.elia.rssexample.data.NewsDao */ @SuppressWarnings("unchecked") public List<NewsItem> getNewsList() { // TODO: exception handling log.trace("Enter getNewsList()."); List<NewsItem> newsList = new ArrayList<NewsItem>(); XmlReader reader = null; try { for (String rssUrl : rssUrlList) { reader = new XmlReader(new URL(rssUrl)); SyndFeed feed = new SyndFeedInput().build(reader); for (SyndEntry entry : (List<SyndEntry>) feed.getEntries()) { NewsItem item = new NewsItem(); item.setTitle(entry.getTitle()); item.setDescription(entry.getDescription().getValue()); item.setLink(entry.getLink()); item.setPublished(entry.getPublishedDate()); newsList.add(item); } } } catch (Exception e) { log.error("Error reading feed.", e); } finally { try { reader.close(); } catch (IOException e) { log.warn("Unable to close xml reader.", e); } } return newsList; }
// @Transactional // @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) public void importEntry( final SyndFeed feed, final SyndEntry entry, final Set<KiWiUriResource> types, final Set<ContentItem> tags, User user, final Collection<ContentItem> output) { if (user == null && entry.getAuthor() != null && !"".equals(entry.getAuthor())) { if (userService.userExists(entry.getAuthor())) { user = userService.getUserByLogin(entry.getAuthor()); } else { // user = userService.createUser(entry.getAuthor()); /* In my opinion, it is not ok to create a user entity * without asking the person if he/she wants to be * created and persisted in the KiWi dataset. * Thus I'm changing the user to 'anonymous', * if he/she is'nt registered with the same nick that * is given in the rss entry. */ user = userService.getUserByLogin("anonymous"); kiwiEntityManager.persist(user); } } log.debug("feed entry: #0 (#1)", entry.getTitle(), entry.getUri()); // create a new content item and copy all data from the feed entry ContentItem item; if (entry.getLink() != null) { item = contentItemService.createExternContentItem(entry.getLink()); } else if (entry.getUri() != null) { try { // try parsing URI; if it is not valid, URI uri = new URI(entry.getUri()); item = contentItemService.createExternContentItem(entry.getUri()); } catch (URISyntaxException e) { item = contentItemService.createExternContentItem(feed.getLink() + "#" + entry.getUri()); } } else { item = contentItemService.createContentItem(); } contentItemService.updateTitle(item, entry.getTitle()); if (feed.getLanguage() != null) item.setLanguage(new Locale(feed.getLanguage())); if (entry.getPublishedDate() != null) { item.setCreated(entry.getPublishedDate()); item.setModified(entry.getPublishedDate()); } if (entry.getUpdatedDate() != null) { if (entry.getPublishedDate() == null) { item.setCreated(entry.getUpdatedDate()); } item.setModified(entry.getUpdatedDate()); } item.setAuthor(user); // read feed content and set it as item's text content List<SyndContent> contents = entry.getContents(); if (contents.size() == 1) { log.debug("using RSS content section provided by item"); contentItemService.updateTextContentItem(item, "<p>" + contents.get(0).getValue() + "</p>"); } else if (contents.size() > 1) { log.warn("feed entry contained more than one content section"); contentItemService.updateTextContentItem(item, "<p>" + contents.get(0).getValue() + "</p>"); } else if (contents.size() == 0) { if (entry.getDescription() != null && entry.getDescription().getValue() != null) { log.debug("using RSS description as no content section was available"); contentItemService.updateTextContentItem( item, "<p>" + entry.getDescription().getValue() + "</p>"); } } // save before tagging contentItemService.saveContentItem(item); // read feed categories and use them as tags for (SyndCategory cat : (List<SyndCategory>) entry.getCategories()) { ContentItem _cat; if (!taggingService.hasTag(item, cat.getName())) { if (cat.getTaxonomyUri() != null) { _cat = contentItemService.getContentItemByUri(cat.getTaxonomyUri()); if (_cat == null) { _cat = contentItemService.createExternContentItem(cat.getTaxonomyUri()); contentItemService.updateTitle(_cat, cat.getName()); _cat.setAuthor(user); contentItemService.saveContentItem(_cat); } taggingService.createTagging(cat.getName(), item, _cat, user); } else { _cat = contentItemService.getContentItemByTitle(cat.getName()); if (_cat == null) { _cat = contentItemService.createContentItem(); contentItemService.updateTitle(_cat, cat.getName()); _cat.setAuthor(user); contentItemService.saveContentItem(_cat); } taggingService.createTagging(cat.getName(), item, _cat, user); } } } // scan for Twitter-style hash tags in title (e.g. #kiwiknows, see KIWI-622) Matcher m_hashtag = p_hashtag.matcher(entry.getTitle()); while (m_hashtag.find()) { String tag_label = m_hashtag.group(1); if (!taggingService.hasTag(item, tag_label)) { ContentItem tag = contentItemService.getContentItemByTitle(tag_label); if (tag == null) { tag = contentItemService.createContentItem(); contentItemService.updateTitle(tag, tag_label); tag.setAuthor(user); contentItemService.saveContentItem(tag); } taggingService.createTagging(tag_label, item, tag, user); } } // check for geo information GeoRSSModule geoRSSModule = GeoRSSUtils.getGeoRSS(entry); if (geoRSSModule != null && geoRSSModule.getPosition() != null) { POI poi = kiwiEntityManager.createFacade(item, POI.class); poi.setLatitude(geoRSSModule.getPosition().getLatitude()); poi.setLongitude(geoRSSModule.getPosition().getLongitude()); kiwiEntityManager.persist(poi); } // check for media information MediaEntryModule mediaModule = (MediaEntryModule) entry.getModule(MediaModule.URI); if (mediaModule != null) { MediaContent[] media = mediaModule.getMediaContents(); if (media.length > 0) { MediaContent m = media[0]; if (m.getReference() instanceof UrlReference) { URL url = ((UrlReference) m.getReference()).getUrl(); String type = m.getType(); String name = url.getFile(); if (name.lastIndexOf("/") > 0) { name = name.substring(name.lastIndexOf("/") + 1); } log.debug("importing media data from URL #0", url.toString()); try { InputStream is = url.openStream(); ByteArrayOutputStream bout = new ByteArrayOutputStream(); int c; while ((c = is.read()) != -1) { bout.write(c); } byte[] data = bout.toByteArray(); contentItemService.updateMediaContentItem(item, data, type, name); is.close(); bout.close(); } catch (IOException ex) { log.error("error importing media content from RSS stream"); } } else { log.info("RSS importer can only import media with URL references"); } } else { log.warn("media module found without content"); } Category[] cats = mediaModule.getMetadata().getCategories(); for (Category cat : cats) { ContentItem _cat; String label = cat.getLabel() != null ? cat.getLabel() : cat.getValue(); if (!taggingService.hasTag(item, label)) { if (cat.getScheme() != null) { _cat = contentItemService.getContentItemByUri(cat.getScheme() + cat.getValue()); if (_cat == null) { _cat = contentItemService.createExternContentItem(cat.getScheme() + cat.getValue()); contentItemService.updateTitle(_cat, label); _cat.setAuthor(user); contentItemService.saveContentItem(_cat); } taggingService.createTagging(label, item, _cat, user); } else { _cat = contentItemService.getContentItemByTitle(label); if (_cat == null) { _cat = contentItemService.createContentItem(); contentItemService.updateTitle(_cat, label); _cat.setAuthor(user); contentItemService.saveContentItem(_cat); } taggingService.createTagging(label, item, _cat, user); } } } } // add parameter categories as tags for (ContentItem tag : tags) { if (!taggingService.hasTag(item, tag.getTitle())) { taggingService.createTagging(tag.getTitle(), item, tag, user); } } // add parameter types as types for (KiWiUriResource type : types) { item.addType(type); } // add kiwi:FeedPost type item.addType(tripleStore.createUriResource(Constants.NS_KIWI_CORE + "FeedPost")); /* the flush is necessary, because CIs or tags will * otherwise be created multiple times when they * appear more than once in one RSS feed */ entityManager.flush(); log.debug("imported content item '#0' with URI '#1'", item.getTitle(), item.getResource()); }
private DocumentPojo buildDocument( SyndEntry entry, SourcePojo source, LinkedList<String> duplicateSources) { String tmpURL = this.cleanUrlStart(entry.getLink().toString()); // (can't return null because called from code which checks this) // create the feed pojo DocumentPojo doc = new DocumentPojo(); doc.setUrl(tmpURL); doc.setCreated(new Date()); doc.setModified(new Date()); // Strip out html if it is present if (entry.getTitle() != null) doc.setTitle(entry.getTitle().replaceAll("\\<.*?\\>", "").trim()); if (entry.getDescription() != null) doc.setDescription(entry.getDescription().getValue().replaceAll("\\<.*?\\>", "").trim()); if (entry.getPublishedDate() != null) { doc.setPublishedDate(entry.getPublishedDate()); } else { doc.setPublishedDate(new Date()); } // Clone from an existing source if we can: if (!duplicateSources.isEmpty() && (null == doc.getUpdateId())) { // (can't duplicate updating document) doc.setDuplicateFrom(duplicateSources.getFirst()); } // GeoRSS GeoRSSModule geoRSSModule = GeoRSSUtils.getGeoRSS(entry); // currently does not handle <georss:circle> if (null != geoRSSModule) { if (null != geoRSSModule.getPosition()) { double lat = geoRSSModule.getPosition().getLatitude(); double lon = geoRSSModule.getPosition().getLongitude(); GeoPojo gp = new GeoPojo(); gp.lat = lat; gp.lon = lon; doc.setDocGeo(gp); } if (null != geoRSSModule.getGeometry()) { AbstractGeometry ag = geoRSSModule.getGeometry(); if (ag.getClass().equals(new LineString().getClass())) { // <georss:line> LineString ls = ((LineString) geoRSSModule.getGeometry()); double latAvg = 0.0; double lonAvg = 0.0; int length = ls.getPositionList().size(); for (int i = 0; i < length; i++) { latAvg += ls.getPositionList().getLatitude(i); lonAvg += ls.getPositionList().getLongitude(i); } latAvg = latAvg / length; lonAvg = lonAvg / length; GeoPojo gp = new GeoPojo(); gp.lat = latAvg; gp.lon = lonAvg; doc.setDocGeo(gp); } else if (ag.getClass().equals(new Polygon().getClass())) // <georss:polygon> { Polygon poly = ((Polygon) geoRSSModule.getGeometry()); AbstractRing ar = poly.getExterior(); LinearRing lr = (LinearRing) ar; double latAvg = 0.0; double lonAvg = 0.0; int length = lr.getPositionList().size(); for (int i = 0; i < length; i++) { latAvg += lr.getPositionList().getLatitude(i); lonAvg += lr.getPositionList().getLongitude(i); } latAvg = latAvg / length; lonAvg = lonAvg / length; GeoPojo gp = new GeoPojo(); gp.lat = latAvg; gp.lon = lonAvg; doc.setDocGeo(gp); } else if (ag.getClass().equals(new Envelope().getClass())) { // <georss:box> Envelope env = ((Envelope) geoRSSModule.getGeometry()); double latAvg = (env.getMaxLatitude() + env.getMinLatitude()) / 2; double lonAvg = (env.getMaxLongitude() + env.getMinLongitude()) / 2; GeoPojo gp = new GeoPojo(); gp.lat = latAvg; gp.lon = lonAvg; doc.setDocGeo(gp); } } } // end if GeoRSS // Arbitrary other metadata: if (null != entry.getForeignMarkup()) { JSONObject rssMetadata = new JSONObject(); @SuppressWarnings("unchecked") List<Element> fms = (List<Element>) entry.getForeignMarkup(); for (Element fm : fms) { try { JSONObject subObj = XML.toJSONObject(new XMLOutputter().outputString(fm)); if (1 == subObj.length()) { for (String name : JSONObject.getNames(subObj)) { rssMetadata.put(name, subObj.get(name)); } } else { // (this will never happen in practice?) rssMetadata.put(fm.getName(), subObj); } } catch (JSONException e) { } // (do nothing just carry on) } if (!fms.isEmpty()) { doc.addToMetadata( "_FEED_METADATA_", XmlToMetadataParser.convertJsonObjectToLinkedHashMap(rssMetadata)); } } // TESTED (longs converted to string, eg edgar:assistantDirector from // "http.www.sec.gov.archives.edgar.usgaap.rss.xml") return doc; }
public Date getPublishedDate() { return entry.getPublishedDate(); }
public int compare(SyndEntry o1, SyndEntry o2) { int pubDateCompare = o2.getPublishedDate().compareTo(o1.getPublishedDate()); return pubDateCompare; }
// build a SubscriptionEntry from Rome SyndEntry and SyndFeed private SubscriptionEntry buildEntry(SyndEntry romeEntry) { // if we don't have a permalink then we can't continue if (romeEntry.getLink() == null) { return null; } SubscriptionEntry newEntry = new SubscriptionEntry(); newEntry.setTitle(romeEntry.getTitle()); newEntry.setPermalink(romeEntry.getLink()); // Play some games to get the author DCModule entrydc = (DCModule) romeEntry.getModule(DCModule.URI); if (romeEntry.getAuthor() != null) { newEntry.setAuthor(romeEntry.getAuthor()); } else { newEntry.setAuthor(entrydc.getCreator()); // use <dc:creator> } // Play some games to get the updated date if (romeEntry.getUpdatedDate() != null) { newEntry.setUpdateTime(new Timestamp(romeEntry.getUpdatedDate().getTime())); } // TODO: should we set a default update time here? // And more games getting publish date if (romeEntry.getPublishedDate() != null) { newEntry.setPubTime(new Timestamp(romeEntry.getPublishedDate().getTime())); // use <pubDate> } else if (entrydc != null && entrydc.getDate() != null) { newEntry.setPubTime(new Timestamp(entrydc.getDate().getTime())); // use <dc:date> } else { newEntry.setPubTime(newEntry.getUpdateTime()); } // get content and unescape if it is 'text/plain' if (romeEntry.getContents().size() > 0) { SyndContent content = (SyndContent) romeEntry.getContents().get(0); if (content != null && content.getType().equals("text/plain")) { newEntry.setText(StringEscapeUtils.unescapeHtml(content.getValue())); } else if (content != null) { newEntry.setText(content.getValue()); } } // no content, try summary if (newEntry.getText() == null || newEntry.getText().trim().length() == 0) { if (romeEntry.getDescription() != null) { newEntry.setText(romeEntry.getDescription().getValue()); } } // copy categories if (romeEntry.getCategories().size() > 0) { List list = new ArrayList(); Iterator cats = romeEntry.getCategories().iterator(); while (cats.hasNext()) { SyndCategory cat = (SyndCategory) cats.next(); list.add(cat.getName()); } newEntry.setCategoriesString(list); } return newEntry; }