Пример #1
0
  public void testReadAtom1() throws FeedException {
    SyndFeedInput input = new SyndFeedInput();

    SyndFeed result =
        input.build(
            new InputSource(getClass().getResourceAsStream("/be/hikage/xml/rome/atom_1.0.xml")));

    assertEquals("RSS Veille Techno", result.getTitle());
    assertEquals("http://svn.cyg.be/", result.getLink());
    assertEquals(1, result.getEntries().size());
  }
  @SuppressWarnings("unchecked")
  private void validateLinksInFeeds(SyndFeed feed) {
    Assert.assertTrue("Feed link is wrong", feed.getLink().startsWith(this.getBaseNexusUrl()));

    List<SyndEntry> entries = feed.getEntries();

    for (SyndEntry syndEntry : entries) {
      Assert.assertNotNull("Feed item link is empty.", syndEntry.getLink());
      Assert.assertTrue(
          "Feed item link is wrong, is: " + syndEntry.getLink(),
          syndEntry.getLink().startsWith(this.getBaseNexusUrl()));
    }
  }
  protected WireFeed createRealFeed(String type, SyndFeed syndFeed) {
    Channel channel = new Channel(type);
    channel.setModules(ModuleUtils.cloneModules(syndFeed.getModules()));

    channel.setEncoding(syndFeed.getEncoding());

    channel.setTitle(syndFeed.getTitle());
    channel.setLink(syndFeed.getLink());
    channel.setDescription(syndFeed.getDescription());
    SyndImage sImage = syndFeed.getImage();
    if (sImage != null) {
      channel.setImage(createRSSImage(sImage));
    }

    List sEntries = syndFeed.getEntries();
    if (sEntries != null) {
      channel.setItems(createRSSItems(sEntries));
    }
    return channel;
  }
Пример #4
0
  //	@Transactional
  //	@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
  public void importEntry(
      final SyndFeed feed,
      final SyndEntry entry,
      final Set<KiWiUriResource> types,
      final Set<ContentItem> tags,
      User user,
      final Collection<ContentItem> output) {
    if (user == null && entry.getAuthor() != null && !"".equals(entry.getAuthor())) {
      if (userService.userExists(entry.getAuthor())) {
        user = userService.getUserByLogin(entry.getAuthor());
      } else {

        //				user = userService.createUser(entry.getAuthor());
        /* In my opinion, it is not ok to create a user entity
         * without asking the person if he/she wants to be
         * created and persisted in the KiWi dataset.
         * Thus I'm changing the user to 'anonymous',
         * if he/she is'nt registered with the same nick that
         * is given in the rss entry.
         */
        user = userService.getUserByLogin("anonymous");
        kiwiEntityManager.persist(user);
      }
    }

    log.debug("feed entry: #0 (#1)", entry.getTitle(), entry.getUri());

    // create a new content item and copy all data from the feed entry
    ContentItem item;
    if (entry.getLink() != null) {
      item = contentItemService.createExternContentItem(entry.getLink());
    } else if (entry.getUri() != null) {
      try {
        // try parsing URI; if it is not valid,
        URI uri = new URI(entry.getUri());
        item = contentItemService.createExternContentItem(entry.getUri());
      } catch (URISyntaxException e) {
        item = contentItemService.createExternContentItem(feed.getLink() + "#" + entry.getUri());
      }
    } else {
      item = contentItemService.createContentItem();
    }
    contentItemService.updateTitle(item, entry.getTitle());

    if (feed.getLanguage() != null) item.setLanguage(new Locale(feed.getLanguage()));

    if (entry.getPublishedDate() != null) {
      item.setCreated(entry.getPublishedDate());
      item.setModified(entry.getPublishedDate());
    }

    if (entry.getUpdatedDate() != null) {
      if (entry.getPublishedDate() == null) {
        item.setCreated(entry.getUpdatedDate());
      }
      item.setModified(entry.getUpdatedDate());
    }

    item.setAuthor(user);

    // read feed content and set it as item's text content
    List<SyndContent> contents = entry.getContents();
    if (contents.size() == 1) {
      log.debug("using RSS content section provided by item");
      contentItemService.updateTextContentItem(item, "<p>" + contents.get(0).getValue() + "</p>");
    } else if (contents.size() > 1) {
      log.warn("feed entry contained more than one content section");
      contentItemService.updateTextContentItem(item, "<p>" + contents.get(0).getValue() + "</p>");
    } else if (contents.size() == 0) {
      if (entry.getDescription() != null && entry.getDescription().getValue() != null) {
        log.debug("using RSS description as no content section was available");
        contentItemService.updateTextContentItem(
            item, "<p>" + entry.getDescription().getValue() + "</p>");
      }
    }

    // save before tagging
    contentItemService.saveContentItem(item);

    // read feed categories and use them as tags
    for (SyndCategory cat : (List<SyndCategory>) entry.getCategories()) {
      ContentItem _cat;
      if (!taggingService.hasTag(item, cat.getName())) {
        if (cat.getTaxonomyUri() != null) {
          _cat = contentItemService.getContentItemByUri(cat.getTaxonomyUri());
          if (_cat == null) {
            _cat = contentItemService.createExternContentItem(cat.getTaxonomyUri());
            contentItemService.updateTitle(_cat, cat.getName());
            _cat.setAuthor(user);
            contentItemService.saveContentItem(_cat);
          }
          taggingService.createTagging(cat.getName(), item, _cat, user);
        } else {
          _cat = contentItemService.getContentItemByTitle(cat.getName());
          if (_cat == null) {
            _cat = contentItemService.createContentItem();
            contentItemService.updateTitle(_cat, cat.getName());
            _cat.setAuthor(user);
            contentItemService.saveContentItem(_cat);
          }
          taggingService.createTagging(cat.getName(), item, _cat, user);
        }
      }
    }
    // scan for Twitter-style hash tags in title (e.g. #kiwiknows, see KIWI-622)
    Matcher m_hashtag = p_hashtag.matcher(entry.getTitle());
    while (m_hashtag.find()) {
      String tag_label = m_hashtag.group(1);
      if (!taggingService.hasTag(item, tag_label)) {
        ContentItem tag = contentItemService.getContentItemByTitle(tag_label);
        if (tag == null) {
          tag = contentItemService.createContentItem();
          contentItemService.updateTitle(tag, tag_label);
          tag.setAuthor(user);
          contentItemService.saveContentItem(tag);
        }
        taggingService.createTagging(tag_label, item, tag, user);
      }
    }

    // check for geo information
    GeoRSSModule geoRSSModule = GeoRSSUtils.getGeoRSS(entry);
    if (geoRSSModule != null && geoRSSModule.getPosition() != null) {
      POI poi = kiwiEntityManager.createFacade(item, POI.class);
      poi.setLatitude(geoRSSModule.getPosition().getLatitude());
      poi.setLongitude(geoRSSModule.getPosition().getLongitude());
      kiwiEntityManager.persist(poi);
    }

    // check for media information
    MediaEntryModule mediaModule = (MediaEntryModule) entry.getModule(MediaModule.URI);
    if (mediaModule != null) {
      MediaContent[] media = mediaModule.getMediaContents();
      if (media.length > 0) {
        MediaContent m = media[0];
        if (m.getReference() instanceof UrlReference) {
          URL url = ((UrlReference) m.getReference()).getUrl();

          String type = m.getType();
          String name = url.getFile();
          if (name.lastIndexOf("/") > 0) {
            name = name.substring(name.lastIndexOf("/") + 1);
          }

          log.debug("importing media data from URL #0", url.toString());

          try {
            InputStream is = url.openStream();

            ByteArrayOutputStream bout = new ByteArrayOutputStream();

            int c;
            while ((c = is.read()) != -1) {
              bout.write(c);
            }

            byte[] data = bout.toByteArray();

            contentItemService.updateMediaContentItem(item, data, type, name);

            is.close();
            bout.close();
          } catch (IOException ex) {
            log.error("error importing media content from RSS stream");
          }
        } else {
          log.info("RSS importer can only import media with URL references");
        }
      } else {
        log.warn("media module found without content");
      }

      Category[] cats = mediaModule.getMetadata().getCategories();
      for (Category cat : cats) {
        ContentItem _cat;

        String label = cat.getLabel() != null ? cat.getLabel() : cat.getValue();

        if (!taggingService.hasTag(item, label)) {
          if (cat.getScheme() != null) {
            _cat = contentItemService.getContentItemByUri(cat.getScheme() + cat.getValue());
            if (_cat == null) {
              _cat = contentItemService.createExternContentItem(cat.getScheme() + cat.getValue());
              contentItemService.updateTitle(_cat, label);
              _cat.setAuthor(user);
              contentItemService.saveContentItem(_cat);
            }
            taggingService.createTagging(label, item, _cat, user);
          } else {
            _cat = contentItemService.getContentItemByTitle(label);
            if (_cat == null) {
              _cat = contentItemService.createContentItem();
              contentItemService.updateTitle(_cat, label);
              _cat.setAuthor(user);
              contentItemService.saveContentItem(_cat);
            }
            taggingService.createTagging(label, item, _cat, user);
          }
        }
      }
    }

    // add parameter categories as tags
    for (ContentItem tag : tags) {
      if (!taggingService.hasTag(item, tag.getTitle())) {
        taggingService.createTagging(tag.getTitle(), item, tag, user);
      }
    }

    // add parameter types as types
    for (KiWiUriResource type : types) {
      item.addType(type);
    }

    // add kiwi:FeedPost type
    item.addType(tripleStore.createUriResource(Constants.NS_KIWI_CORE + "FeedPost"));

    /* the flush is necessary, because CIs or tags will
     * otherwise be created multiple times when they
     * appear more than once in one RSS feed */
    entityManager.flush();
    log.debug("imported content item '#0' with URI '#1'", item.getTitle(), item.getResource());
  }
 private String getBlogHomepageFromRss(String rss) {
   SyndFeed syndFeed = syndFeedFactory.createFor(rss);
   return syndFeed.getLink();
 }
Пример #6
0
  /** @inheritDoc */
  public Subscription fetchSubscription(String feedURL, Date lastModified) throws FetcherException {

    if (feedURL == null) {
      throw new IllegalArgumentException("feed url cannot be null");
    }

    // setup Rome feed fetcher
    FeedFetcher feedFetcher = getRomeFetcher();

    // fetch the feed
    log.debug("Fetching feed: " + feedURL);
    SyndFeed feed;
    try {
      feed = feedFetcher.retrieveFeed(new URL(feedURL));
    } catch (Exception ex) {
      throw new FetcherException("Error fetching subscription - " + feedURL, ex);
    }

    log.debug("Feed pulled, extracting data into Subscription");

    // build planet subscription from fetched feed
    Subscription newSub = new Subscription();
    newSub.setFeedURL(feedURL);
    newSub.setSiteURL(feed.getLink());
    newSub.setTitle(feed.getTitle());
    newSub.setAuthor(feed.getAuthor());
    newSub.setLastUpdated(feed.getPublishedDate());

    // normalize any data that couldn't be properly extracted
    if (newSub.getSiteURL() == null) {
      // set the site url to the feed url then
      newSub.setSiteURL(newSub.getFeedURL());
    }
    if (newSub.getAuthor() == null) {
      // set the author to the title
      newSub.setAuthor(newSub.getTitle());
    }
    if (newSub.getLastUpdated() == null) {
      // no update time specified in feed, so try consulting feed info cache
      FeedFetcherCache feedCache = getRomeFetcherCache();
      try {
        SyndFeedInfo feedInfo = feedCache.getFeedInfo(new URL(newSub.getFeedURL()));
        if (feedInfo.getLastModified() != null) {
          long lastUpdatedLong = ((Long) feedInfo.getLastModified()).longValue();
          if (lastUpdatedLong != 0) {
            newSub.setLastUpdated(new Date(lastUpdatedLong));
          }
        }
      } catch (MalformedURLException ex) {
        // should never happen since we check this above
      }
    }

    // check if feed is unchanged and bail now if so
    if (lastModified != null
        && newSub.getLastUpdated() != null
        && !newSub.getLastUpdated().after(lastModified)) {
      return null;
    }

    if (log.isDebugEnabled()) {
      log.debug("Subscription is: " + newSub.toString());
    }

    // some kludge to deal with feeds w/ no entry dates
    // we assign arbitrary dates chronologically by entry starting either
    // from the current time or the last update time of the subscription
    Calendar cal = Calendar.getInstance();
    if (newSub.getLastUpdated() != null) {
      cal.setTime(newSub.getLastUpdated());
    } else {
      cal.setTime(new Date());
      cal.add(Calendar.DATE, -1);
    }

    // add entries
    List<SyndEntry> feedEntries = feed.getEntries();
    for (SyndEntry feedEntry : feedEntries) {
      SubscriptionEntry newEntry = buildEntry(feedEntry);

      // some kludge to handle feeds with no entry dates
      if (newEntry.getPubTime() == null) {
        log.debug("No published date, assigning fake date for " + feedURL);
        newEntry.setPubTime(new Timestamp(cal.getTimeInMillis()));
        cal.add(Calendar.DATE, -1);
      }

      if (newEntry != null) {
        newSub.addEntry(newEntry);
      }
    }

    log.debug(feedEntries.size() + " entries included");

    return newSub;
  }