コード例 #1
0
  @Override
  public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    statuses = new LinkedBlockingQueue<Status>(1000);
    this.spoutOutputCollector = collector;

    ConfigurationBuilder configurationBuilder = new ConfigurationBuilder();
    configurationBuilder
        .setOAuthConsumerKey(consumerKey)
        .setOAuthConsumerSecret(consumerSecret)
        .setOAuthAccessToken(accessToken)
        .setOAuthAccessTokenSecret(accessTokenSecret);

    OAuthAuthorization authAuthorization = new OAuthAuthorization(configurationBuilder.build());

    twitterStream = new TwitterStreamFactory().getInstance(authAuthorization);
    twitterStream.addListener(
        new StatusListener() {
          @Override
          public void onStatus(Status status) {
            statuses.offer(status);
          }

          @Override
          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          @Override
          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          @Override
          public void onScrubGeo(long userId, long upToStatusId) {}

          @Override
          public void onStallWarning(StallWarning warning) {}

          @Override
          public void onException(Exception ex) {}
        });

    twitter = new TwitterFactory().getInstance(authAuthorization);
    filterQuery = new FilterQuery();

    if (filterQuery == null) {
      twitterStream.sample();
      ;
    } else {
      twitterStream.filter(filterQuery.track(filterWords));
      twitterStream.filter(filterQuery.language(filterLanguages));
    }
  }
コード例 #2
0
 public void updateTwitterStream(Location location, int radius) {
   Log.d(TAG, "updateTwitterStream()");
   double[][] mapRegion = GeoCalculationsHelper.getMapRegion(location, radius);
   FilterQuery filterQuery = new FilterQuery();
   filterQuery.locations(mapRegion);
   twitterStream.filter(filterQuery);
 }
コード例 #3
0
  /**
   * Start processing events. This uses the Twitter Streaming API to sample Twitter, and process
   * tweets.
   */
  @Override
  public void start() {
    // The channel is the piece of Flume that sits between the Source and Sink,
    // and is used to process events.
    final ChannelProcessor channel = getChannelProcessor();

    final Map<String, String> headers = new HashMap<String, String>();

    // The StatusListener is a twitter4j API, which can be added to a Twitter
    // stream, and will execute methods every time a message comes in through
    // the stream.
    StatusListener listener =
        new StatusListener() {
          // The onStatus method is executed every time a new tweet comes in.
          public void onStatus(Status status) {
            // The EventBuilder is used to build an event using the headers and
            // the raw JSON of a tweet
            logger.debug(status.getUser().getScreenName() + ": " + status.getText());

            headers.put("timestamp", String.valueOf(status.getCreatedAt().getTime()));
            Event event =
                EventBuilder.withBody(DataObjectFactory.getRawJSON(status).getBytes(), headers);

            channel.processEvent(event);
          }

          // This listener will ignore everything except for new tweets
          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          public void onScrubGeo(long userId, long upToStatusId) {}

          public void onException(Exception ex) {}

          public void onStallWarning(StallWarning warning) {}
        };

    logger.debug(
        "Setting up Twitter sample stream using consumer key {} and" + " access token {}",
        new String[] {consumerKey, accessToken});
    // Set up the stream's listener (defined above), and set any necessary
    // security information.
    twitterStream.addListener(listener);
    twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
    AccessToken token = new AccessToken(accessToken, accessTokenSecret);
    twitterStream.setOAuthAccessToken(token);

    // Set up a filter to pull out industry-relevant tweets
    if (keywords.length == 0) {
      logger.debug("Starting up Twitter sampling...");
      twitterStream.sample();
    } else {
      logger.debug("Starting up Twitter filtering...");
      FilterQuery query = new FilterQuery().track(keywords);
      twitterStream.filter(query);
    }
    super.start();
  }
コード例 #4
0
  @Override
  public void update() {
    if (running) {
      return;
    }
    running = true;

    tweetsFlowRegulator.startTweetPicker();
    twitterStream.filter(query);
  }
コード例 #5
0
  /**
   * Main entry of this application.
   *
   * @param args follow(comma separated user ids) track(comma separated filter terms)
   * @throws twitter4j.TwitterException
   */
  public static void main(String[] args) throws TwitterException {
    if (args.length < 1) {
      System.out.println(
          "Usage: java twitter4j.examples.PrintFilterStream [follow(comma separated numerical user ids)] [track(comma separated filter terms)]");
      System.exit(-1);
    }

    StatusListener listener =
        new StatusListener() {
          public void onStatus(Status status) {
            System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText());
          }

          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            System.out.println(
                "Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
          }

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
          }

          public void onScrubGeo(long userId, long upToStatusId) {
            System.out.println(
                "Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
          }

          public void onException(Exception ex) {
            ex.printStackTrace();
          }
        };

    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
    twitterStream.addListener(listener);
    ArrayList<Long> follow = new ArrayList<Long>();
    ArrayList<String> track = new ArrayList<String>();
    for (String arg : args) {
      if (isNumericalArgument(arg)) {
        for (String id : arg.split(",")) {
          follow.add(Long.parseLong(id));
        }
      } else {
        track.addAll(Arrays.asList(arg.split(",")));
      }
    }
    long[] followArray = new long[follow.size()];
    for (int i = 0; i < follow.size(); i++) {
      followArray[i] = follow.get(i);
    }
    String[] trackArray = track.toArray(new String[track.size()]);

    // filter() method internally creates a thread which manipulates TwitterStream and calls these
    // adequate listener methods continuously.
    twitterStream.filter(new FilterQuery(0, followArray, trackArray));
  }
コード例 #6
0
 /**
  * Simply starts the stream.
  *
  * @param id the id
  */
 @RequestMapping(value = "/start/{id}", method = RequestMethod.GET)
 @ResponseStatus(value = HttpStatus.OK, reason = "Successfully started stream")
 public final void start(@PathVariable final long id) {
   LOGGER.info("Shutting down stream");
   stream.shutdown();
   stream.addListener(listener);
   List<String> portfolioTokens = portfolioService.getStreamTokens(id);
   stream.filter(
       new FilterQuery(
           0, new long[0], portfolioTokens.toArray(new String[portfolioTokens.size()])));
   LOGGER.info("Successfully started stream: " + portfolioTokens);
 }
コード例 #7
0
  public static void main(String[] args) throws TwitterException {

    FilterQuery query = new FilterQuery();

    // wfrank 7897072
    // ponja 38299186

    String[] tracks = {"diegoram", "cfkargentina"};
    // query.track(tracks);

    long[] ids = {7897072L, 38299186L, 17760769L};

    query.follow(ids);

    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
    StatusListener listener =
        new StatusListener() {
          @Override
          public void onStatus(Status status) {
            System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText());
          }

          @Override
          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            System.out.println(
                "Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
          }

          @Override
          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
          }

          @Override
          public void onScrubGeo(long userId, long upToStatusId) {
            System.out.println(
                "Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
          }

          // @Override
          // public void onStallWarning(StallWarning warning) {
          //    System.out.println("Got stall warning:" + warning);
          // }

          @Override
          public void onException(Exception ex) {
            ex.printStackTrace();
          }
        };
    twitterStream.addListener(listener);
    twitterStream.filter(query);
  }
コード例 #8
0
ファイル: TweetsRetriever.java プロジェクト: Lefteris008/EvS
  /**
   * Method that handles the Twitter streaming API. <br>
   * <b>WARNING:</b> Method does not terminate by itself, due to the fact that the streamer runs in
   * a different thread.
   *
   * @param keywords The keywords for which the streamer searches for tweets.
   * @param mongoDB A handler for the MongoDB database.
   * @param config A configuration object.
   */
  public final void retrieveTweetsWithStreamingAPI(
      String[] keywords, MongoHandler mongoDB, Config config) {

    ConfigurationBuilder cb = getAuthorization();
    TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance();

    final StatusListener listener;
    listener =
        new StatusListener() {

          @Override
          public final void onStatus(Status status) {
            // Insert tweet to MongoDB
            mongoDB.insertSingleTweetIntoMongoDB(status, "NULL");
          }

          @Override
          public final void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            PrintUtilities.printInfoMessageln(
                "Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
          }

          @Override
          public final void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            PrintUtilities.printInfoMessageln(
                "Got track limitation notice:" + numberOfLimitedStatuses);
          }

          @Override
          public final void onScrubGeo(long userId, long upToStatusId) {
            PrintUtilities.printInfoMessageln(
                "Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
          }

          @Override
          public final void onStallWarning(StallWarning warning) {
            PrintUtilities.printInfoMessageln("Got stall warning:" + warning);
          }

          @Override
          public final void onException(Exception ex) {
            ex.printStackTrace(System.out);
          }
        };

    FilterQuery fq = new FilterQuery();
    fq.language("en"); // Set language of tweets to "English"
    fq.track(keywords); // Load the search terms

    twitterStream.addListener(listener); // Start listening to the stream
    twitterStream.filter(fq); // Apply the search filters
  }
コード例 #9
0
  @Override
  public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    queue = new LinkedBlockingQueue<Status>(1000);
    _collector = collector;

    StatusListener listener =
        new StatusListener() {

          @Override
          public void onStatus(Status status) {

            queue.offer(status);
          }

          @Override
          public void onDeletionNotice(StatusDeletionNotice sdn) {}

          @Override
          public void onTrackLimitationNotice(int i) {}

          @Override
          public void onScrubGeo(long l, long l1) {}

          @Override
          public void onException(Exception ex) {}

          @Override
          public void onStallWarning(StallWarning arg0) {
            // TODO Auto-generated method stub

          }
        };

    _twitterStream =
        new TwitterStreamFactory(new ConfigurationBuilder().setJSONStoreEnabled(true).build())
            .getInstance();

    _twitterStream.addListener(listener);
    _twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
    AccessToken token = new AccessToken(accessToken, accessTokenSecret);
    _twitterStream.setOAuthAccessToken(token);

    if (keyWords.length == 0) {

      _twitterStream.sample();
    } else {

      FilterQuery query = new FilterQuery().track(keyWords);
      _twitterStream.filter(query);
    }
  }
コード例 #10
0
  /** Starts listening for twitter events */
  @Override
  public void start() {
    // The channel connects from source to sink
    final ChannelProcessor channel = getChannelProcessor();

    final Map<String, String> headers = new HashMap<String, String>();

    // The method onStatus() will be called everytime a tweet comes.
    StatusListener listener =
        new StatusListener() {
          // capture new tweet notification
          public void onStatus(Status status) {
            // add creation time in the header
            headers.put("timestamp", String.valueOf(status.getCreatedAt().getTime()));
            Event event =
                EventBuilder.withBody(DataObjectFactory.getRawJSON(status).getBytes(), headers);
            // send to sink
            channel.processEvent(event);
          }

          // ignore all other notifications
          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          public void onScrubGeo(long userId, long upToStatusId) {}

          public void onException(Exception ex) {}

          @Override
          public void onStallWarning(StallWarning arg0) {
            // do nothing
          }
        };

    // add the listener we created + tell the stream all about the security info required
    twitterStream.addListener(listener);
    twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
    AccessToken token = new AccessToken(accessToken, accessTokenSecret);
    twitterStream.setOAuthAccessToken(token);

    // Set up a filter to pull out industry-relevant tweets
    if (searchFor.length == 0) {
      System.out.println("Please setup filter keyword in Flume conf");
    } else {
      // create a filter query for filtering out only the required info
      FilterQuery query = new FilterQuery().track(searchFor);
      twitterStream.filter(query);
    }
    super.start();
  }
コード例 #11
0
ファイル: Streams.java プロジェクト: janpascal/tweet-play2
  private static void startStream(List<String> terms) throws TwitterException {
    if (twitter != null) {
      twitter.cleanUp();
    }
    if (esClient != null) {
      esClient.close();
      esClient = null;
    }

    play.Configuration pconf = Play.application().configuration();
    String elasticSearchCluster = pconf.getString("tweet.elasticsearch.cluster.name");
    if (elasticSearchCluster != null) {
      Logger.info("Configuring ElasticSearch...");
      Settings settings =
          ImmutableSettings.settingsBuilder().put("cluster.name", elasticSearchCluster).build();

      esClient =
          new TransportClient(settings)
              .addTransportAddress(
                  new InetSocketTransportAddress(
                      pconf.getString("tweet.elasticsearch.transport.host"),
                      pconf.getInt("tweet.elasticsearch.transport.port")));
    } else {
      esClient = null;
    }

    twitter4j.conf.Configuration tconf = Application.getTwitterConfiguration();
    TwitterStreamFactory tf = new TwitterStreamFactory(tconf);
    twitter = tf.getInstance();
    StatusListener l =
        new TweetListener(
            terms,
            esClient,
            pconf.getString("tweet.elasticsearch.index"),
            pconf.getString("tweet.elasticsearch.type"));
    twitter.addListener(l);

    String[] tracks = new String[terms.size()];
    StringBuffer termsString = new StringBuffer();
    for (int i = 0; i < terms.size(); i++) {
      tracks[i] = terms.get(i);
      if (i != 0) termsString.append(",");
      termsString.append(terms.get(i));
    }
    FilterQuery q = new FilterQuery().track(tracks);
    twitter.filter(q);
    Logger.info("Starting listening for tweets using terms " + termsString.toString() + "...");
  }
コード例 #12
0
  @Override
  public void filter(final TweetFilterQuery filterQuery) {
    final TwitterStream twitterStream =
        new TwitterStreamFactory(TwitterOAuth.getConfiguration()).getInstance();

    twitterStream.addListener(
        new StatusAdapter() {

          @Override
          public void onStatus(Status status) {
            synchronized (TwitterTweetStream.this) {
              if (null != tweetConsumer) {
                tweetConsumer.accept(new TwitterTweet(status));
              }
            }
          }
        });
    twitterStream.filter(getFilterQuery(filterQuery));
  }
コード例 #13
0
  /**
   * This is where the automagic happens We are listening on tweets through the streaming api,
   * whenever we get a tweet "fireStatusEvent" is called
   */
  @PostConstruct
  public void startTweetStream() {
    TwitterStream twitterStream =
        new TwitterStreamFactory(TwitterConfigBuilder.getConfig()).getInstance();

    StatusListener statusListener =
        new StatusListener() {
          @Override
          public void onStatus(Status status) {
            logger.info("Received a status from " + status.getUser().getScreenName());
            Tweet tweet = new Tweet(status);
            fireStatusEvent(tweet);
          }

          @Override
          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          @Override
          public void onTrackLimitationNotice(int i) {}

          @Override
          public void onScrubGeo(long l, long l1) {}

          @Override
          public void onStallWarning(StallWarning stallWarning) {}

          @Override
          public void onException(Exception e) {
            e.printStackTrace();
          }
        };

    FilterQuery filter = new FilterQuery();
    String[] keywords = {"#testtestWOW"};
    filter.track(keywords);

    twitterStream.addListener(statusListener);
    twitterStream.filter(filter);
    logger.warn("Started to listen for " + keywords[0]);
  }
コード例 #14
0
 public static void streamTweets(String keywords, String location, Boolean hideRetweets)
     throws URLs.ConnectionException, URLs.HTTPQueryException, GeolocationSearch.NoKeyException,
         GeolocationSearch.SearchLocationException, MalformedURLException {
   twitter4j.TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
   twitterStream.addListener(
       new StatusAdapter() {
         public void onStatus(Status status) {
           if (!(status.isRetweet() && hideRetweets)) {
             printTweet(status, true);
           }
         }
       });
   Double[] coordinates = getCoordinatesByQuery(location);
   double[][] doubleCoordinates = {
     {coordinates[1] - RADIUS, coordinates[0] - RADIUS},
     {coordinates[1] + RADIUS, coordinates[0] + RADIUS}
   };
   String[] arrayKeywords = {keywords};
   FilterQuery fq = new FilterQuery();
   fq.locations(doubleCoordinates);
   fq.track(arrayKeywords);
   twitterStream.filter(fq);
 }
コード例 #15
0
  public void loadMenu() throws InterruptedException {

    System.out.print("Please choose a name for your stream:\t");

    Scanner input = new Scanner(System.in);
    String keyword = input.nextLine();

    connectdb(keyword);

    cb = new ConfigurationBuilder();
    cb.setDebugEnabled(true);
    cb.setOAuthConsumerKey("XXX");
    cb.setOAuthConsumerSecret("XXX");
    cb.setOAuthAccessToken("XXX");
    cb.setOAuthAccessTokenSecret("XXX");

    TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
    StatusListener listener =
        new StatusListener() {

          public void onStatus(Status status) {
            System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText());

            BasicDBObject basicObj = new BasicDBObject();
            basicObj.put("user_name", status.getUser().getScreenName());
            basicObj.put("retweet_count", status.getRetweetCount());
            basicObj.put("tweet_followers_count", status.getUser().getFollowersCount());
            basicObj.put("source", status.getSource());
            // basicObj.put("coordinates",tweet.getGeoLocation());

            UserMentionEntity[] mentioned = status.getUserMentionEntities();
            basicObj.put("tweet_mentioned_count", mentioned.length);
            basicObj.put("tweet_ID", status.getId());
            basicObj.put("tweet_text", status.getText());

            try {
              items.insert(basicObj);
            } catch (Exception e) {
              System.out.println("MongoDB Connection Error : " + e.getMessage());
            }
          }

          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            System.out.println(
                "Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
          }

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
          }

          public void onScrubGeo(long userId, long upToStatusId) {
            System.out.println(
                "Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
          }

          @Override
          public void onStallWarning(StallWarning stallWarning) {
            // To change body of implemented methods use File | Settings | File Templates.
          }

          public void onException(Exception ex) {
            ex.printStackTrace();
          }
        };

    FilterQuery fq = new FilterQuery();
    String keywords[] = {"Germany"};

    fq.track(keywords);

    twitterStream.addListener(listener);
    twitterStream.filter(fq);
  }
コード例 #16
0
  private void start(final Context context) throws IOException {

    // Producer properties
    Properties props = new Properties();
    props.put("metadata.broker.list", context.getString(TwitterSourceConstant.BROKER_LIST));
    props.put("serializer.class", context.getString(TwitterSourceConstant.SERIALIZER));
    props.put("partitioner.class", context.getString(TwitterSourceConstant.PARTITIONER));
    props.put("request.required.acks", context.getString(TwitterSourceConstant.REQUIRED_ACKS));

    ProducerConfig config = new ProducerConfig(props);

    final Producer<String, String> producer = new Producer<String, String>(config);

    /** Twitter properties * */
    consumerKey = context.getString(TwitterSourceConstant.CONSUMER_KEY_KEY);
    consumerSecret = context.getString(TwitterSourceConstant.CONSUMER_SECRET_KEY);
    accessToken = context.getString(TwitterSourceConstant.ACCESS_TOKEN_KEY);
    accessTokenSecret = context.getString(TwitterSourceConstant.ACCESS_TOKEN_SECRET_KEY);

    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setOAuthConsumerKey(consumerKey);
    cb.setOAuthConsumerSecret(consumerSecret);
    cb.setOAuthAccessToken(accessToken);
    cb.setOAuthAccessTokenSecret(accessTokenSecret);
    cb.setJSONStoreEnabled(true);
    cb.setIncludeEntitiesEnabled(true);
    cb.setHttpProxyHost("proxy.tcs.com");
    cb.setHttpProxyPort(8080);
    cb.setHttpProxyUser("876216");
    cb.setHttpProxyPassword("Apple@123");
    twitterStream = new TwitterStreamFactory(cb.build()).getInstance();

    final Map<String, String> headers = new HashMap<String, String>();

    /** Twitter listener * */
    StatusListener listener =
        new StatusListener() {
          // The onStatus method is executed every time a new tweet comes
          // in.
          public void onStatus(Status status) {
            // The EventBuilder is used to build an event using the
            // the raw JSON of a tweet

            System.out.println("Listening :");
            KeyedMessage<String, String> data =
                new KeyedMessage<String, String>(
                    "testing1", TwitterObjectFactory.getRawJSON(status));

            producer.send(data);
            System.out.println(data);
          }

          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          public void onScrubGeo(long userId, long upToStatusId) {}

          public void onException(Exception ex) {
            logger.info("ShutDown");
            twitterStream.shutdown();
          }

          public void onStallWarning(StallWarning warning) {}
        };

    twitterStream.addListener(listener);
    /** GOGOGO * */
    twitterStream.sample();
    FilterQuery query =
        new FilterQuery()
            .track(
                Tweety.hashtags[0],
                Tweety.hashtags[1],
                Tweety.hashtags[2],
                Tweety.hashtags[3],
                Tweety.hashtags[4]);
    twitterStream.filter(query);
    /** Bind the listener * */
  }
コード例 #17
0
  /**
   * Start processing events. This uses the Twitter Streaming API to sample Twitter, and process
   * tweets.
   */
  @Override
  public void start() {
    // The channel is the piece of Flume that sits between the Source and Sink,
    // and is used to process events.
    final ChannelProcessor channel = getChannelProcessor();

    final Map<String, String> headers = new HashMap<String, String>();

    // The StatusListener is a twitter4j API, which can be added to a Twitter
    // stream, and will execute methods every time a message comes in through
    // the stream.
    StatusListener listener =
        new StatusListener() {
          // The onStatus method is executed every time a new tweet comes in.
          public void onStatus(Status status) {
            // The EventBuilder is used to build an event using the headers and
            // the raw JSON of a tweet
            logger.debug(status.getUser().getScreenName() + ": " + status.getText());

            headers.put("timestamp", String.valueOf(status.getCreatedAt().getTime()));
            Event event =
                EventBuilder.withBody(DataObjectFactory.getRawJSON(status).getBytes(), headers);
            channel.processEvent(event);
          }

          // This listener will ignore everything except for new tweets
          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          public void onScrubGeo(long userId, long upToStatusId) {}

          public void onException(Exception ex) {
            logger.error("Stream Error ", ex);
          }

          public void onStallWarning(StallWarning warning) {
            int percentFull = warning.getPercentFull();
            logger.warn("Stall Warning Received ", warning);
            if (percentFull > 95) {
              logger.warn("Stallwarning Stream full more han 95 %. Going to wait for 2 minutes");
              try {
                Thread.sleep(2 * 60 * 000);
              } catch (InterruptedException e) {
                e.printStackTrace();
              }
            }
          }
        };

    logger.debug(
        "Setting up Twitter sample stream using consumer key {} and" + " access token {}",
        new String[] {consumerKey, accessToken});
    // Set up the stream's listener (defined above),
    twitterStream.addListener(listener);

    // Set up a filter to pull out industry-relevant tweets
    if (keywords.length == 0) {
      logger.debug("Starting up Twitter sampling...");
      twitterStream.sample();
    } else {
      logger.debug("Starting up Twitter filtering...");

      FilterQuery query = new FilterQuery().track(keywords);
      twitterStream.filter(query);
    }
    super.start();
  }
コード例 #18
0
ファイル: TwitterProducer.java プロジェクト: ATM-SALEH/pankh
  private void start(final Context context) {

    /** Producer properties * */
    Properties props = new Properties();
    props.put("metadata.broker.list", context.getString(BROKER_LIST));
    props.put("serializer.class", context.getString(SERIALIZER));
    props.put("request.required.acks", context.getString(REQUIRED_ACKS));

    ProducerConfig config = new ProducerConfig(props);
    final Producer<String, String> producer = new Producer<String, String>(config);

    /** Twitter properties * */
    consumerKey = context.getString(CONSUMER_KEY_KEY);
    consumerSecret = context.getString(CONSUMER_SECRET_KEY);
    accessToken = context.getString(ACCESS_TOKEN_KEY);
    accessTokenSecret = context.getString(ACCESS_TOKEN_SECRET_KEY);

    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setOAuthConsumerKey(consumerKey);
    cb.setOAuthConsumerSecret(consumerSecret);
    cb.setOAuthAccessToken(accessToken);
    cb.setOAuthAccessTokenSecret(accessTokenSecret);
    cb.setJSONStoreEnabled(true);
    cb.setIncludeEntitiesEnabled(true);

    twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
    final boolean shouldPrintTweetsOnScreen =
        Boolean.parseBoolean(context.getString(printTweetsOnScreen));
    final boolean shouldSendTweetsToKafka =
        Boolean.parseBoolean(context.getString(sendTweetsToKafka));

    final StatusListener listener =
        new StatusListener() {
          // The onStatus method is executed every time a new tweet comes
          // in.
          public void onStatus(Status status) {
            // The EventBuilder is used to build an event using the
            // the raw JSON of a tweet
            if (shouldPrintTweetsOnScreen) {
              logger.info(status.getUser().getScreenName() + ": " + status.getText());
            }

            if (shouldSendTweetsToKafka) {
              KeyedMessage<String, String> data =
                  new KeyedMessage<String, String>(
                      context.getString(KAFKA_TOPIC), TwitterObjectFactory.getRawJSON(status));
              producer.send(data);
            }
          }

          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          public void onScrubGeo(long userId, long upToStatusId) {}

          public void onException(Exception ex) {
            logger.info("Shutting down Twitter sample stream...");
            twitterStream.shutdown();
          }

          public void onStallWarning(StallWarning warning) {}
        };

    twitterStream.addListener(listener);

    twitterStream.filter(new FilterQuery().track(context.getString(keywords).split(",")));
  }