private void initStream() {
    tweetsFlowRegulator = TweetsFlowRegulator.newInstance(listener);
    twitterStream.addListener(
        new StatusListener() {
          @Override
          public void onStatus(Status status) {
            tweetsFlowRegulator.addTweet(status);
          }

          @Override
          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            // No-op
          }

          @Override
          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            // No-op
          }

          @Override
          public void onScrubGeo(long userId, long upToStatusId) {
            // No-op
          }

          @Override
          public void onStallWarning(StallWarning warning) {
            Log.w(warning);
          }

          @Override
          public void onException(Exception ex) {
            Log.e(ex);
          }
        });
  }
Example #2
0
  public static void main(String[] args) throws Exception {
    ConfigurationBuilder builder = new ConfigurationBuilder();
    builder.setOAuthConsumerKey(CONSUMER_KEY);
    builder.setOAuthConsumerSecret(CONSUMER_SECRET);
    builder.setOAuthAccessToken(ACCESS_TOKEN);
    builder.setOAuthAccessTokenSecret(ACCESS_TOKEN_SECRET);

    // 現行のTwitter4JではAPIのデフォルト呼び先がbetastream.twitter.comになっているので修正
    builder.setUserStreamBaseURL("https://userstream.twitter.com/2/");

    // Configurationを生成
    Configuration configuration = builder.build();

    /*        Configuration configuration = new ConfigurationBuilder().setOAuthConsumerKey(CONSUMER_KEY)
                    .setOAuthConsumerSecret(CONSUMER_SECRET)
                    .setOAuthAccessToken(ACCESS_TOKEN)
                    .setOAuthAccessTokenSecret(ACCESS_TOKEN_SECRET)
                    .build();
    */
    TwitterStream twStream = new TwitterStreamFactory(configuration).getInstance();
    // TwitterStream twStream = new TwitterStreamFactory().getInstance();
    twStream.addListener(new MyStatusListener());
    twStream.user();
    // twStream.sample();
  }
  @Override
  public IBinder onBind(Intent intent) {
    Log.d(TAG, "onBind()");
    configurationBuilder = new ConfigurationBuilder();
    configurationBuilder
        .setDebugEnabled(true)
        .setOAuthConsumerKey("JmPCgGdftlNXuh21WQ7hFA")
        .setOAuthConsumerSecret("drMLhPvOWs2Crol2LwQuqdKVRTFCVbQlkJQOCrV8uI")
        .setOAuthAccessToken("72023528-NFWdbv2h4vDVdZC1ML2jNT0gXt9fqZLpMdvtGDjnH")
        .setOAuthAccessTokenSecret("JW7Y2e8D086oDsU1wpNKgtsPZAwF1TQl5KkMdbHdnQ");
    twitterStream = new TwitterStreamFactory(configurationBuilder.build()).getInstance();

    StatusListener statusListener =
        new StatusListener() {

          @Override
          public void onException(Exception arg0) {
            Log.d(TAG, "onException: cause," + arg0.getCause() + " message," + arg0.getMessage());
          }

          @Override
          public void onDeletionNotice(StatusDeletionNotice arg0) {
            Log.d(TAG, "onDeletionNotice()");
          }

          @Override
          public void onScrubGeo(long arg0, long arg1) {
            Log.d(TAG, "onScrubGeo()");
          }

          @Override
          public void onStallWarning(StallWarning stallWarning) {
            Log.d(TAG, "onStallWarning()");
          }

          @Override
          public void onStatus(Status status) {
            Log.d(
                TAG,
                "Twitter Status Received: Geolocation?:"
                    + (status.getGeoLocation() != null)
                    + " / "
                    + status.getUser().getScreenName()
                    + ": "
                    + status.getText());
            if (status.getGeoLocation() != null) {
              addNewTwitterStatus(status);
            }
          }

          @Override
          public void onTrackLimitationNotice(int arg0) {
            Log.d(TAG, "onTrackLimitationNotice()");
          }
        };
    twitterStream.addListener(statusListener);

    return mBinder;
  }
  /**
   * Start processing events. This uses the Twitter Streaming API to sample Twitter, and process
   * tweets.
   */
  @Override
  public void start() {
    // The channel is the piece of Flume that sits between the Source and Sink,
    // and is used to process events.
    final ChannelProcessor channel = getChannelProcessor();

    final Map<String, String> headers = new HashMap<String, String>();

    // The StatusListener is a twitter4j API, which can be added to a Twitter
    // stream, and will execute methods every time a message comes in through
    // the stream.
    StatusListener listener =
        new StatusListener() {
          // The onStatus method is executed every time a new tweet comes in.
          public void onStatus(Status status) {
            // The EventBuilder is used to build an event using the headers and
            // the raw JSON of a tweet
            logger.debug(status.getUser().getScreenName() + ": " + status.getText());

            headers.put("timestamp", String.valueOf(status.getCreatedAt().getTime()));
            Event event =
                EventBuilder.withBody(DataObjectFactory.getRawJSON(status).getBytes(), headers);

            channel.processEvent(event);
          }

          // This listener will ignore everything except for new tweets
          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          public void onScrubGeo(long userId, long upToStatusId) {}

          public void onException(Exception ex) {}

          public void onStallWarning(StallWarning warning) {}
        };

    logger.debug(
        "Setting up Twitter sample stream using consumer key {} and" + " access token {}",
        new String[] {consumerKey, accessToken});
    // Set up the stream's listener (defined above), and set any necessary
    // security information.
    twitterStream.addListener(listener);
    twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
    AccessToken token = new AccessToken(accessToken, accessTokenSecret);
    twitterStream.setOAuthAccessToken(token);

    // Set up a filter to pull out industry-relevant tweets
    if (keywords.length == 0) {
      logger.debug("Starting up Twitter sampling...");
      twitterStream.sample();
    } else {
      logger.debug("Starting up Twitter filtering...");
      FilterQuery query = new FilterQuery().track(keywords);
      twitterStream.filter(query);
    }
    super.start();
  }
 public boolean startSample(long lifeTime) {
   if (accessConfigBuilder == null) return false;
   StatusListener listener = new SampleStreamListener(lifeTime);
   TwitterStream twitterStream =
       new TwitterStreamFactory(accessConfigBuilder.build()).getInstance();
   twitterStream.addListener(listener);
   twitterStream.sample();
   return true;
 }
Example #6
0
  /** Start listening to tweets. */
  public void start() throws IllegalStateException, TwitterException {
    twitter = new TwitterFactory().getInstance();
    stream = new TwitterStreamFactory().getInstance();

    LOGGER.info("Connected to Twitter as: {}", twitter.getScreenName());

    LOGGER.info("Starting the Twitter stream listener");
    stream.addListener(new GetMentionsListener(twitter));
    stream.user(); // Start reading to user stream
  }
  /**
   * Main entry of this application.
   *
   * @param args follow(comma separated user ids) track(comma separated filter terms)
   * @throws twitter4j.TwitterException
   */
  public static void main(String[] args) throws TwitterException {
    if (args.length < 1) {
      System.out.println(
          "Usage: java twitter4j.examples.PrintFilterStream [follow(comma separated numerical user ids)] [track(comma separated filter terms)]");
      System.exit(-1);
    }

    StatusListener listener =
        new StatusListener() {
          public void onStatus(Status status) {
            System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText());
          }

          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            System.out.println(
                "Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
          }

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
          }

          public void onScrubGeo(long userId, long upToStatusId) {
            System.out.println(
                "Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
          }

          public void onException(Exception ex) {
            ex.printStackTrace();
          }
        };

    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
    twitterStream.addListener(listener);
    ArrayList<Long> follow = new ArrayList<Long>();
    ArrayList<String> track = new ArrayList<String>();
    for (String arg : args) {
      if (isNumericalArgument(arg)) {
        for (String id : arg.split(",")) {
          follow.add(Long.parseLong(id));
        }
      } else {
        track.addAll(Arrays.asList(arg.split(",")));
      }
    }
    long[] followArray = new long[follow.size()];
    for (int i = 0; i < follow.size(); i++) {
      followArray[i] = follow.get(i);
    }
    String[] trackArray = track.toArray(new String[track.size()]);

    // filter() method internally creates a thread which manipulates TwitterStream and calls these
    // adequate listener methods continuously.
    twitterStream.filter(new FilterQuery(0, followArray, trackArray));
  }
 /**
  * Simply starts the stream.
  *
  * @param id the id
  */
 @RequestMapping(value = "/start/{id}", method = RequestMethod.GET)
 @ResponseStatus(value = HttpStatus.OK, reason = "Successfully started stream")
 public final void start(@PathVariable final long id) {
   LOGGER.info("Shutting down stream");
   stream.shutdown();
   stream.addListener(listener);
   List<String> portfolioTokens = portfolioService.getStreamTokens(id);
   stream.filter(
       new FilterQuery(
           0, new long[0], portfolioTokens.toArray(new String[portfolioTokens.size()])));
   LOGGER.info("Successfully started stream: " + portfolioTokens);
 }
Example #9
0
  /**
   * Method that handles the Twitter streaming API. <br>
   * <b>WARNING:</b> Method does not terminate by itself, due to the fact that the streamer runs in
   * a different thread.
   *
   * @param keywords The keywords for which the streamer searches for tweets.
   * @param mongoDB A handler for the MongoDB database.
   * @param config A configuration object.
   */
  public final void retrieveTweetsWithStreamingAPI(
      String[] keywords, MongoHandler mongoDB, Config config) {

    ConfigurationBuilder cb = getAuthorization();
    TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance();

    final StatusListener listener;
    listener =
        new StatusListener() {

          @Override
          public final void onStatus(Status status) {
            // Insert tweet to MongoDB
            mongoDB.insertSingleTweetIntoMongoDB(status, "NULL");
          }

          @Override
          public final void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            PrintUtilities.printInfoMessageln(
                "Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
          }

          @Override
          public final void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            PrintUtilities.printInfoMessageln(
                "Got track limitation notice:" + numberOfLimitedStatuses);
          }

          @Override
          public final void onScrubGeo(long userId, long upToStatusId) {
            PrintUtilities.printInfoMessageln(
                "Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
          }

          @Override
          public final void onStallWarning(StallWarning warning) {
            PrintUtilities.printInfoMessageln("Got stall warning:" + warning);
          }

          @Override
          public final void onException(Exception ex) {
            ex.printStackTrace(System.out);
          }
        };

    FilterQuery fq = new FilterQuery();
    fq.language("en"); // Set language of tweets to "English"
    fq.track(keywords); // Load the search terms

    twitterStream.addListener(listener); // Start listening to the stream
    twitterStream.filter(fq); // Apply the search filters
  }
  public static void main(String[] args) throws TwitterException {

    FilterQuery query = new FilterQuery();

    // wfrank 7897072
    // ponja 38299186

    String[] tracks = {"diegoram", "cfkargentina"};
    // query.track(tracks);

    long[] ids = {7897072L, 38299186L, 17760769L};

    query.follow(ids);

    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
    StatusListener listener =
        new StatusListener() {
          @Override
          public void onStatus(Status status) {
            System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText());
          }

          @Override
          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            System.out.println(
                "Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
          }

          @Override
          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
          }

          @Override
          public void onScrubGeo(long userId, long upToStatusId) {
            System.out.println(
                "Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
          }

          // @Override
          // public void onStallWarning(StallWarning warning) {
          //    System.out.println("Got stall warning:" + warning);
          // }

          @Override
          public void onException(Exception ex) {
            ex.printStackTrace();
          }
        };
    twitterStream.addListener(listener);
    twitterStream.filter(query);
  }
  /** Starts listening for twitter events */
  @Override
  public void start() {
    // The channel connects from source to sink
    final ChannelProcessor channel = getChannelProcessor();

    final Map<String, String> headers = new HashMap<String, String>();

    // The method onStatus() will be called everytime a tweet comes.
    StatusListener listener =
        new StatusListener() {
          // capture new tweet notification
          public void onStatus(Status status) {
            // add creation time in the header
            headers.put("timestamp", String.valueOf(status.getCreatedAt().getTime()));
            Event event =
                EventBuilder.withBody(DataObjectFactory.getRawJSON(status).getBytes(), headers);
            // send to sink
            channel.processEvent(event);
          }

          // ignore all other notifications
          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          public void onScrubGeo(long userId, long upToStatusId) {}

          public void onException(Exception ex) {}

          @Override
          public void onStallWarning(StallWarning arg0) {
            // do nothing
          }
        };

    // add the listener we created + tell the stream all about the security info required
    twitterStream.addListener(listener);
    twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
    AccessToken token = new AccessToken(accessToken, accessTokenSecret);
    twitterStream.setOAuthAccessToken(token);

    // Set up a filter to pull out industry-relevant tweets
    if (searchFor.length == 0) {
      System.out.println("Please setup filter keyword in Flume conf");
    } else {
      // create a filter query for filtering out only the required info
      FilterQuery query = new FilterQuery().track(searchFor);
      twitterStream.filter(query);
    }
    super.start();
  }
  @Override
  public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    queue = new LinkedBlockingQueue<Status>(1000);
    _collector = collector;

    StatusListener listener =
        new StatusListener() {

          @Override
          public void onStatus(Status status) {

            queue.offer(status);
          }

          @Override
          public void onDeletionNotice(StatusDeletionNotice sdn) {}

          @Override
          public void onTrackLimitationNotice(int i) {}

          @Override
          public void onScrubGeo(long l, long l1) {}

          @Override
          public void onException(Exception ex) {}

          @Override
          public void onStallWarning(StallWarning arg0) {
            // TODO Auto-generated method stub

          }
        };

    _twitterStream =
        new TwitterStreamFactory(new ConfigurationBuilder().setJSONStoreEnabled(true).build())
            .getInstance();

    _twitterStream.addListener(listener);
    _twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
    AccessToken token = new AccessToken(accessToken, accessTokenSecret);
    _twitterStream.setOAuthAccessToken(token);

    if (keyWords.length == 0) {

      _twitterStream.sample();
    } else {

      FilterQuery query = new FilterQuery().track(keyWords);
      _twitterStream.filter(query);
    }
  }
Example #13
0
  @Override
  public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    statuses = new LinkedBlockingQueue<Status>(1000);
    this.spoutOutputCollector = collector;

    ConfigurationBuilder configurationBuilder = new ConfigurationBuilder();
    configurationBuilder
        .setOAuthConsumerKey(consumerKey)
        .setOAuthConsumerSecret(consumerSecret)
        .setOAuthAccessToken(accessToken)
        .setOAuthAccessTokenSecret(accessTokenSecret);

    OAuthAuthorization authAuthorization = new OAuthAuthorization(configurationBuilder.build());

    twitterStream = new TwitterStreamFactory().getInstance(authAuthorization);
    twitterStream.addListener(
        new StatusListener() {
          @Override
          public void onStatus(Status status) {
            statuses.offer(status);
          }

          @Override
          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          @Override
          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          @Override
          public void onScrubGeo(long userId, long upToStatusId) {}

          @Override
          public void onStallWarning(StallWarning warning) {}

          @Override
          public void onException(Exception ex) {}
        });

    twitter = new TwitterFactory().getInstance(authAuthorization);
    filterQuery = new FilterQuery();

    if (filterQuery == null) {
      twitterStream.sample();
      ;
    } else {
      twitterStream.filter(filterQuery.track(filterWords));
      twitterStream.filter(filterQuery.language(filterLanguages));
    }
  }
Example #14
0
  private static void startStream(List<String> terms) throws TwitterException {
    if (twitter != null) {
      twitter.cleanUp();
    }
    if (esClient != null) {
      esClient.close();
      esClient = null;
    }

    play.Configuration pconf = Play.application().configuration();
    String elasticSearchCluster = pconf.getString("tweet.elasticsearch.cluster.name");
    if (elasticSearchCluster != null) {
      Logger.info("Configuring ElasticSearch...");
      Settings settings =
          ImmutableSettings.settingsBuilder().put("cluster.name", elasticSearchCluster).build();

      esClient =
          new TransportClient(settings)
              .addTransportAddress(
                  new InetSocketTransportAddress(
                      pconf.getString("tweet.elasticsearch.transport.host"),
                      pconf.getInt("tweet.elasticsearch.transport.port")));
    } else {
      esClient = null;
    }

    twitter4j.conf.Configuration tconf = Application.getTwitterConfiguration();
    TwitterStreamFactory tf = new TwitterStreamFactory(tconf);
    twitter = tf.getInstance();
    StatusListener l =
        new TweetListener(
            terms,
            esClient,
            pconf.getString("tweet.elasticsearch.index"),
            pconf.getString("tweet.elasticsearch.type"));
    twitter.addListener(l);

    String[] tracks = new String[terms.size()];
    StringBuffer termsString = new StringBuffer();
    for (int i = 0; i < terms.size(); i++) {
      tracks[i] = terms.get(i);
      if (i != 0) termsString.append(",");
      termsString.append(terms.get(i));
    }
    FilterQuery q = new FilterQuery().track(tracks);
    twitter.filter(q);
    Logger.info("Starting listening for tweets using terms " + termsString.toString() + "...");
  }
 public static void startStreaming() {
   if (sTwitterStream != null) {
     if (!sTwitterStreamConnected) {
       sUserStreamAdapter.start();
       sTwitterStream.setOAuthAccessToken(AccessTokenManager.getAccessToken());
       sTwitterStream.user();
     }
     return;
   }
   sTwitterStream = getTwitterStream();
   sUserStreamAdapter = new MyUserStreamAdapter();
   sTwitterStream.addListener(sUserStreamAdapter);
   sTwitterStream.addConnectionLifeCycleListener(new MyConnectionLifeCycleListener());
   sTwitterStream.user();
   BasicSettings.resetNotification();
 }
  public static void main(String[] args) {

    Twitter twitter = new TwitterFactory().getInstance();
    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
    MessageListener messageListener = null;
    Thread alertsWatcher = null;
    try {
      messageListener = new MessageListener(twitter, context);
      twitterStream.addListener(messageListener);
      twitterStream.user();
      alertsWatcher = new Thread(new AlertsWatcher(twitter, 15000));
      alertsWatcher.start();
    } catch (TwitterException e) {
      e.printStackTrace();
    }
  }
  @Override
  public void setTweetsListener(TweetsUpdateListener tweetsListener) {
    this.tweetsUpdateListener = tweetsListener;
    String accessToken = PersistUtils.getInstance(context).getAccessToken();
    String accessTokenSecret = PersistUtils.getInstance(context).getAccessTokenSecret();

    Configuration configuration =
        new ConfigurationBuilder()
            .setOAuthConsumerKey(Constants.CONSUMER_KEY)
            .setOAuthConsumerSecret(Constants.CONSUMER_SECRET)
            .setOAuthAccessToken(accessToken)
            .setOAuthAccessTokenSecret(accessTokenSecret)
            .build();

    twitterStream = new TwitterStreamFactory(configuration).getInstance();
    twitterStream.addListener(statusListener);
    twitterStream.user();
  }
  @Override
  public void filter(final TweetFilterQuery filterQuery) {
    final TwitterStream twitterStream =
        new TwitterStreamFactory(TwitterOAuth.getConfiguration()).getInstance();

    twitterStream.addListener(
        new StatusAdapter() {

          @Override
          public void onStatus(Status status) {
            synchronized (TwitterTweetStream.this) {
              if (null != tweetConsumer) {
                tweetConsumer.accept(new TwitterTweet(status));
              }
            }
          }
        });
    twitterStream.filter(getFilterQuery(filterQuery));
  }
  public void initialize(String[] args) {

    String connectionString = "amqp://*****:*****@localhost:5672/";

    if (args.length > 0 && args[0].startsWith("amqp://")) {

      connectionString = args[0];

      this.cmdlineArgs = pruneFirst(args);
    } else {

      this.cmdlineArgs = args;
    }

    // Manually configure the EventManager
    AmqpConfiguration config =
        AmqpConfiguration.getDefault("tweetstream", new AmqpConnectionParameters(connectionString));

    // Initialize the EventManager
    this.eventManager = new AmqpEventManager(config);

    // Start the EventManager
    this.eventManager.start();

    // Create a Twitter Stream instance (3rd Party API)
    this.twitterStream = new TwitterStreamFactory().getInstance();

    // Create a new instance of our Twitter listener that will
    // publish incoming Tweets onto the bus.
    TweetPublisher publishOnBusListener = new TweetPublisher(this.eventManager);

    // Register our listener with the Twitter Stream API
    twitterStream.addListener(publishOnBusListener);

    // Start publishing the TwitterStreamMode
    new TwitterStreamModePublisher(this.getTwitterStreamMode(), this.eventManager).start();

    wireUpBus(this.eventManager);

    startUpTwitter(twitterStream);
  }
  /**
   * This is where the automagic happens We are listening on tweets through the streaming api,
   * whenever we get a tweet "fireStatusEvent" is called
   */
  @PostConstruct
  public void startTweetStream() {
    TwitterStream twitterStream =
        new TwitterStreamFactory(TwitterConfigBuilder.getConfig()).getInstance();

    StatusListener statusListener =
        new StatusListener() {
          @Override
          public void onStatus(Status status) {
            logger.info("Received a status from " + status.getUser().getScreenName());
            Tweet tweet = new Tweet(status);
            fireStatusEvent(tweet);
          }

          @Override
          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          @Override
          public void onTrackLimitationNotice(int i) {}

          @Override
          public void onScrubGeo(long l, long l1) {}

          @Override
          public void onStallWarning(StallWarning stallWarning) {}

          @Override
          public void onException(Exception e) {
            e.printStackTrace();
          }
        };

    FilterQuery filter = new FilterQuery();
    String[] keywords = {"#testtestWOW"};
    filter.track(keywords);

    twitterStream.addListener(statusListener);
    twitterStream.filter(filter);
    logger.warn("Started to listen for " + keywords[0]);
  }
Example #21
0
  /**
   * Main entry of this application.
   *
   * @param args arguments doesn't take effect with this example
   */
  public static void main(String[] args) throws TwitterException {
    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
    StatusListener listener =
        new StatusListener() {
          @Override
          public void onStatus(Status status) {
            System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText());
          }

          @Override
          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            System.out.println(
                "Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
          }

          @Override
          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
          }

          @Override
          public void onScrubGeo(long userId, long upToStatusId) {
            System.out.println(
                "Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
          }

          @Override
          public void onStallWarning(StallWarning warning) {
            System.out.println("Got stall warning:" + warning);
          }

          public void onException(Exception ex) {
            ex.printStackTrace();
          }
        };
    twitterStream.addListener(listener);
    twitterStream.links(0);
  }
 public static void streamTweets(String keywords, String location, Boolean hideRetweets)
     throws URLs.ConnectionException, URLs.HTTPQueryException, GeolocationSearch.NoKeyException,
         GeolocationSearch.SearchLocationException, MalformedURLException {
   twitter4j.TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
   twitterStream.addListener(
       new StatusAdapter() {
         public void onStatus(Status status) {
           if (!(status.isRetweet() && hideRetweets)) {
             printTweet(status, true);
           }
         }
       });
   Double[] coordinates = getCoordinatesByQuery(location);
   double[][] doubleCoordinates = {
     {coordinates[1] - RADIUS, coordinates[0] - RADIUS},
     {coordinates[1] + RADIUS, coordinates[0] + RADIUS}
   };
   String[] arrayKeywords = {keywords};
   FilterQuery fq = new FilterQuery();
   fq.locations(doubleCoordinates);
   fq.track(arrayKeywords);
   twitterStream.filter(fq);
 }
  public void loadMenu() throws InterruptedException {

    System.out.print("Please choose a name for your stream:\t");

    Scanner input = new Scanner(System.in);
    String keyword = input.nextLine();

    connectdb(keyword);

    cb = new ConfigurationBuilder();
    cb.setDebugEnabled(true);
    cb.setOAuthConsumerKey("XXX");
    cb.setOAuthConsumerSecret("XXX");
    cb.setOAuthAccessToken("XXX");
    cb.setOAuthAccessTokenSecret("XXX");

    TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
    StatusListener listener =
        new StatusListener() {

          public void onStatus(Status status) {
            System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText());

            BasicDBObject basicObj = new BasicDBObject();
            basicObj.put("user_name", status.getUser().getScreenName());
            basicObj.put("retweet_count", status.getRetweetCount());
            basicObj.put("tweet_followers_count", status.getUser().getFollowersCount());
            basicObj.put("source", status.getSource());
            // basicObj.put("coordinates",tweet.getGeoLocation());

            UserMentionEntity[] mentioned = status.getUserMentionEntities();
            basicObj.put("tweet_mentioned_count", mentioned.length);
            basicObj.put("tweet_ID", status.getId());
            basicObj.put("tweet_text", status.getText());

            try {
              items.insert(basicObj);
            } catch (Exception e) {
              System.out.println("MongoDB Connection Error : " + e.getMessage());
            }
          }

          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            System.out.println(
                "Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
          }

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
          }

          public void onScrubGeo(long userId, long upToStatusId) {
            System.out.println(
                "Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
          }

          @Override
          public void onStallWarning(StallWarning stallWarning) {
            // To change body of implemented methods use File | Settings | File Templates.
          }

          public void onException(Exception ex) {
            ex.printStackTrace();
          }
        };

    FilterQuery fq = new FilterQuery();
    String keywords[] = {"Germany"};

    fq.track(keywords);

    twitterStream.addListener(listener);
    twitterStream.filter(fq);
  }
  /**
   * Start processing events. This uses the Twitter Streaming API to sample Twitter, and process
   * tweets.
   */
  @Override
  public void start() {
    // The channel is the piece of Flume that sits between the Source and Sink,
    // and is used to process events.
    final ChannelProcessor channel = getChannelProcessor();

    final Map<String, String> headers = new HashMap<String, String>();

    // The StatusListener is a twitter4j API, which can be added to a Twitter
    // stream, and will execute methods every time a message comes in through
    // the stream.
    StatusListener listener =
        new StatusListener() {
          // The onStatus method is executed every time a new tweet comes in.
          public void onStatus(Status status) {
            // The EventBuilder is used to build an event using the headers and
            // the raw JSON of a tweet
            logger.debug(status.getUser().getScreenName() + ": " + status.getText());

            headers.put("timestamp", String.valueOf(status.getCreatedAt().getTime()));
            Event event =
                EventBuilder.withBody(DataObjectFactory.getRawJSON(status).getBytes(), headers);
            channel.processEvent(event);
          }

          // This listener will ignore everything except for new tweets
          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          public void onScrubGeo(long userId, long upToStatusId) {}

          public void onException(Exception ex) {
            logger.error("Stream Error ", ex);
          }

          public void onStallWarning(StallWarning warning) {
            int percentFull = warning.getPercentFull();
            logger.warn("Stall Warning Received ", warning);
            if (percentFull > 95) {
              logger.warn("Stallwarning Stream full more han 95 %. Going to wait for 2 minutes");
              try {
                Thread.sleep(2 * 60 * 000);
              } catch (InterruptedException e) {
                e.printStackTrace();
              }
            }
          }
        };

    logger.debug(
        "Setting up Twitter sample stream using consumer key {} and" + " access token {}",
        new String[] {consumerKey, accessToken});
    // Set up the stream's listener (defined above),
    twitterStream.addListener(listener);

    // Set up a filter to pull out industry-relevant tweets
    if (keywords.length == 0) {
      logger.debug("Starting up Twitter sampling...");
      twitterStream.sample();
    } else {
      logger.debug("Starting up Twitter filtering...");

      FilterQuery query = new FilterQuery().track(keywords);
      twitterStream.filter(query);
    }
    super.start();
  }
  private void start(final Context context) throws IOException {

    // Producer properties
    Properties props = new Properties();
    props.put("metadata.broker.list", context.getString(TwitterSourceConstant.BROKER_LIST));
    props.put("serializer.class", context.getString(TwitterSourceConstant.SERIALIZER));
    props.put("partitioner.class", context.getString(TwitterSourceConstant.PARTITIONER));
    props.put("request.required.acks", context.getString(TwitterSourceConstant.REQUIRED_ACKS));

    ProducerConfig config = new ProducerConfig(props);

    final Producer<String, String> producer = new Producer<String, String>(config);

    /** Twitter properties * */
    consumerKey = context.getString(TwitterSourceConstant.CONSUMER_KEY_KEY);
    consumerSecret = context.getString(TwitterSourceConstant.CONSUMER_SECRET_KEY);
    accessToken = context.getString(TwitterSourceConstant.ACCESS_TOKEN_KEY);
    accessTokenSecret = context.getString(TwitterSourceConstant.ACCESS_TOKEN_SECRET_KEY);

    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setOAuthConsumerKey(consumerKey);
    cb.setOAuthConsumerSecret(consumerSecret);
    cb.setOAuthAccessToken(accessToken);
    cb.setOAuthAccessTokenSecret(accessTokenSecret);
    cb.setJSONStoreEnabled(true);
    cb.setIncludeEntitiesEnabled(true);
    cb.setHttpProxyHost("proxy.tcs.com");
    cb.setHttpProxyPort(8080);
    cb.setHttpProxyUser("876216");
    cb.setHttpProxyPassword("Apple@123");
    twitterStream = new TwitterStreamFactory(cb.build()).getInstance();

    final Map<String, String> headers = new HashMap<String, String>();

    /** Twitter listener * */
    StatusListener listener =
        new StatusListener() {
          // The onStatus method is executed every time a new tweet comes
          // in.
          public void onStatus(Status status) {
            // The EventBuilder is used to build an event using the
            // the raw JSON of a tweet

            System.out.println("Listening :");
            KeyedMessage<String, String> data =
                new KeyedMessage<String, String>(
                    "testing1", TwitterObjectFactory.getRawJSON(status));

            producer.send(data);
            System.out.println(data);
          }

          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          public void onScrubGeo(long userId, long upToStatusId) {}

          public void onException(Exception ex) {
            logger.info("ShutDown");
            twitterStream.shutdown();
          }

          public void onStallWarning(StallWarning warning) {}
        };

    twitterStream.addListener(listener);
    /** GOGOGO * */
    twitterStream.sample();
    FilterQuery query =
        new FilterQuery()
            .track(
                Tweety.hashtags[0],
                Tweety.hashtags[1],
                Tweety.hashtags[2],
                Tweety.hashtags[3],
                Tweety.hashtags[4]);
    twitterStream.filter(query);
    /** Bind the listener * */
  }
Example #26
0
  /*

  * Main entry of this application.
  *
  * @param args
  */
  public static void main(String[] args) throws TwitterException {
    TwitterStream twitterStream = new TwitterStreamFactory().getInstance();

    final DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    final String startDate = formatter.format(new Timestamp((new Date()).getTime()));
    final TweetStats stats = new TweetStats();
    final long startTime = System.currentTimeMillis();
    // to split the tweet text into words
    final Pattern splitPattern = Pattern.compile("\\b");
    final Pattern nameFilter = Pattern.compile("^[a-zA-Z0-9_]+$");

    // System.out.println("digraph mentions {");
    // I did not find a different way to count words:
    // hashtables in java allows to manage sets of data with a <key,value> structure
    // in this part of the worl, the key is the word and the value is the count.
    final Hashtable<String, Integer> wordstats = new Hashtable<String, Integer>();
    final DecimalFormat df = new DecimalFormat("#.###");
    StatusListener listener =
        new StatusListener() {
          public void onStatus(Status status) {
            long actualTime = System.currentTimeMillis();
            String text = status.getText().toLowerCase();
            String wordsPerText[] = splitPattern.split(text);

            // to avoid problems with the counting process, this work use synchronized
            synchronized (this) {
              stats.incrementTweetCount();
              long tweetCount = stats.getTweetCount();
              long timeElapsed = (actualTime - startTime) / 1000;
              long statET = stats.getElapsedTime();
              long mentionCount;

              // with this code, I show the stats every 15 seconds, comparing the change of time

              List<String> names;
              Extractor extractor = new Extractor();

              names = extractor.extractMentionedScreennames(status.getText());
              mentionCount = stats.getMentionCount();

              String userName = status.getUser().getName().toLowerCase();

              // I had some problems with international names and the graphic tools, so i decided
              // To filer all the names with non ascii characters.
              Matcher matcherUserName = nameFilter.matcher(userName);
              if (matcherUserName.find() && names.size() > 0) {
                // I'm countiong the act of mention one or more tweeters as one action.
                stats.incrementMentionCount();
                for (String name : names) {
                  String mentionName = name.toLowerCase();
                  Matcher matcherMention = nameFilter.matcher(mentionName);
                  if (matcherMention.find()) {
                    // userName = userName.replaceAll("\\u202E|\\u200E|\\t|\\r|\\n", "").trim();
                    // mentionName = mentionName.replaceAll("\\u202E|\\u200E|\\t|\\r|\\n",
                    // "").trim();
                    // Are there other cleanings on names???
                    System.out.println("\"" + userName + "\",\"" + mentionName + "\"");
                  }
                }
              }

              if (mentionCount > 30000
                  && timeElapsed > 0
                  && statET != timeElapsed
                  && timeElapsed % 15 == 0) {
                // System.out.println("}");
                stats.setElapsedTime(timeElapsed);
                String report = "";
                report += " |Date: " + startDate;
                report += " |Tweet Count: " + tweetCount;
                report += " |Mention Cpunt: " + mentionCount;
                report += " |Elapsed Time (s): " + timeElapsed;
                report += " |Avg. (Tweets per Second): " + (double) (tweetCount / timeElapsed);
                System.out.println(report);

                System.exit(0);
              }
            }
          }

          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
            // System.out.println("Got a status deletion notice id:" +
            // statusDeletionNotice.getStatusId());
          }

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
            // System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
          }

          public void onScrubGeo(long userId, long upToStatusId) {
            // System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" +
            // upToStatusId);
          }

          public void onException(Exception ex) {
            ex.printStackTrace();
          }
        };
    twitterStream.addListener(listener);
    twitterStream.sample();
  }
Example #27
0
  public static void main(String[] args)
      throws IOException, TwitterException, FileNotFoundException {
    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setDebugEnabled(true)
        .setOAuthConsumerKey("vGsMAca82HjVYYm7wQlN5Q")
        .setOAuthConsumerSecret("6mpRAAlMI6sIWruQNpomBzN1kIfodHexYBrKWPKmsuM")
        .setOAuthAccessToken("776768857-hCnK0Eu3orKK6qP98W7LyTF29QXadxydL21gIuon")
        .setOAuthAccessTokenSecret("V3VAJgO2x1zWO2JMHsRRrWLzYOeXxmlittF1pBBNWA4");

    TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
    System.out.println("time " + System.currentTimeMillis());

    StatusListener listener =
        new StatusListener() {
          int counter = 0;
          FileWriter fstream = new FileWriter("/Users/jacobportnoff/Desktop/week7SunJ.txt");
          BufferedWriter out = new BufferedWriter(fstream);
          long start = System.currentTimeMillis();
          double elapsedTimeSec = 0.0;

          public void onStatus(Status status) {
            HashtagEntity[] hashtags = status.getHashtagEntities();
            String hashes = "";
            if (hashtags.length == 0) {
              hashes = "null";
            } else {
              for (HashtagEntity hash : hashtags) {
                hashes = hashes + hash.getText() + " ";
              }
            }
            String tweet =
                status.getCreatedAt()
                    + "\t"
                    + status.getUser().getScreenName()
                    + "\t"
                    + status.getText()
                    + "\t"
                    + status.getGeoLocation()
                    + "\t"
                    + hashes
                    + "\n";
            // System.out.print(tweet);
            try {
              out.write(tweet);
            } catch (Exception e) { // Catch exception if any
              System.err.println("Error: " + e.getMessage());
            }
          }

          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          public void onScrubGeo(long userId, long upToStatusId) {}

          public void onException(Exception ex) {
            ex.printStackTrace();
          }
        };
    twitterStream.addListener(listener);
    twitterStream.sample();
  }
Example #28
0
  private void start(final Context context) {

    /** Producer properties * */
    Properties props = new Properties();
    props.put("metadata.broker.list", context.getString(BROKER_LIST));
    props.put("serializer.class", context.getString(SERIALIZER));
    props.put("request.required.acks", context.getString(REQUIRED_ACKS));

    ProducerConfig config = new ProducerConfig(props);
    final Producer<String, String> producer = new Producer<String, String>(config);

    /** Twitter properties * */
    consumerKey = context.getString(CONSUMER_KEY_KEY);
    consumerSecret = context.getString(CONSUMER_SECRET_KEY);
    accessToken = context.getString(ACCESS_TOKEN_KEY);
    accessTokenSecret = context.getString(ACCESS_TOKEN_SECRET_KEY);

    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setOAuthConsumerKey(consumerKey);
    cb.setOAuthConsumerSecret(consumerSecret);
    cb.setOAuthAccessToken(accessToken);
    cb.setOAuthAccessTokenSecret(accessTokenSecret);
    cb.setJSONStoreEnabled(true);
    cb.setIncludeEntitiesEnabled(true);

    twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
    final boolean shouldPrintTweetsOnScreen =
        Boolean.parseBoolean(context.getString(printTweetsOnScreen));
    final boolean shouldSendTweetsToKafka =
        Boolean.parseBoolean(context.getString(sendTweetsToKafka));

    final StatusListener listener =
        new StatusListener() {
          // The onStatus method is executed every time a new tweet comes
          // in.
          public void onStatus(Status status) {
            // The EventBuilder is used to build an event using the
            // the raw JSON of a tweet
            if (shouldPrintTweetsOnScreen) {
              logger.info(status.getUser().getScreenName() + ": " + status.getText());
            }

            if (shouldSendTweetsToKafka) {
              KeyedMessage<String, String> data =
                  new KeyedMessage<String, String>(
                      context.getString(KAFKA_TOPIC), TwitterObjectFactory.getRawJSON(status));
              producer.send(data);
            }
          }

          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          public void onScrubGeo(long userId, long upToStatusId) {}

          public void onException(Exception ex) {
            logger.info("Shutting down Twitter sample stream...");
            twitterStream.shutdown();
          }

          public void onStallWarning(StallWarning warning) {}
        };

    twitterStream.addListener(listener);

    twitterStream.filter(new FilterQuery().track(context.getString(keywords).split(",")));
  }
Example #29
0
  public static void main(String args[]) {
    FilterQuery tweetFilterQuery = new FilterQuery();
    String a[] = {"en"};
    tweetFilterQuery.language(a);
    ConfigurationBuilder _configurationBuilder = new ConfigurationBuilder();
    _configurationBuilder
        .setOAuthConsumerKey(consumerKey)
        .setOAuthConsumerSecret(consumerSecret)
        .setOAuthAccessToken(accessToken)
        .setOAuthAccessTokenSecret(accessTokenSecret)
        .setJSONStoreEnabled(true);

    twitterStream = new TwitterStreamFactory(_configurationBuilder.build()).getInstance();

    StatusListener listener =
        new StatusListener() {

          @Override
          public void onException(Exception arg0) {
            System.out.println("Error occured: " + arg0.getMessage());
            arg0.printStackTrace();
          }

          @Override
          public void onTrackLimitationNotice(int arg0) {
            System.out.println("Track limitation notice for " + arg0);
          }

          @Override
          public void onStatus(Status status) {
            // if(status.getLang().equals("en"))
            //	System.out.println(status.getText().replaceAll("[\n\r]", ""));
            String filename = totalCount / 100 + ".txt";
            File f = new File(filename);
            PrintWriter out = null;

            if (f.exists() && !f.isDirectory()) {
              // System.out.println("@" + status.getUser().getScreenName() + " - " +
              // status.getText() + " -> "+ status.getCreatedAt());
              // String mapstring = status.getUser().getScreenName() + " - " + status.getText() + "
              // -> "+ status.getCreatedAt() +"\n";
              String mapstring = status.getText().replace("\n", " ") + "\n";

              try {
                out = new PrintWriter(new FileOutputStream(new File(filename), true));
              } catch (FileNotFoundException e) {
                e.printStackTrace();
              }
              out.append(mapstring);
              out.close();
            } else {

              try {
                out = new PrintWriter(filename);
              } catch (FileNotFoundException e) {
                e.printStackTrace();
              }
              String mapstring = status.getText().replace("\n", " ") + "\n";
              out.println(mapstring);
              out.close();
            }
            totalCount++;
          }

          @Override
          public void onStallWarning(StallWarning arg0) {
            // TODO Auto-generated method stub

          }

          @Override
          public void onScrubGeo(long arg0, long arg1) {
            // TODO Auto-generated method stub

          }

          @Override
          public void onDeletionNotice(StatusDeletionNotice arg0) {
            // TODO Auto-generated method stub

          }
        };
    twitterStream.addListener(listener);

    twitterStream.sample();
  }