private void initStream() {
tweetsFlowRegulator = TweetsFlowRegulator.newInstance(listener);
twitterStream.addListener(
new StatusListener() {
@Override
public void onStatus(Status status) {
tweetsFlowRegulator.addTweet(status);
}
@Override
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
// No-op
}
@Override
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
// No-op
}
@Override
public void onScrubGeo(long userId, long upToStatusId) {
// No-op
}
@Override
public void onStallWarning(StallWarning warning) {
Log.w(warning);
}
@Override
public void onException(Exception ex) {
Log.e(ex);
}
});
}
public static void main(String[] args) throws Exception {
ConfigurationBuilder builder = new ConfigurationBuilder();
builder.setOAuthConsumerKey(CONSUMER_KEY);
builder.setOAuthConsumerSecret(CONSUMER_SECRET);
builder.setOAuthAccessToken(ACCESS_TOKEN);
builder.setOAuthAccessTokenSecret(ACCESS_TOKEN_SECRET);
// 現行のTwitter4JではAPIのデフォルト呼び先がbetastream.twitter.comになっているので修正
builder.setUserStreamBaseURL("https://userstream.twitter.com/2/");
// Configurationを生成
Configuration configuration = builder.build();
/* Configuration configuration = new ConfigurationBuilder().setOAuthConsumerKey(CONSUMER_KEY)
.setOAuthConsumerSecret(CONSUMER_SECRET)
.setOAuthAccessToken(ACCESS_TOKEN)
.setOAuthAccessTokenSecret(ACCESS_TOKEN_SECRET)
.build();
*/
TwitterStream twStream = new TwitterStreamFactory(configuration).getInstance();
// TwitterStream twStream = new TwitterStreamFactory().getInstance();
twStream.addListener(new MyStatusListener());
twStream.user();
// twStream.sample();
}
@Override
public IBinder onBind(Intent intent) {
Log.d(TAG, "onBind()");
configurationBuilder = new ConfigurationBuilder();
configurationBuilder
.setDebugEnabled(true)
.setOAuthConsumerKey("JmPCgGdftlNXuh21WQ7hFA")
.setOAuthConsumerSecret("drMLhPvOWs2Crol2LwQuqdKVRTFCVbQlkJQOCrV8uI")
.setOAuthAccessToken("72023528-NFWdbv2h4vDVdZC1ML2jNT0gXt9fqZLpMdvtGDjnH")
.setOAuthAccessTokenSecret("JW7Y2e8D086oDsU1wpNKgtsPZAwF1TQl5KkMdbHdnQ");
twitterStream = new TwitterStreamFactory(configurationBuilder.build()).getInstance();
StatusListener statusListener =
new StatusListener() {
@Override
public void onException(Exception arg0) {
Log.d(TAG, "onException: cause," + arg0.getCause() + " message," + arg0.getMessage());
}
@Override
public void onDeletionNotice(StatusDeletionNotice arg0) {
Log.d(TAG, "onDeletionNotice()");
}
@Override
public void onScrubGeo(long arg0, long arg1) {
Log.d(TAG, "onScrubGeo()");
}
@Override
public void onStallWarning(StallWarning stallWarning) {
Log.d(TAG, "onStallWarning()");
}
@Override
public void onStatus(Status status) {
Log.d(
TAG,
"Twitter Status Received: Geolocation?:"
+ (status.getGeoLocation() != null)
+ " / "
+ status.getUser().getScreenName()
+ ": "
+ status.getText());
if (status.getGeoLocation() != null) {
addNewTwitterStatus(status);
}
}
@Override
public void onTrackLimitationNotice(int arg0) {
Log.d(TAG, "onTrackLimitationNotice()");
}
};
twitterStream.addListener(statusListener);
return mBinder;
}
/**
* Start processing events. This uses the Twitter Streaming API to sample Twitter, and process
* tweets.
*/
@Override
public void start() {
// The channel is the piece of Flume that sits between the Source and Sink,
// and is used to process events.
final ChannelProcessor channel = getChannelProcessor();
final Map<String, String> headers = new HashMap<String, String>();
// The StatusListener is a twitter4j API, which can be added to a Twitter
// stream, and will execute methods every time a message comes in through
// the stream.
StatusListener listener =
new StatusListener() {
// The onStatus method is executed every time a new tweet comes in.
public void onStatus(Status status) {
// The EventBuilder is used to build an event using the headers and
// the raw JSON of a tweet
logger.debug(status.getUser().getScreenName() + ": " + status.getText());
headers.put("timestamp", String.valueOf(status.getCreatedAt().getTime()));
Event event =
EventBuilder.withBody(DataObjectFactory.getRawJSON(status).getBytes(), headers);
channel.processEvent(event);
}
// This listener will ignore everything except for new tweets
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}
public void onScrubGeo(long userId, long upToStatusId) {}
public void onException(Exception ex) {}
public void onStallWarning(StallWarning warning) {}
};
logger.debug(
"Setting up Twitter sample stream using consumer key {} and" + " access token {}",
new String[] {consumerKey, accessToken});
// Set up the stream's listener (defined above), and set any necessary
// security information.
twitterStream.addListener(listener);
twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
AccessToken token = new AccessToken(accessToken, accessTokenSecret);
twitterStream.setOAuthAccessToken(token);
// Set up a filter to pull out industry-relevant tweets
if (keywords.length == 0) {
logger.debug("Starting up Twitter sampling...");
twitterStream.sample();
} else {
logger.debug("Starting up Twitter filtering...");
FilterQuery query = new FilterQuery().track(keywords);
twitterStream.filter(query);
}
super.start();
}
public boolean startSample(long lifeTime) {
if (accessConfigBuilder == null) return false;
StatusListener listener = new SampleStreamListener(lifeTime);
TwitterStream twitterStream =
new TwitterStreamFactory(accessConfigBuilder.build()).getInstance();
twitterStream.addListener(listener);
twitterStream.sample();
return true;
}
/** Start listening to tweets. */
public void start() throws IllegalStateException, TwitterException {
twitter = new TwitterFactory().getInstance();
stream = new TwitterStreamFactory().getInstance();
LOGGER.info("Connected to Twitter as: {}", twitter.getScreenName());
LOGGER.info("Starting the Twitter stream listener");
stream.addListener(new GetMentionsListener(twitter));
stream.user(); // Start reading to user stream
}
/**
* Main entry of this application.
*
* @param args follow(comma separated user ids) track(comma separated filter terms)
* @throws twitter4j.TwitterException
*/
public static void main(String[] args) throws TwitterException {
if (args.length < 1) {
System.out.println(
"Usage: java twitter4j.examples.PrintFilterStream [follow(comma separated numerical user ids)] [track(comma separated filter terms)]");
System.exit(-1);
}
StatusListener listener =
new StatusListener() {
public void onStatus(Status status) {
System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText());
}
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
System.out.println(
"Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
}
public void onScrubGeo(long userId, long upToStatusId) {
System.out.println(
"Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
}
public void onException(Exception ex) {
ex.printStackTrace();
}
};
TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
twitterStream.addListener(listener);
ArrayList<Long> follow = new ArrayList<Long>();
ArrayList<String> track = new ArrayList<String>();
for (String arg : args) {
if (isNumericalArgument(arg)) {
for (String id : arg.split(",")) {
follow.add(Long.parseLong(id));
}
} else {
track.addAll(Arrays.asList(arg.split(",")));
}
}
long[] followArray = new long[follow.size()];
for (int i = 0; i < follow.size(); i++) {
followArray[i] = follow.get(i);
}
String[] trackArray = track.toArray(new String[track.size()]);
// filter() method internally creates a thread which manipulates TwitterStream and calls these
// adequate listener methods continuously.
twitterStream.filter(new FilterQuery(0, followArray, trackArray));
}
/**
* Simply starts the stream.
*
* @param id the id
*/
@RequestMapping(value = "/start/{id}", method = RequestMethod.GET)
@ResponseStatus(value = HttpStatus.OK, reason = "Successfully started stream")
public final void start(@PathVariable final long id) {
LOGGER.info("Shutting down stream");
stream.shutdown();
stream.addListener(listener);
List<String> portfolioTokens = portfolioService.getStreamTokens(id);
stream.filter(
new FilterQuery(
0, new long[0], portfolioTokens.toArray(new String[portfolioTokens.size()])));
LOGGER.info("Successfully started stream: " + portfolioTokens);
}
/**
* Method that handles the Twitter streaming API. <br>
* <b>WARNING:</b> Method does not terminate by itself, due to the fact that the streamer runs in
* a different thread.
*
* @param keywords The keywords for which the streamer searches for tweets.
* @param mongoDB A handler for the MongoDB database.
* @param config A configuration object.
*/
public final void retrieveTweetsWithStreamingAPI(
String[] keywords, MongoHandler mongoDB, Config config) {
ConfigurationBuilder cb = getAuthorization();
TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
final StatusListener listener;
listener =
new StatusListener() {
@Override
public final void onStatus(Status status) {
// Insert tweet to MongoDB
mongoDB.insertSingleTweetIntoMongoDB(status, "NULL");
}
@Override
public final void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
PrintUtilities.printInfoMessageln(
"Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
}
@Override
public final void onTrackLimitationNotice(int numberOfLimitedStatuses) {
PrintUtilities.printInfoMessageln(
"Got track limitation notice:" + numberOfLimitedStatuses);
}
@Override
public final void onScrubGeo(long userId, long upToStatusId) {
PrintUtilities.printInfoMessageln(
"Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
}
@Override
public final void onStallWarning(StallWarning warning) {
PrintUtilities.printInfoMessageln("Got stall warning:" + warning);
}
@Override
public final void onException(Exception ex) {
ex.printStackTrace(System.out);
}
};
FilterQuery fq = new FilterQuery();
fq.language("en"); // Set language of tweets to "English"
fq.track(keywords); // Load the search terms
twitterStream.addListener(listener); // Start listening to the stream
twitterStream.filter(fq); // Apply the search filters
}
public static void main(String[] args) throws TwitterException {
FilterQuery query = new FilterQuery();
// wfrank 7897072
// ponja 38299186
String[] tracks = {"diegoram", "cfkargentina"};
// query.track(tracks);
long[] ids = {7897072L, 38299186L, 17760769L};
query.follow(ids);
TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
StatusListener listener =
new StatusListener() {
@Override
public void onStatus(Status status) {
System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText());
}
@Override
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
System.out.println(
"Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
}
@Override
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
}
@Override
public void onScrubGeo(long userId, long upToStatusId) {
System.out.println(
"Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
}
// @Override
// public void onStallWarning(StallWarning warning) {
// System.out.println("Got stall warning:" + warning);
// }
@Override
public void onException(Exception ex) {
ex.printStackTrace();
}
};
twitterStream.addListener(listener);
twitterStream.filter(query);
}
/** Starts listening for twitter events */
@Override
public void start() {
// The channel connects from source to sink
final ChannelProcessor channel = getChannelProcessor();
final Map<String, String> headers = new HashMap<String, String>();
// The method onStatus() will be called everytime a tweet comes.
StatusListener listener =
new StatusListener() {
// capture new tweet notification
public void onStatus(Status status) {
// add creation time in the header
headers.put("timestamp", String.valueOf(status.getCreatedAt().getTime()));
Event event =
EventBuilder.withBody(DataObjectFactory.getRawJSON(status).getBytes(), headers);
// send to sink
channel.processEvent(event);
}
// ignore all other notifications
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}
public void onScrubGeo(long userId, long upToStatusId) {}
public void onException(Exception ex) {}
@Override
public void onStallWarning(StallWarning arg0) {
// do nothing
}
};
// add the listener we created + tell the stream all about the security info required
twitterStream.addListener(listener);
twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
AccessToken token = new AccessToken(accessToken, accessTokenSecret);
twitterStream.setOAuthAccessToken(token);
// Set up a filter to pull out industry-relevant tweets
if (searchFor.length == 0) {
System.out.println("Please setup filter keyword in Flume conf");
} else {
// create a filter query for filtering out only the required info
FilterQuery query = new FilterQuery().track(searchFor);
twitterStream.filter(query);
}
super.start();
}
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
queue = new LinkedBlockingQueue<Status>(1000);
_collector = collector;
StatusListener listener =
new StatusListener() {
@Override
public void onStatus(Status status) {
queue.offer(status);
}
@Override
public void onDeletionNotice(StatusDeletionNotice sdn) {}
@Override
public void onTrackLimitationNotice(int i) {}
@Override
public void onScrubGeo(long l, long l1) {}
@Override
public void onException(Exception ex) {}
@Override
public void onStallWarning(StallWarning arg0) {
// TODO Auto-generated method stub
}
};
_twitterStream =
new TwitterStreamFactory(new ConfigurationBuilder().setJSONStoreEnabled(true).build())
.getInstance();
_twitterStream.addListener(listener);
_twitterStream.setOAuthConsumer(consumerKey, consumerSecret);
AccessToken token = new AccessToken(accessToken, accessTokenSecret);
_twitterStream.setOAuthAccessToken(token);
if (keyWords.length == 0) {
_twitterStream.sample();
} else {
FilterQuery query = new FilterQuery().track(keyWords);
_twitterStream.filter(query);
}
}
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
statuses = new LinkedBlockingQueue<Status>(1000);
this.spoutOutputCollector = collector;
ConfigurationBuilder configurationBuilder = new ConfigurationBuilder();
configurationBuilder
.setOAuthConsumerKey(consumerKey)
.setOAuthConsumerSecret(consumerSecret)
.setOAuthAccessToken(accessToken)
.setOAuthAccessTokenSecret(accessTokenSecret);
OAuthAuthorization authAuthorization = new OAuthAuthorization(configurationBuilder.build());
twitterStream = new TwitterStreamFactory().getInstance(authAuthorization);
twitterStream.addListener(
new StatusListener() {
@Override
public void onStatus(Status status) {
statuses.offer(status);
}
@Override
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}
@Override
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}
@Override
public void onScrubGeo(long userId, long upToStatusId) {}
@Override
public void onStallWarning(StallWarning warning) {}
@Override
public void onException(Exception ex) {}
});
twitter = new TwitterFactory().getInstance(authAuthorization);
filterQuery = new FilterQuery();
if (filterQuery == null) {
twitterStream.sample();
;
} else {
twitterStream.filter(filterQuery.track(filterWords));
twitterStream.filter(filterQuery.language(filterLanguages));
}
}
private static void startStream(List<String> terms) throws TwitterException {
if (twitter != null) {
twitter.cleanUp();
}
if (esClient != null) {
esClient.close();
esClient = null;
}
play.Configuration pconf = Play.application().configuration();
String elasticSearchCluster = pconf.getString("tweet.elasticsearch.cluster.name");
if (elasticSearchCluster != null) {
Logger.info("Configuring ElasticSearch...");
Settings settings =
ImmutableSettings.settingsBuilder().put("cluster.name", elasticSearchCluster).build();
esClient =
new TransportClient(settings)
.addTransportAddress(
new InetSocketTransportAddress(
pconf.getString("tweet.elasticsearch.transport.host"),
pconf.getInt("tweet.elasticsearch.transport.port")));
} else {
esClient = null;
}
twitter4j.conf.Configuration tconf = Application.getTwitterConfiguration();
TwitterStreamFactory tf = new TwitterStreamFactory(tconf);
twitter = tf.getInstance();
StatusListener l =
new TweetListener(
terms,
esClient,
pconf.getString("tweet.elasticsearch.index"),
pconf.getString("tweet.elasticsearch.type"));
twitter.addListener(l);
String[] tracks = new String[terms.size()];
StringBuffer termsString = new StringBuffer();
for (int i = 0; i < terms.size(); i++) {
tracks[i] = terms.get(i);
if (i != 0) termsString.append(",");
termsString.append(terms.get(i));
}
FilterQuery q = new FilterQuery().track(tracks);
twitter.filter(q);
Logger.info("Starting listening for tweets using terms " + termsString.toString() + "...");
}
public static void startStreaming() {
if (sTwitterStream != null) {
if (!sTwitterStreamConnected) {
sUserStreamAdapter.start();
sTwitterStream.setOAuthAccessToken(AccessTokenManager.getAccessToken());
sTwitterStream.user();
}
return;
}
sTwitterStream = getTwitterStream();
sUserStreamAdapter = new MyUserStreamAdapter();
sTwitterStream.addListener(sUserStreamAdapter);
sTwitterStream.addConnectionLifeCycleListener(new MyConnectionLifeCycleListener());
sTwitterStream.user();
BasicSettings.resetNotification();
}
public static void main(String[] args) {
Twitter twitter = new TwitterFactory().getInstance();
TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
MessageListener messageListener = null;
Thread alertsWatcher = null;
try {
messageListener = new MessageListener(twitter, context);
twitterStream.addListener(messageListener);
twitterStream.user();
alertsWatcher = new Thread(new AlertsWatcher(twitter, 15000));
alertsWatcher.start();
} catch (TwitterException e) {
e.printStackTrace();
}
}
@Override
public void setTweetsListener(TweetsUpdateListener tweetsListener) {
this.tweetsUpdateListener = tweetsListener;
String accessToken = PersistUtils.getInstance(context).getAccessToken();
String accessTokenSecret = PersistUtils.getInstance(context).getAccessTokenSecret();
Configuration configuration =
new ConfigurationBuilder()
.setOAuthConsumerKey(Constants.CONSUMER_KEY)
.setOAuthConsumerSecret(Constants.CONSUMER_SECRET)
.setOAuthAccessToken(accessToken)
.setOAuthAccessTokenSecret(accessTokenSecret)
.build();
twitterStream = new TwitterStreamFactory(configuration).getInstance();
twitterStream.addListener(statusListener);
twitterStream.user();
}
@Override
public void filter(final TweetFilterQuery filterQuery) {
final TwitterStream twitterStream =
new TwitterStreamFactory(TwitterOAuth.getConfiguration()).getInstance();
twitterStream.addListener(
new StatusAdapter() {
@Override
public void onStatus(Status status) {
synchronized (TwitterTweetStream.this) {
if (null != tweetConsumer) {
tweetConsumer.accept(new TwitterTweet(status));
}
}
}
});
twitterStream.filter(getFilterQuery(filterQuery));
}
public void initialize(String[] args) {
String connectionString = "amqp://*****:*****@localhost:5672/";
if (args.length > 0 && args[0].startsWith("amqp://")) {
connectionString = args[0];
this.cmdlineArgs = pruneFirst(args);
} else {
this.cmdlineArgs = args;
}
// Manually configure the EventManager
AmqpConfiguration config =
AmqpConfiguration.getDefault("tweetstream", new AmqpConnectionParameters(connectionString));
// Initialize the EventManager
this.eventManager = new AmqpEventManager(config);
// Start the EventManager
this.eventManager.start();
// Create a Twitter Stream instance (3rd Party API)
this.twitterStream = new TwitterStreamFactory().getInstance();
// Create a new instance of our Twitter listener that will
// publish incoming Tweets onto the bus.
TweetPublisher publishOnBusListener = new TweetPublisher(this.eventManager);
// Register our listener with the Twitter Stream API
twitterStream.addListener(publishOnBusListener);
// Start publishing the TwitterStreamMode
new TwitterStreamModePublisher(this.getTwitterStreamMode(), this.eventManager).start();
wireUpBus(this.eventManager);
startUpTwitter(twitterStream);
}
/**
* This is where the automagic happens We are listening on tweets through the streaming api,
* whenever we get a tweet "fireStatusEvent" is called
*/
@PostConstruct
public void startTweetStream() {
TwitterStream twitterStream =
new TwitterStreamFactory(TwitterConfigBuilder.getConfig()).getInstance();
StatusListener statusListener =
new StatusListener() {
@Override
public void onStatus(Status status) {
logger.info("Received a status from " + status.getUser().getScreenName());
Tweet tweet = new Tweet(status);
fireStatusEvent(tweet);
}
@Override
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}
@Override
public void onTrackLimitationNotice(int i) {}
@Override
public void onScrubGeo(long l, long l1) {}
@Override
public void onStallWarning(StallWarning stallWarning) {}
@Override
public void onException(Exception e) {
e.printStackTrace();
}
};
FilterQuery filter = new FilterQuery();
String[] keywords = {"#testtestWOW"};
filter.track(keywords);
twitterStream.addListener(statusListener);
twitterStream.filter(filter);
logger.warn("Started to listen for " + keywords[0]);
}
/**
* Main entry of this application.
*
* @param args arguments doesn't take effect with this example
*/
public static void main(String[] args) throws TwitterException {
TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
StatusListener listener =
new StatusListener() {
@Override
public void onStatus(Status status) {
System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText());
}
@Override
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
System.out.println(
"Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
}
@Override
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
}
@Override
public void onScrubGeo(long userId, long upToStatusId) {
System.out.println(
"Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
}
@Override
public void onStallWarning(StallWarning warning) {
System.out.println("Got stall warning:" + warning);
}
public void onException(Exception ex) {
ex.printStackTrace();
}
};
twitterStream.addListener(listener);
twitterStream.links(0);
}
public static void streamTweets(String keywords, String location, Boolean hideRetweets)
throws URLs.ConnectionException, URLs.HTTPQueryException, GeolocationSearch.NoKeyException,
GeolocationSearch.SearchLocationException, MalformedURLException {
twitter4j.TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
twitterStream.addListener(
new StatusAdapter() {
public void onStatus(Status status) {
if (!(status.isRetweet() && hideRetweets)) {
printTweet(status, true);
}
}
});
Double[] coordinates = getCoordinatesByQuery(location);
double[][] doubleCoordinates = {
{coordinates[1] - RADIUS, coordinates[0] - RADIUS},
{coordinates[1] + RADIUS, coordinates[0] + RADIUS}
};
String[] arrayKeywords = {keywords};
FilterQuery fq = new FilterQuery();
fq.locations(doubleCoordinates);
fq.track(arrayKeywords);
twitterStream.filter(fq);
}
public void loadMenu() throws InterruptedException {
System.out.print("Please choose a name for your stream:\t");
Scanner input = new Scanner(System.in);
String keyword = input.nextLine();
connectdb(keyword);
cb = new ConfigurationBuilder();
cb.setDebugEnabled(true);
cb.setOAuthConsumerKey("XXX");
cb.setOAuthConsumerSecret("XXX");
cb.setOAuthAccessToken("XXX");
cb.setOAuthAccessTokenSecret("XXX");
TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
StatusListener listener =
new StatusListener() {
public void onStatus(Status status) {
System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText());
BasicDBObject basicObj = new BasicDBObject();
basicObj.put("user_name", status.getUser().getScreenName());
basicObj.put("retweet_count", status.getRetweetCount());
basicObj.put("tweet_followers_count", status.getUser().getFollowersCount());
basicObj.put("source", status.getSource());
// basicObj.put("coordinates",tweet.getGeoLocation());
UserMentionEntity[] mentioned = status.getUserMentionEntities();
basicObj.put("tweet_mentioned_count", mentioned.length);
basicObj.put("tweet_ID", status.getId());
basicObj.put("tweet_text", status.getText());
try {
items.insert(basicObj);
} catch (Exception e) {
System.out.println("MongoDB Connection Error : " + e.getMessage());
}
}
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
System.out.println(
"Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
}
public void onScrubGeo(long userId, long upToStatusId) {
System.out.println(
"Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
}
@Override
public void onStallWarning(StallWarning stallWarning) {
// To change body of implemented methods use File | Settings | File Templates.
}
public void onException(Exception ex) {
ex.printStackTrace();
}
};
FilterQuery fq = new FilterQuery();
String keywords[] = {"Germany"};
fq.track(keywords);
twitterStream.addListener(listener);
twitterStream.filter(fq);
}
/**
* Start processing events. This uses the Twitter Streaming API to sample Twitter, and process
* tweets.
*/
@Override
public void start() {
// The channel is the piece of Flume that sits between the Source and Sink,
// and is used to process events.
final ChannelProcessor channel = getChannelProcessor();
final Map<String, String> headers = new HashMap<String, String>();
// The StatusListener is a twitter4j API, which can be added to a Twitter
// stream, and will execute methods every time a message comes in through
// the stream.
StatusListener listener =
new StatusListener() {
// The onStatus method is executed every time a new tweet comes in.
public void onStatus(Status status) {
// The EventBuilder is used to build an event using the headers and
// the raw JSON of a tweet
logger.debug(status.getUser().getScreenName() + ": " + status.getText());
headers.put("timestamp", String.valueOf(status.getCreatedAt().getTime()));
Event event =
EventBuilder.withBody(DataObjectFactory.getRawJSON(status).getBytes(), headers);
channel.processEvent(event);
}
// This listener will ignore everything except for new tweets
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}
public void onScrubGeo(long userId, long upToStatusId) {}
public void onException(Exception ex) {
logger.error("Stream Error ", ex);
}
public void onStallWarning(StallWarning warning) {
int percentFull = warning.getPercentFull();
logger.warn("Stall Warning Received ", warning);
if (percentFull > 95) {
logger.warn("Stallwarning Stream full more han 95 %. Going to wait for 2 minutes");
try {
Thread.sleep(2 * 60 * 000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
};
logger.debug(
"Setting up Twitter sample stream using consumer key {} and" + " access token {}",
new String[] {consumerKey, accessToken});
// Set up the stream's listener (defined above),
twitterStream.addListener(listener);
// Set up a filter to pull out industry-relevant tweets
if (keywords.length == 0) {
logger.debug("Starting up Twitter sampling...");
twitterStream.sample();
} else {
logger.debug("Starting up Twitter filtering...");
FilterQuery query = new FilterQuery().track(keywords);
twitterStream.filter(query);
}
super.start();
}
private void start(final Context context) throws IOException {
// Producer properties
Properties props = new Properties();
props.put("metadata.broker.list", context.getString(TwitterSourceConstant.BROKER_LIST));
props.put("serializer.class", context.getString(TwitterSourceConstant.SERIALIZER));
props.put("partitioner.class", context.getString(TwitterSourceConstant.PARTITIONER));
props.put("request.required.acks", context.getString(TwitterSourceConstant.REQUIRED_ACKS));
ProducerConfig config = new ProducerConfig(props);
final Producer<String, String> producer = new Producer<String, String>(config);
/** Twitter properties * */
consumerKey = context.getString(TwitterSourceConstant.CONSUMER_KEY_KEY);
consumerSecret = context.getString(TwitterSourceConstant.CONSUMER_SECRET_KEY);
accessToken = context.getString(TwitterSourceConstant.ACCESS_TOKEN_KEY);
accessTokenSecret = context.getString(TwitterSourceConstant.ACCESS_TOKEN_SECRET_KEY);
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setOAuthConsumerKey(consumerKey);
cb.setOAuthConsumerSecret(consumerSecret);
cb.setOAuthAccessToken(accessToken);
cb.setOAuthAccessTokenSecret(accessTokenSecret);
cb.setJSONStoreEnabled(true);
cb.setIncludeEntitiesEnabled(true);
cb.setHttpProxyHost("proxy.tcs.com");
cb.setHttpProxyPort(8080);
cb.setHttpProxyUser("876216");
cb.setHttpProxyPassword("Apple@123");
twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
final Map<String, String> headers = new HashMap<String, String>();
/** Twitter listener * */
StatusListener listener =
new StatusListener() {
// The onStatus method is executed every time a new tweet comes
// in.
public void onStatus(Status status) {
// The EventBuilder is used to build an event using the
// the raw JSON of a tweet
System.out.println("Listening :");
KeyedMessage<String, String> data =
new KeyedMessage<String, String>(
"testing1", TwitterObjectFactory.getRawJSON(status));
producer.send(data);
System.out.println(data);
}
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}
public void onScrubGeo(long userId, long upToStatusId) {}
public void onException(Exception ex) {
logger.info("ShutDown");
twitterStream.shutdown();
}
public void onStallWarning(StallWarning warning) {}
};
twitterStream.addListener(listener);
/** GOGOGO * */
twitterStream.sample();
FilterQuery query =
new FilterQuery()
.track(
Tweety.hashtags[0],
Tweety.hashtags[1],
Tweety.hashtags[2],
Tweety.hashtags[3],
Tweety.hashtags[4]);
twitterStream.filter(query);
/** Bind the listener * */
}
/*
* Main entry of this application.
*
* @param args
*/
public static void main(String[] args) throws TwitterException {
TwitterStream twitterStream = new TwitterStreamFactory().getInstance();
final DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
final String startDate = formatter.format(new Timestamp((new Date()).getTime()));
final TweetStats stats = new TweetStats();
final long startTime = System.currentTimeMillis();
// to split the tweet text into words
final Pattern splitPattern = Pattern.compile("\\b");
final Pattern nameFilter = Pattern.compile("^[a-zA-Z0-9_]+$");
// System.out.println("digraph mentions {");
// I did not find a different way to count words:
// hashtables in java allows to manage sets of data with a <key,value> structure
// in this part of the worl, the key is the word and the value is the count.
final Hashtable<String, Integer> wordstats = new Hashtable<String, Integer>();
final DecimalFormat df = new DecimalFormat("#.###");
StatusListener listener =
new StatusListener() {
public void onStatus(Status status) {
long actualTime = System.currentTimeMillis();
String text = status.getText().toLowerCase();
String wordsPerText[] = splitPattern.split(text);
// to avoid problems with the counting process, this work use synchronized
synchronized (this) {
stats.incrementTweetCount();
long tweetCount = stats.getTweetCount();
long timeElapsed = (actualTime - startTime) / 1000;
long statET = stats.getElapsedTime();
long mentionCount;
// with this code, I show the stats every 15 seconds, comparing the change of time
List<String> names;
Extractor extractor = new Extractor();
names = extractor.extractMentionedScreennames(status.getText());
mentionCount = stats.getMentionCount();
String userName = status.getUser().getName().toLowerCase();
// I had some problems with international names and the graphic tools, so i decided
// To filer all the names with non ascii characters.
Matcher matcherUserName = nameFilter.matcher(userName);
if (matcherUserName.find() && names.size() > 0) {
// I'm countiong the act of mention one or more tweeters as one action.
stats.incrementMentionCount();
for (String name : names) {
String mentionName = name.toLowerCase();
Matcher matcherMention = nameFilter.matcher(mentionName);
if (matcherMention.find()) {
// userName = userName.replaceAll("\\u202E|\\u200E|\\t|\\r|\\n", "").trim();
// mentionName = mentionName.replaceAll("\\u202E|\\u200E|\\t|\\r|\\n",
// "").trim();
// Are there other cleanings on names???
System.out.println("\"" + userName + "\",\"" + mentionName + "\"");
}
}
}
if (mentionCount > 30000
&& timeElapsed > 0
&& statET != timeElapsed
&& timeElapsed % 15 == 0) {
// System.out.println("}");
stats.setElapsedTime(timeElapsed);
String report = "";
report += " |Date: " + startDate;
report += " |Tweet Count: " + tweetCount;
report += " |Mention Cpunt: " + mentionCount;
report += " |Elapsed Time (s): " + timeElapsed;
report += " |Avg. (Tweets per Second): " + (double) (tweetCount / timeElapsed);
System.out.println(report);
System.exit(0);
}
}
}
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
// System.out.println("Got a status deletion notice id:" +
// statusDeletionNotice.getStatusId());
}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
// System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
}
public void onScrubGeo(long userId, long upToStatusId) {
// System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" +
// upToStatusId);
}
public void onException(Exception ex) {
ex.printStackTrace();
}
};
twitterStream.addListener(listener);
twitterStream.sample();
}
public static void main(String[] args)
throws IOException, TwitterException, FileNotFoundException {
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setDebugEnabled(true)
.setOAuthConsumerKey("vGsMAca82HjVYYm7wQlN5Q")
.setOAuthConsumerSecret("6mpRAAlMI6sIWruQNpomBzN1kIfodHexYBrKWPKmsuM")
.setOAuthAccessToken("776768857-hCnK0Eu3orKK6qP98W7LyTF29QXadxydL21gIuon")
.setOAuthAccessTokenSecret("V3VAJgO2x1zWO2JMHsRRrWLzYOeXxmlittF1pBBNWA4");
TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
System.out.println("time " + System.currentTimeMillis());
StatusListener listener =
new StatusListener() {
int counter = 0;
FileWriter fstream = new FileWriter("/Users/jacobportnoff/Desktop/week7SunJ.txt");
BufferedWriter out = new BufferedWriter(fstream);
long start = System.currentTimeMillis();
double elapsedTimeSec = 0.0;
public void onStatus(Status status) {
HashtagEntity[] hashtags = status.getHashtagEntities();
String hashes = "";
if (hashtags.length == 0) {
hashes = "null";
} else {
for (HashtagEntity hash : hashtags) {
hashes = hashes + hash.getText() + " ";
}
}
String tweet =
status.getCreatedAt()
+ "\t"
+ status.getUser().getScreenName()
+ "\t"
+ status.getText()
+ "\t"
+ status.getGeoLocation()
+ "\t"
+ hashes
+ "\n";
// System.out.print(tweet);
try {
out.write(tweet);
} catch (Exception e) { // Catch exception if any
System.err.println("Error: " + e.getMessage());
}
}
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}
public void onScrubGeo(long userId, long upToStatusId) {}
public void onException(Exception ex) {
ex.printStackTrace();
}
};
twitterStream.addListener(listener);
twitterStream.sample();
}
private void start(final Context context) {
/** Producer properties * */
Properties props = new Properties();
props.put("metadata.broker.list", context.getString(BROKER_LIST));
props.put("serializer.class", context.getString(SERIALIZER));
props.put("request.required.acks", context.getString(REQUIRED_ACKS));
ProducerConfig config = new ProducerConfig(props);
final Producer<String, String> producer = new Producer<String, String>(config);
/** Twitter properties * */
consumerKey = context.getString(CONSUMER_KEY_KEY);
consumerSecret = context.getString(CONSUMER_SECRET_KEY);
accessToken = context.getString(ACCESS_TOKEN_KEY);
accessTokenSecret = context.getString(ACCESS_TOKEN_SECRET_KEY);
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setOAuthConsumerKey(consumerKey);
cb.setOAuthConsumerSecret(consumerSecret);
cb.setOAuthAccessToken(accessToken);
cb.setOAuthAccessTokenSecret(accessTokenSecret);
cb.setJSONStoreEnabled(true);
cb.setIncludeEntitiesEnabled(true);
twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
final boolean shouldPrintTweetsOnScreen =
Boolean.parseBoolean(context.getString(printTweetsOnScreen));
final boolean shouldSendTweetsToKafka =
Boolean.parseBoolean(context.getString(sendTweetsToKafka));
final StatusListener listener =
new StatusListener() {
// The onStatus method is executed every time a new tweet comes
// in.
public void onStatus(Status status) {
// The EventBuilder is used to build an event using the
// the raw JSON of a tweet
if (shouldPrintTweetsOnScreen) {
logger.info(status.getUser().getScreenName() + ": " + status.getText());
}
if (shouldSendTweetsToKafka) {
KeyedMessage<String, String> data =
new KeyedMessage<String, String>(
context.getString(KAFKA_TOPIC), TwitterObjectFactory.getRawJSON(status));
producer.send(data);
}
}
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}
public void onScrubGeo(long userId, long upToStatusId) {}
public void onException(Exception ex) {
logger.info("Shutting down Twitter sample stream...");
twitterStream.shutdown();
}
public void onStallWarning(StallWarning warning) {}
};
twitterStream.addListener(listener);
twitterStream.filter(new FilterQuery().track(context.getString(keywords).split(",")));
}
public static void main(String args[]) {
FilterQuery tweetFilterQuery = new FilterQuery();
String a[] = {"en"};
tweetFilterQuery.language(a);
ConfigurationBuilder _configurationBuilder = new ConfigurationBuilder();
_configurationBuilder
.setOAuthConsumerKey(consumerKey)
.setOAuthConsumerSecret(consumerSecret)
.setOAuthAccessToken(accessToken)
.setOAuthAccessTokenSecret(accessTokenSecret)
.setJSONStoreEnabled(true);
twitterStream = new TwitterStreamFactory(_configurationBuilder.build()).getInstance();
StatusListener listener =
new StatusListener() {
@Override
public void onException(Exception arg0) {
System.out.println("Error occured: " + arg0.getMessage());
arg0.printStackTrace();
}
@Override
public void onTrackLimitationNotice(int arg0) {
System.out.println("Track limitation notice for " + arg0);
}
@Override
public void onStatus(Status status) {
// if(status.getLang().equals("en"))
// System.out.println(status.getText().replaceAll("[\n\r]", ""));
String filename = totalCount / 100 + ".txt";
File f = new File(filename);
PrintWriter out = null;
if (f.exists() && !f.isDirectory()) {
// System.out.println("@" + status.getUser().getScreenName() + " - " +
// status.getText() + " -> "+ status.getCreatedAt());
// String mapstring = status.getUser().getScreenName() + " - " + status.getText() + "
// -> "+ status.getCreatedAt() +"\n";
String mapstring = status.getText().replace("\n", " ") + "\n";
try {
out = new PrintWriter(new FileOutputStream(new File(filename), true));
} catch (FileNotFoundException e) {
e.printStackTrace();
}
out.append(mapstring);
out.close();
} else {
try {
out = new PrintWriter(filename);
} catch (FileNotFoundException e) {
e.printStackTrace();
}
String mapstring = status.getText().replace("\n", " ") + "\n";
out.println(mapstring);
out.close();
}
totalCount++;
}
@Override
public void onStallWarning(StallWarning arg0) {
// TODO Auto-generated method stub
}
@Override
public void onScrubGeo(long arg0, long arg1) {
// TODO Auto-generated method stub
}
@Override
public void onDeletionNotice(StatusDeletionNotice arg0) {
// TODO Auto-generated method stub
}
};
twitterStream.addListener(listener);
twitterStream.sample();
}
