public static void main(String[] args) throws Exception {
Context context = new Context("conf/producer.conf");
Tweety tp = new Tweety();
try {
Class.forName("com.mysql.jdbc.Driver");
String url = "jdbc:mysql://172.20.95.137/mongo_test?" + "user=manoj&password=qwerty1!";
Connection conn = DriverManager.getConnection(url);
System.out.println(url);
Statement stmt = conn.createStatement();
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setOAuthConsumerKey("ujS6V0corQ4v8K4dp6bIlVYsq");
cb.setOAuthConsumerSecret("XhmTLcS4BI6zy9fZ0xIOy1ZZkAnQYZ1jFrlHojWuWpx2RCWH9H");
cb.setOAuthAccessToken("30427118-YNLB3WfviD4j7MvNgU4gZ7SPQbCDBXWh4kPZcHXrY");
cb.setOAuthAccessTokenSecret("LkhcZUJbj33RBc1DTXrxJps4ELgog8WxoRS6gCReemOHc");
cb.setJSONStoreEnabled(true);
cb.setIncludeEntitiesEnabled(true);
cb.setHttpProxyHost("proxy.tcs.com");
cb.setHttpProxyPort(8080);
cb.setHttpProxyUser("467098");
cb.setHttpProxyPassword("See@0615");
Twitter twitter = new TwitterFactory(cb.build()).getInstance();
ResponseList<Location> locations;
locations = twitter.getAvailableTrends();
System.out.println("Showing available trends");
Trends trends = twitter.getPlaceTrends(2295386);
stmt.executeUpdate("drop table hashtag");
stmt.executeUpdate("create table hashtag(name varchar(50),count int)");
for (int i = 0; i <= 5; i++) {
System.out.println(trends.getTrends()[i].getName());
hashtags[i] = trends.getTrends()[i].getName();
stmt.executeUpdate("insert into hashtag values (\'" + hashtags[i] + "\'" + "," + 0 + ")");
}
System.out.println("done.");
} catch (Exception te) {
te.printStackTrace();
System.out.println("Failed to get trends: " + te.getMessage());
System.exit(-1);
}
tp.start(context);
}
/**
* The initialization method for the Source. The context contains all the Flume configuration
* info, and can be used to retrieve any configuration values necessary to set up the Source.
*/
@Override
public void configure(Context context) {
consumerKey = context.getString(TwitterSourceConstants.CONSUMER_KEY_KEY);
consumerSecret = context.getString(TwitterSourceConstants.CONSUMER_SECRET_KEY);
accessToken = context.getString(TwitterSourceConstants.ACCESS_TOKEN_KEY);
accessTokenSecret = context.getString(TwitterSourceConstants.ACCESS_TOKEN_SECRET_KEY);
String keywordString = context.getString(TwitterSourceConstants.KEYWORDS_KEY, "");
keywords = keywordString.split(",");
for (int i = 0; i < keywords.length; i++) {
keywords[i] = keywords[i].trim();
}
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setOAuthConsumerKey(consumerKey);
cb.setOAuthConsumerSecret(consumerSecret);
cb.setOAuthAccessToken(accessToken);
cb.setOAuthAccessTokenSecret(accessTokenSecret);
cb.setJSONStoreEnabled(true);
cb.setIncludeEntitiesEnabled(true);
twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
}
private void start(final Context context) {
/** Producer properties * */
Properties props = new Properties();
props.put("metadata.broker.list", context.getString(BROKER_LIST));
props.put("serializer.class", context.getString(SERIALIZER));
props.put("request.required.acks", context.getString(REQUIRED_ACKS));
ProducerConfig config = new ProducerConfig(props);
final Producer<String, String> producer = new Producer<String, String>(config);
/** Twitter properties * */
consumerKey = context.getString(CONSUMER_KEY_KEY);
consumerSecret = context.getString(CONSUMER_SECRET_KEY);
accessToken = context.getString(ACCESS_TOKEN_KEY);
accessTokenSecret = context.getString(ACCESS_TOKEN_SECRET_KEY);
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setOAuthConsumerKey(consumerKey);
cb.setOAuthConsumerSecret(consumerSecret);
cb.setOAuthAccessToken(accessToken);
cb.setOAuthAccessTokenSecret(accessTokenSecret);
cb.setJSONStoreEnabled(true);
cb.setIncludeEntitiesEnabled(true);
twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
final boolean shouldPrintTweetsOnScreen =
Boolean.parseBoolean(context.getString(printTweetsOnScreen));
final boolean shouldSendTweetsToKafka =
Boolean.parseBoolean(context.getString(sendTweetsToKafka));
final StatusListener listener =
new StatusListener() {
// The onStatus method is executed every time a new tweet comes
// in.
public void onStatus(Status status) {
// The EventBuilder is used to build an event using the
// the raw JSON of a tweet
if (shouldPrintTweetsOnScreen) {
logger.info(status.getUser().getScreenName() + ": " + status.getText());
}
if (shouldSendTweetsToKafka) {
KeyedMessage<String, String> data =
new KeyedMessage<String, String>(
context.getString(KAFKA_TOPIC), TwitterObjectFactory.getRawJSON(status));
producer.send(data);
}
}
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}
public void onScrubGeo(long userId, long upToStatusId) {}
public void onException(Exception ex) {
logger.info("Shutting down Twitter sample stream...");
twitterStream.shutdown();
}
public void onStallWarning(StallWarning warning) {}
};
twitterStream.addListener(listener);
twitterStream.filter(new FilterQuery().track(context.getString(keywords).split(",")));
}
private void start(final Context context) throws IOException {
// Producer properties
Properties props = new Properties();
props.put("metadata.broker.list", context.getString(TwitterSourceConstant.BROKER_LIST));
props.put("serializer.class", context.getString(TwitterSourceConstant.SERIALIZER));
props.put("partitioner.class", context.getString(TwitterSourceConstant.PARTITIONER));
props.put("request.required.acks", context.getString(TwitterSourceConstant.REQUIRED_ACKS));
ProducerConfig config = new ProducerConfig(props);
final Producer<String, String> producer = new Producer<String, String>(config);
/** Twitter properties * */
consumerKey = context.getString(TwitterSourceConstant.CONSUMER_KEY_KEY);
consumerSecret = context.getString(TwitterSourceConstant.CONSUMER_SECRET_KEY);
accessToken = context.getString(TwitterSourceConstant.ACCESS_TOKEN_KEY);
accessTokenSecret = context.getString(TwitterSourceConstant.ACCESS_TOKEN_SECRET_KEY);
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setOAuthConsumerKey(consumerKey);
cb.setOAuthConsumerSecret(consumerSecret);
cb.setOAuthAccessToken(accessToken);
cb.setOAuthAccessTokenSecret(accessTokenSecret);
cb.setJSONStoreEnabled(true);
cb.setIncludeEntitiesEnabled(true);
cb.setHttpProxyHost("proxy.tcs.com");
cb.setHttpProxyPort(8080);
cb.setHttpProxyUser("876216");
cb.setHttpProxyPassword("Apple@123");
twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
final Map<String, String> headers = new HashMap<String, String>();
/** Twitter listener * */
StatusListener listener =
new StatusListener() {
// The onStatus method is executed every time a new tweet comes
// in.
public void onStatus(Status status) {
// The EventBuilder is used to build an event using the
// the raw JSON of a tweet
System.out.println("Listening :");
KeyedMessage<String, String> data =
new KeyedMessage<String, String>(
"testing1", TwitterObjectFactory.getRawJSON(status));
producer.send(data);
System.out.println(data);
}
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}
public void onScrubGeo(long userId, long upToStatusId) {}
public void onException(Exception ex) {
logger.info("ShutDown");
twitterStream.shutdown();
}
public void onStallWarning(StallWarning warning) {}
};
twitterStream.addListener(listener);
/** GOGOGO * */
twitterStream.sample();
FilterQuery query =
new FilterQuery()
.track(
Tweety.hashtags[0],
Tweety.hashtags[1],
Tweety.hashtags[2],
Tweety.hashtags[3],
Tweety.hashtags[4]);
twitterStream.filter(query);
/** Bind the listener * */
}
private static ArrayList<String> ProcessTimeLine(String user)
throws InterruptedException, TwitterException {
ArrayList<String> Tweets = new ArrayList<String>();
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setDebugEnabled(true)
.setOAuthConsumerKey(KEY)
.setOAuthConsumerSecret(SECRET)
.setOAuthAccessToken(ACCESSTOKEN)
.setOAuthAccessTokenSecret(ACCESSSECRET);
cb.setJSONStoreEnabled(true);
// gets Twitter instance with default credentials
boolean bWait = true;
Twitter twitter = new TwitterFactory(cb.build()).getInstance();
do {
try {
Map<String, RateLimitStatus> oRT = twitter.getRateLimitStatus();
RateLimitStatus rateLimit = oRT.get("/statuses/user_timeline");
int remaining = rateLimit.getRemaining();
System.out.print("(Remaining API calls: " + remaining + ")");
int remainingTime = rateLimit.getSecondsUntilReset();
if (remaining <= NUM_TWEETS / 200 + 1) {
System.out.println("Waiting " + remainingTime + " seconds");
Thread.sleep(remainingTime * 1000);
} else bWait = false;
} catch (Exception te) {
if (te.toString().toLowerCase().contains("rate limit")
&& !te.toString().toLowerCase().contains("bad authentication data")) {
System.out.println("Waiting 60s");
Thread.sleep(60 * 1000);
} else {
bWait = false;
}
}
} while (bWait);
try {
Detector detector = DetectorFactory.create();
List<Status> statuses;
int iPage = 1;
int iTweets = 0;
do {
int iPageSize = 0;
if (iTweets + 200 < NUM_TWEETS) {
iPageSize = 200;
} else {
iPageSize = NUM_TWEETS - iTweets;
}
statuses = twitter.getUserTimeline(user, new Paging(iPage, iPageSize));
for (Status status : statuses) {
String sStatusId = "-1";
try {
if ((status.getRetweetedStatus() != null)
&& (status.getRetweetedStatus().getUser() != null)) {
continue;
}
try {
detector.append(Simplify(status.getText()));
if (detector.detect().equalsIgnoreCase("es")) {
String sStatusJSON = DataObjectFactory.getRawJSON(status);
Tweets.add(sStatusJSON);
}
} catch (Exception exl) {
}
} catch (Exception ex) {
System.out.println("ERROR in status id " + sStatusId);
}
iTweets++;
}
iPage++;
} while (statuses.size() > 0 && iTweets < NUM_TWEETS);
} catch (TwitterException te) {
te.printStackTrace();
System.out.println("Failed to get timeline: " + te.getMessage());
} catch (Exception ex) {
}
System.out.println("..." + Tweets.size() + " tweets.");
return Tweets;
}
