public static void main(String[] args) throws Exception {
Context context = new Context("conf/producer.conf");
Tweety tp = new Tweety();
try {
Class.forName("com.mysql.jdbc.Driver");
String url = "jdbc:mysql://172.20.95.137/mongo_test?" + "user=manoj&password=qwerty1!";
Connection conn = DriverManager.getConnection(url);
System.out.println(url);
Statement stmt = conn.createStatement();
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setOAuthConsumerKey("ujS6V0corQ4v8K4dp6bIlVYsq");
cb.setOAuthConsumerSecret("XhmTLcS4BI6zy9fZ0xIOy1ZZkAnQYZ1jFrlHojWuWpx2RCWH9H");
cb.setOAuthAccessToken("30427118-YNLB3WfviD4j7MvNgU4gZ7SPQbCDBXWh4kPZcHXrY");
cb.setOAuthAccessTokenSecret("LkhcZUJbj33RBc1DTXrxJps4ELgog8WxoRS6gCReemOHc");
cb.setJSONStoreEnabled(true);
cb.setIncludeEntitiesEnabled(true);
cb.setHttpProxyHost("proxy.tcs.com");
cb.setHttpProxyPort(8080);
cb.setHttpProxyUser("467098");
cb.setHttpProxyPassword("See@0615");
Twitter twitter = new TwitterFactory(cb.build()).getInstance();
ResponseList<Location> locations;
locations = twitter.getAvailableTrends();
System.out.println("Showing available trends");
Trends trends = twitter.getPlaceTrends(2295386);
stmt.executeUpdate("drop table hashtag");
stmt.executeUpdate("create table hashtag(name varchar(50),count int)");
for (int i = 0; i <= 5; i++) {
System.out.println(trends.getTrends()[i].getName());
hashtags[i] = trends.getTrends()[i].getName();
stmt.executeUpdate("insert into hashtag values (\'" + hashtags[i] + "\'" + "," + 0 + ")");
}
System.out.println("done.");
} catch (Exception te) {
te.printStackTrace();
System.out.println("Failed to get trends: " + te.getMessage());
System.exit(-1);
}
tp.start(context);
}
/**
* The initialization method for the Source. The context contains all the Flume configuration
* info, and can be used to retrieve any configuration values necessary to set up the Source.
*/
@Override
public void configure(Context context) {
consumerKey = context.getString(TwitterSourceConstants.CONSUMER_KEY_KEY);
consumerSecret = context.getString(TwitterSourceConstants.CONSUMER_SECRET_KEY);
accessToken = context.getString(TwitterSourceConstants.ACCESS_TOKEN_KEY);
accessTokenSecret = context.getString(TwitterSourceConstants.ACCESS_TOKEN_SECRET_KEY);
String keywordString = context.getString(TwitterSourceConstants.KEYWORDS_KEY, "");
keywords = keywordString.split(",");
for (int i = 0; i < keywords.length; i++) {
keywords[i] = keywords[i].trim();
}
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setOAuthConsumerKey(consumerKey);
cb.setOAuthConsumerSecret(consumerSecret);
cb.setOAuthAccessToken(accessToken);
cb.setOAuthAccessTokenSecret(accessTokenSecret);
cb.setJSONStoreEnabled(true);
cb.setIncludeEntitiesEnabled(true);
twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
}
private void start(final Context context) {
/** Producer properties * */
Properties props = new Properties();
props.put("metadata.broker.list", context.getString(BROKER_LIST));
props.put("serializer.class", context.getString(SERIALIZER));
props.put("request.required.acks", context.getString(REQUIRED_ACKS));
ProducerConfig config = new ProducerConfig(props);
final Producer<String, String> producer = new Producer<String, String>(config);
/** Twitter properties * */
consumerKey = context.getString(CONSUMER_KEY_KEY);
consumerSecret = context.getString(CONSUMER_SECRET_KEY);
accessToken = context.getString(ACCESS_TOKEN_KEY);
accessTokenSecret = context.getString(ACCESS_TOKEN_SECRET_KEY);
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setOAuthConsumerKey(consumerKey);
cb.setOAuthConsumerSecret(consumerSecret);
cb.setOAuthAccessToken(accessToken);
cb.setOAuthAccessTokenSecret(accessTokenSecret);
cb.setJSONStoreEnabled(true);
cb.setIncludeEntitiesEnabled(true);
twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
final boolean shouldPrintTweetsOnScreen =
Boolean.parseBoolean(context.getString(printTweetsOnScreen));
final boolean shouldSendTweetsToKafka =
Boolean.parseBoolean(context.getString(sendTweetsToKafka));
final StatusListener listener =
new StatusListener() {
// The onStatus method is executed every time a new tweet comes
// in.
public void onStatus(Status status) {
// The EventBuilder is used to build an event using the
// the raw JSON of a tweet
if (shouldPrintTweetsOnScreen) {
logger.info(status.getUser().getScreenName() + ": " + status.getText());
}
if (shouldSendTweetsToKafka) {
KeyedMessage<String, String> data =
new KeyedMessage<String, String>(
context.getString(KAFKA_TOPIC), TwitterObjectFactory.getRawJSON(status));
producer.send(data);
}
}
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}
public void onScrubGeo(long userId, long upToStatusId) {}
public void onException(Exception ex) {
logger.info("Shutting down Twitter sample stream...");
twitterStream.shutdown();
}
public void onStallWarning(StallWarning warning) {}
};
twitterStream.addListener(listener);
twitterStream.filter(new FilterQuery().track(context.getString(keywords).split(",")));
}
private void start(final Context context) throws IOException {
// Producer properties
Properties props = new Properties();
props.put("metadata.broker.list", context.getString(TwitterSourceConstant.BROKER_LIST));
props.put("serializer.class", context.getString(TwitterSourceConstant.SERIALIZER));
props.put("partitioner.class", context.getString(TwitterSourceConstant.PARTITIONER));
props.put("request.required.acks", context.getString(TwitterSourceConstant.REQUIRED_ACKS));
ProducerConfig config = new ProducerConfig(props);
final Producer<String, String> producer = new Producer<String, String>(config);
/** Twitter properties * */
consumerKey = context.getString(TwitterSourceConstant.CONSUMER_KEY_KEY);
consumerSecret = context.getString(TwitterSourceConstant.CONSUMER_SECRET_KEY);
accessToken = context.getString(TwitterSourceConstant.ACCESS_TOKEN_KEY);
accessTokenSecret = context.getString(TwitterSourceConstant.ACCESS_TOKEN_SECRET_KEY);
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setOAuthConsumerKey(consumerKey);
cb.setOAuthConsumerSecret(consumerSecret);
cb.setOAuthAccessToken(accessToken);
cb.setOAuthAccessTokenSecret(accessTokenSecret);
cb.setJSONStoreEnabled(true);
cb.setIncludeEntitiesEnabled(true);
cb.setHttpProxyHost("proxy.tcs.com");
cb.setHttpProxyPort(8080);
cb.setHttpProxyUser("876216");
cb.setHttpProxyPassword("Apple@123");
twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
final Map<String, String> headers = new HashMap<String, String>();
/** Twitter listener * */
StatusListener listener =
new StatusListener() {
// The onStatus method is executed every time a new tweet comes
// in.
public void onStatus(Status status) {
// The EventBuilder is used to build an event using the
// the raw JSON of a tweet
System.out.println("Listening :");
KeyedMessage<String, String> data =
new KeyedMessage<String, String>(
"testing1", TwitterObjectFactory.getRawJSON(status));
producer.send(data);
System.out.println(data);
}
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}
public void onScrubGeo(long userId, long upToStatusId) {}
public void onException(Exception ex) {
logger.info("ShutDown");
twitterStream.shutdown();
}
public void onStallWarning(StallWarning warning) {}
};
twitterStream.addListener(listener);
/** GOGOGO * */
twitterStream.sample();
FilterQuery query =
new FilterQuery()
.track(
Tweety.hashtags[0],
Tweety.hashtags[1],
Tweety.hashtags[2],
Tweety.hashtags[3],
Tweety.hashtags[4]);
twitterStream.filter(query);
/** Bind the listener * */
}