private void pullTweets(Key key, Twitter twitter) {
try {
RateLimitStatus rateLimitStatus = twitter.getRateLimitStatus();
if (rateLimitStatus.getRemainingHits() == 0) return;
double currentRate =
(rateLimitStatus.getHourlyLimit() - rateLimitStatus.getRemainingHits())
/ (3601 - rateLimitStatus.getSecondsUntilReset());
logger.info(
"Current twitter refresh rate: {}/h, official refresh rate: {}",
String.format("%1.1f", currentRate),
rateLimitStatus.getHourlyLimit());
if (currentRate > rateLimitStatus.getHourlyLimit()) {
logger.info("Skipped refreshing Twitter feeds to limit the refresh rate");
return;
}
ResponseList<Status> newFriendsTimeline = twitter.getHomeTimeline();
ResponseList<Status> newMentions = twitter.getMentions();
synchronized (this) {
mergeStatuses(timeline.get(key), newFriendsTimeline);
mergeStatuses(mentions.get(key), newMentions);
}
} catch (TwitterException e) {
logger.warn("Twitter reported an error: " + e.getMessage(), e);
}
}
public Map<String, RateLimitStatus> getRateLimit() {
try {
return twitter.getRateLimitStatus();
} catch (TwitterException e) {
e.printStackTrace();
return null;
}
}
public static void main(String[] args) {
try {
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setDebugEnabled(true).setOAuthConsumerKey("fSjrQedRpuRcUv3AmzUqkicCR");
cb.setDebugEnabled(true)
.setOAuthConsumerSecret("nGY8pyGD1CsYXaBb8SpeqoHHWYHE8pWeNmiR8Kc7UfWK5bJvTe");
cb.setDebugEnabled(true)
.setOAuthAccessToken("98946486-yZqWgilcljm4oVkT82euEDTl0UNna7t8R3F9rGEKZ");
cb.setDebugEnabled(true)
.setOAuthAccessTokenSecret("4GKOaU5OAksbNRz5rJy8uGxDM7vtzD3mJBCVJ2uU4qyNa");
TwitterFactory tf = new TwitterFactory(cb.build());
Twitter twitter = tf.getInstance();
ArrayList<TwitterUser> users = new ArrayList<>();
RateLimitStatus followers = twitter.getRateLimitStatus("followers").get("/followers/ids");
System.out.println("followers " + followers.getRemaining());
RateLimitStatus following = twitter.getRateLimitStatus("friends").get("/friends/ids");
System.out.println("following " + following.getRemaining());
if (followers.getRemaining() < 2 && following.getRemaining() < 2) {
System.out.println("Durmiendo por aqui...");
Thread.sleep(900000);
}
TwitterUser me = new TwitterUser(twitter);
me.setUserID(twitter.getId());
me.setFollowers(); // 1 follower
me.setFollowings(); // 1 following
users.add(me);
for (int i = 0; i < 10; i++) { // 20 requests
if (followers.getRemaining() < 2 && following.getRemaining() < 2) {
System.out.println("Durmiendo por aqui... I=" + i);
Thread.sleep(900000);
}
TwitterUser userX = new TwitterUser(me.getFollowers()[i], twitter);
userX.setFollowers();
userX.setFollowings();
users.add(userX);
for (int j = 0; j < 10; j++) {
if (followers.getRemaining() < 2 && following.getRemaining() < 2) {
System.out.println("Durmiendo por aqui... I=" + i + "J=" + j);
Thread.sleep(900000);
}
TwitterUser userX_1 = new TwitterUser(userX.getFollowers()[j], twitter);
userX_1.setFollowers();
userX_1.setFollowings();
users.add(userX_1);
}
}
for (int i = 0; i < 10; i++) { // 20 requests
if (followers.getRemaining() < 2 && following.getRemaining() < 2) {
System.out.println("Durmiendo por aqui... I=" + i);
Thread.sleep(900000);
}
TwitterUser userX = new TwitterUser(me.getFollowings()[i], twitter);
userX.setFollowers();
userX.setFollowings();
users.add(userX);
for (int j = 0; j < 10; j++) {
if (followers.getRemaining() < 2 && following.getRemaining() < 2) {
System.out.println("Durmiendo por aqui... I=" + i + "J=" + j);
Thread.sleep(900000);
}
TwitterUser userX_1 = new TwitterUser(userX.getFollowings()[j], twitter);
userX_1.setFollowers();
userX_1.setFollowings();
users.add(userX_1);
}
}
String content = "";
for (TwitterUser user : users) {
content += user.toString();
}
File file = new File("/users/cbanegas/Desktop/filename.txt");
if (!file.exists()) {
file.createNewFile();
}
FileWriter fw = new FileWriter(file.getAbsoluteFile());
BufferedWriter bw = new BufferedWriter(fw);
bw.write(content);
bw.close();
System.out.println("Done");
} catch (TwitterException te) {
Logger.getLogger(TwitterUser.class.getName()).log(Level.SEVERE, null, te);
System.out.println("twitter basura");
} catch (IOException ex) {
Logger.getLogger(TwitterGraph.class.getName()).log(Level.SEVERE, null, ex);
} catch (InterruptedException ex) {
Logger.getLogger(TwitterGraph.class.getName()).log(Level.SEVERE, null, ex);
}
}
private static ArrayList<String> ProcessTimeLine(String user)
throws InterruptedException, TwitterException {
ArrayList<String> Tweets = new ArrayList<String>();
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setDebugEnabled(true)
.setOAuthConsumerKey(KEY)
.setOAuthConsumerSecret(SECRET)
.setOAuthAccessToken(ACCESSTOKEN)
.setOAuthAccessTokenSecret(ACCESSSECRET);
cb.setJSONStoreEnabled(true);
// gets Twitter instance with default credentials
boolean bWait = true;
Twitter twitter = new TwitterFactory(cb.build()).getInstance();
do {
try {
Map<String, RateLimitStatus> oRT = twitter.getRateLimitStatus();
RateLimitStatus rateLimit = oRT.get("/statuses/user_timeline");
int remaining = rateLimit.getRemaining();
System.out.print("(Remaining API calls: " + remaining + ")");
int remainingTime = rateLimit.getSecondsUntilReset();
if (remaining <= NUM_TWEETS / 200 + 1) {
System.out.println("Waiting " + remainingTime + " seconds");
Thread.sleep(remainingTime * 1000);
} else bWait = false;
} catch (Exception te) {
if (te.toString().toLowerCase().contains("rate limit")
&& !te.toString().toLowerCase().contains("bad authentication data")) {
System.out.println("Waiting 60s");
Thread.sleep(60 * 1000);
} else {
bWait = false;
}
}
} while (bWait);
try {
Detector detector = DetectorFactory.create();
List<Status> statuses;
int iPage = 1;
int iTweets = 0;
do {
int iPageSize = 0;
if (iTweets + 200 < NUM_TWEETS) {
iPageSize = 200;
} else {
iPageSize = NUM_TWEETS - iTweets;
}
statuses = twitter.getUserTimeline(user, new Paging(iPage, iPageSize));
for (Status status : statuses) {
String sStatusId = "-1";
try {
if ((status.getRetweetedStatus() != null)
&& (status.getRetweetedStatus().getUser() != null)) {
continue;
}
try {
detector.append(Simplify(status.getText()));
if (detector.detect().equalsIgnoreCase("es")) {
String sStatusJSON = DataObjectFactory.getRawJSON(status);
Tweets.add(sStatusJSON);
}
} catch (Exception exl) {
}
} catch (Exception ex) {
System.out.println("ERROR in status id " + sStatusId);
}
iTweets++;
}
iPage++;
} while (statuses.size() > 0 && iTweets < NUM_TWEETS);
} catch (TwitterException te) {
te.printStackTrace();
System.out.println("Failed to get timeline: " + te.getMessage());
} catch (Exception ex) {
}
System.out.println("..." + Tweets.size() + " tweets.");
return Tweets;
}
