private void pullTweets(Key key, Twitter twitter) { try { RateLimitStatus rateLimitStatus = twitter.getRateLimitStatus(); if (rateLimitStatus.getRemainingHits() == 0) return; double currentRate = (rateLimitStatus.getHourlyLimit() - rateLimitStatus.getRemainingHits()) / (3601 - rateLimitStatus.getSecondsUntilReset()); logger.info( "Current twitter refresh rate: {}/h, official refresh rate: {}", String.format("%1.1f", currentRate), rateLimitStatus.getHourlyLimit()); if (currentRate > rateLimitStatus.getHourlyLimit()) { logger.info("Skipped refreshing Twitter feeds to limit the refresh rate"); return; } ResponseList<Status> newFriendsTimeline = twitter.getHomeTimeline(); ResponseList<Status> newMentions = twitter.getMentions(); synchronized (this) { mergeStatuses(timeline.get(key), newFriendsTimeline); mergeStatuses(mentions.get(key), newMentions); } } catch (TwitterException e) { logger.warn("Twitter reported an error: " + e.getMessage(), e); } }
public Map<String, RateLimitStatus> getRateLimit() { try { return twitter.getRateLimitStatus(); } catch (TwitterException e) { e.printStackTrace(); return null; } }
public static void main(String[] args) { try { ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey("fSjrQedRpuRcUv3AmzUqkicCR"); cb.setDebugEnabled(true) .setOAuthConsumerSecret("nGY8pyGD1CsYXaBb8SpeqoHHWYHE8pWeNmiR8Kc7UfWK5bJvTe"); cb.setDebugEnabled(true) .setOAuthAccessToken("98946486-yZqWgilcljm4oVkT82euEDTl0UNna7t8R3F9rGEKZ"); cb.setDebugEnabled(true) .setOAuthAccessTokenSecret("4GKOaU5OAksbNRz5rJy8uGxDM7vtzD3mJBCVJ2uU4qyNa"); TwitterFactory tf = new TwitterFactory(cb.build()); Twitter twitter = tf.getInstance(); ArrayList<TwitterUser> users = new ArrayList<>(); RateLimitStatus followers = twitter.getRateLimitStatus("followers").get("/followers/ids"); System.out.println("followers " + followers.getRemaining()); RateLimitStatus following = twitter.getRateLimitStatus("friends").get("/friends/ids"); System.out.println("following " + following.getRemaining()); if (followers.getRemaining() < 2 && following.getRemaining() < 2) { System.out.println("Durmiendo por aqui..."); Thread.sleep(900000); } TwitterUser me = new TwitterUser(twitter); me.setUserID(twitter.getId()); me.setFollowers(); // 1 follower me.setFollowings(); // 1 following users.add(me); for (int i = 0; i < 10; i++) { // 20 requests if (followers.getRemaining() < 2 && following.getRemaining() < 2) { System.out.println("Durmiendo por aqui... I=" + i); Thread.sleep(900000); } TwitterUser userX = new TwitterUser(me.getFollowers()[i], twitter); userX.setFollowers(); userX.setFollowings(); users.add(userX); for (int j = 0; j < 10; j++) { if (followers.getRemaining() < 2 && following.getRemaining() < 2) { System.out.println("Durmiendo por aqui... I=" + i + "J=" + j); Thread.sleep(900000); } TwitterUser userX_1 = new TwitterUser(userX.getFollowers()[j], twitter); userX_1.setFollowers(); userX_1.setFollowings(); users.add(userX_1); } } for (int i = 0; i < 10; i++) { // 20 requests if (followers.getRemaining() < 2 && following.getRemaining() < 2) { System.out.println("Durmiendo por aqui... I=" + i); Thread.sleep(900000); } TwitterUser userX = new TwitterUser(me.getFollowings()[i], twitter); userX.setFollowers(); userX.setFollowings(); users.add(userX); for (int j = 0; j < 10; j++) { if (followers.getRemaining() < 2 && following.getRemaining() < 2) { System.out.println("Durmiendo por aqui... I=" + i + "J=" + j); Thread.sleep(900000); } TwitterUser userX_1 = new TwitterUser(userX.getFollowings()[j], twitter); userX_1.setFollowers(); userX_1.setFollowings(); users.add(userX_1); } } String content = ""; for (TwitterUser user : users) { content += user.toString(); } File file = new File("/users/cbanegas/Desktop/filename.txt"); if (!file.exists()) { file.createNewFile(); } FileWriter fw = new FileWriter(file.getAbsoluteFile()); BufferedWriter bw = new BufferedWriter(fw); bw.write(content); bw.close(); System.out.println("Done"); } catch (TwitterException te) { Logger.getLogger(TwitterUser.class.getName()).log(Level.SEVERE, null, te); System.out.println("twitter basura"); } catch (IOException ex) { Logger.getLogger(TwitterGraph.class.getName()).log(Level.SEVERE, null, ex); } catch (InterruptedException ex) { Logger.getLogger(TwitterGraph.class.getName()).log(Level.SEVERE, null, ex); } }
private static ArrayList<String> ProcessTimeLine(String user) throws InterruptedException, TwitterException { ArrayList<String> Tweets = new ArrayList<String>(); ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true) .setOAuthConsumerKey(KEY) .setOAuthConsumerSecret(SECRET) .setOAuthAccessToken(ACCESSTOKEN) .setOAuthAccessTokenSecret(ACCESSSECRET); cb.setJSONStoreEnabled(true); // gets Twitter instance with default credentials boolean bWait = true; Twitter twitter = new TwitterFactory(cb.build()).getInstance(); do { try { Map<String, RateLimitStatus> oRT = twitter.getRateLimitStatus(); RateLimitStatus rateLimit = oRT.get("/statuses/user_timeline"); int remaining = rateLimit.getRemaining(); System.out.print("(Remaining API calls: " + remaining + ")"); int remainingTime = rateLimit.getSecondsUntilReset(); if (remaining <= NUM_TWEETS / 200 + 1) { System.out.println("Waiting " + remainingTime + " seconds"); Thread.sleep(remainingTime * 1000); } else bWait = false; } catch (Exception te) { if (te.toString().toLowerCase().contains("rate limit") && !te.toString().toLowerCase().contains("bad authentication data")) { System.out.println("Waiting 60s"); Thread.sleep(60 * 1000); } else { bWait = false; } } } while (bWait); try { Detector detector = DetectorFactory.create(); List<Status> statuses; int iPage = 1; int iTweets = 0; do { int iPageSize = 0; if (iTweets + 200 < NUM_TWEETS) { iPageSize = 200; } else { iPageSize = NUM_TWEETS - iTweets; } statuses = twitter.getUserTimeline(user, new Paging(iPage, iPageSize)); for (Status status : statuses) { String sStatusId = "-1"; try { if ((status.getRetweetedStatus() != null) && (status.getRetweetedStatus().getUser() != null)) { continue; } try { detector.append(Simplify(status.getText())); if (detector.detect().equalsIgnoreCase("es")) { String sStatusJSON = DataObjectFactory.getRawJSON(status); Tweets.add(sStatusJSON); } } catch (Exception exl) { } } catch (Exception ex) { System.out.println("ERROR in status id " + sStatusId); } iTweets++; } iPage++; } while (statuses.size() > 0 && iTweets < NUM_TWEETS); } catch (TwitterException te) { te.printStackTrace(); System.out.println("Failed to get timeline: " + te.getMessage()); } catch (Exception ex) { } System.out.println("..." + Tweets.size() + " tweets."); return Tweets; }