private void pullTweets(Key key, Twitter twitter) {
try {
RateLimitStatus rateLimitStatus = twitter.getRateLimitStatus();
if (rateLimitStatus.getRemainingHits() == 0) return;
double currentRate =
(rateLimitStatus.getHourlyLimit() - rateLimitStatus.getRemainingHits())
/ (3601 - rateLimitStatus.getSecondsUntilReset());
logger.info(
"Current twitter refresh rate: {}/h, official refresh rate: {}",
String.format("%1.1f", currentRate),
rateLimitStatus.getHourlyLimit());
if (currentRate > rateLimitStatus.getHourlyLimit()) {
logger.info("Skipped refreshing Twitter feeds to limit the refresh rate");
return;
}
ResponseList<Status> newFriendsTimeline = twitter.getHomeTimeline();
ResponseList<Status> newMentions = twitter.getMentions();
synchronized (this) {
mergeStatuses(timeline.get(key), newFriendsTimeline);
mergeStatuses(mentions.get(key), newMentions);
}
} catch (TwitterException e) {
logger.warn("Twitter reported an error: " + e.getMessage(), e);
}
}
protected boolean handleReceivingRateLimitStatus(RateLimitStatus rateLimitStatus) {
try {
int secondsUntilReset = rateLimitStatus.getSecondsUntilReset();
int remainingHits = rateLimitStatus.getRemainingHits();
if (remainingHits == 0) {
logger.debug("rate status limit service returned 0 for the remaining hits value");
return false;
}
if (secondsUntilReset == 0) {
logger.debug(
"rate status limit service returned 0 for the seconds until reset period value");
return false;
}
int secondsUntilWeCanPullAgain = secondsUntilReset / remainingHits;
long msUntilWeCanPullAgain = secondsUntilWeCanPullAgain * 1000;
logger.debug(
"need to Thread.sleep() "
+ secondsUntilWeCanPullAgain
+ " seconds until the next timeline pull. Have "
+ remainingHits
+ " remaining pull this rate period. The period ends in "
+ secondsUntilReset);
Thread.sleep(msUntilWeCanPullAgain);
} catch (Throwable throwable) {
logger.debug(
"encountered an error when"
+ " trying to refresh the timeline: "
+ ExceptionUtils.getFullStackTrace(throwable));
}
return true;
}
public void reloadHomeTL(long Id) throws TwitterException {
User user = null;
Image icon;
if (page == null) {
page = new Paging(1, 20);
} else {
page = new Paging(Id);
}
homeTL = twitter.getHomeTimeline(page);
userInfoList.clear();
if (homeTL.size() == 0) {
System.out.println("---> 新着無し");
} else {
for (Status status : homeTL) {
user = status.getUser();
if (iconMap.get(user.getScreenName()) == null) {
iconMap.put(user.getScreenName(), new Image(user.getBiggerProfileImageURL()));
}
icon = iconMap.get(user.getScreenName());
userInfoList.add(0, new UserInfo(icon, status));
}
}
if (homeTL.size() != 0) {
for (UserInfo userInfo : userInfoList) {
observableList.add(0, userInfo);
}
lastStatusId = homeTL.get(0).getId();
listView.setItems(observableList);
}
// TwitterAPI制限の確認(コンソール)
rateLimit = homeTL.getRateLimitStatus();
System.out.println("残りアクセス可能回数 : " + rateLimit.getRemaining() + "回");
System.out.println("再アクセス可能まで : " + rateLimit.getSecondsUntilReset() + "秒");
if (rateLimit.getRemaining() == 0) {
System.out.println("API制限だー");
System.exit(0);
}
}
public static void main(String[] args) {
try {
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setDebugEnabled(true).setOAuthConsumerKey("fSjrQedRpuRcUv3AmzUqkicCR");
cb.setDebugEnabled(true)
.setOAuthConsumerSecret("nGY8pyGD1CsYXaBb8SpeqoHHWYHE8pWeNmiR8Kc7UfWK5bJvTe");
cb.setDebugEnabled(true)
.setOAuthAccessToken("98946486-yZqWgilcljm4oVkT82euEDTl0UNna7t8R3F9rGEKZ");
cb.setDebugEnabled(true)
.setOAuthAccessTokenSecret("4GKOaU5OAksbNRz5rJy8uGxDM7vtzD3mJBCVJ2uU4qyNa");
TwitterFactory tf = new TwitterFactory(cb.build());
Twitter twitter = tf.getInstance();
ArrayList<TwitterUser> users = new ArrayList<>();
RateLimitStatus followers = twitter.getRateLimitStatus("followers").get("/followers/ids");
System.out.println("followers " + followers.getRemaining());
RateLimitStatus following = twitter.getRateLimitStatus("friends").get("/friends/ids");
System.out.println("following " + following.getRemaining());
if (followers.getRemaining() < 2 && following.getRemaining() < 2) {
System.out.println("Durmiendo por aqui...");
Thread.sleep(900000);
}
TwitterUser me = new TwitterUser(twitter);
me.setUserID(twitter.getId());
me.setFollowers(); // 1 follower
me.setFollowings(); // 1 following
users.add(me);
for (int i = 0; i < 10; i++) { // 20 requests
if (followers.getRemaining() < 2 && following.getRemaining() < 2) {
System.out.println("Durmiendo por aqui... I=" + i);
Thread.sleep(900000);
}
TwitterUser userX = new TwitterUser(me.getFollowers()[i], twitter);
userX.setFollowers();
userX.setFollowings();
users.add(userX);
for (int j = 0; j < 10; j++) {
if (followers.getRemaining() < 2 && following.getRemaining() < 2) {
System.out.println("Durmiendo por aqui... I=" + i + "J=" + j);
Thread.sleep(900000);
}
TwitterUser userX_1 = new TwitterUser(userX.getFollowers()[j], twitter);
userX_1.setFollowers();
userX_1.setFollowings();
users.add(userX_1);
}
}
for (int i = 0; i < 10; i++) { // 20 requests
if (followers.getRemaining() < 2 && following.getRemaining() < 2) {
System.out.println("Durmiendo por aqui... I=" + i);
Thread.sleep(900000);
}
TwitterUser userX = new TwitterUser(me.getFollowings()[i], twitter);
userX.setFollowers();
userX.setFollowings();
users.add(userX);
for (int j = 0; j < 10; j++) {
if (followers.getRemaining() < 2 && following.getRemaining() < 2) {
System.out.println("Durmiendo por aqui... I=" + i + "J=" + j);
Thread.sleep(900000);
}
TwitterUser userX_1 = new TwitterUser(userX.getFollowings()[j], twitter);
userX_1.setFollowers();
userX_1.setFollowings();
users.add(userX_1);
}
}
String content = "";
for (TwitterUser user : users) {
content += user.toString();
}
File file = new File("/users/cbanegas/Desktop/filename.txt");
if (!file.exists()) {
file.createNewFile();
}
FileWriter fw = new FileWriter(file.getAbsoluteFile());
BufferedWriter bw = new BufferedWriter(fw);
bw.write(content);
bw.close();
System.out.println("Done");
} catch (TwitterException te) {
Logger.getLogger(TwitterUser.class.getName()).log(Level.SEVERE, null, te);
System.out.println("twitter basura");
} catch (IOException ex) {
Logger.getLogger(TwitterGraph.class.getName()).log(Level.SEVERE, null, ex);
} catch (InterruptedException ex) {
Logger.getLogger(TwitterGraph.class.getName()).log(Level.SEVERE, null, ex);
}
}
public static int handleTwitterError(Twitter twitter, Long id, Exception exception) {
if (exception instanceof TwitterException) {
TwitterException e = (TwitterException) exception;
if (e.exceededRateLimitation()) {
long millisUntilReset = retry;
final RateLimitStatus rateLimitStatus = e.getRateLimitStatus();
if (rateLimitStatus != null) {
millisUntilReset = rateLimitStatus.getSecondsUntilReset() * 1000;
}
LOGGER.warn("Rate Limit Exceeded. Will retry in {} seconds...", millisUntilReset / 1000);
try {
Thread.sleep(millisUntilReset);
} catch (InterruptedException e1) {
Thread.currentThread().interrupt();
}
return 1;
} else if (e.isCausedByNetworkIssue()) {
LOGGER.info("Twitter Network Issues Detected. Backing off...");
LOGGER.info("{} - {}", e.getExceptionCode(), e.getLocalizedMessage());
try {
Thread.sleep(retry);
} catch (InterruptedException e1) {
Thread.currentThread().interrupt();
}
return 1;
} else if (e.isErrorMessageAvailable()) {
if (e.getMessage().toLowerCase().contains("does not exist")) {
if (id != null) LOGGER.warn("User does not exist: {}", id);
else LOGGER.warn("User does not exist");
return 100;
} else {
return 1;
}
} else {
if (e.getExceptionCode().equals("ced778ef-0c669ac0")) {
// This is a known weird issue, not exactly sure the cause, but you'll never be able to
// get the data.
return 5;
} else if (e.getExceptionCode().equals("4be80492-0a7bf7c7")) {
// This is a 401 reflecting credentials don't have access to the requested resource.
if (id != null) LOGGER.warn("Authentication Exception accessing id: {}", id);
else LOGGER.warn("Authentication Exception");
return 5;
} else {
LOGGER.warn("Unknown Twitter Exception...");
LOGGER.warn(" Account: {}", twitter);
LOGGER.warn(" Access: {}", e.getAccessLevel());
LOGGER.warn(" Code: {}", e.getExceptionCode());
LOGGER.warn(" Message: {}", e.getLocalizedMessage());
return 1;
}
}
} else if (exception instanceof RuntimeException) {
LOGGER.warn("TwitterGrabber: Unknown Runtime Error", exception.getMessage());
return 1;
} else {
LOGGER.info("Completely Unknown Exception: {}", exception);
return 1;
}
}
private static ArrayList<String> ProcessTimeLine(String user)
throws InterruptedException, TwitterException {
ArrayList<String> Tweets = new ArrayList<String>();
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setDebugEnabled(true)
.setOAuthConsumerKey(KEY)
.setOAuthConsumerSecret(SECRET)
.setOAuthAccessToken(ACCESSTOKEN)
.setOAuthAccessTokenSecret(ACCESSSECRET);
cb.setJSONStoreEnabled(true);
// gets Twitter instance with default credentials
boolean bWait = true;
Twitter twitter = new TwitterFactory(cb.build()).getInstance();
do {
try {
Map<String, RateLimitStatus> oRT = twitter.getRateLimitStatus();
RateLimitStatus rateLimit = oRT.get("/statuses/user_timeline");
int remaining = rateLimit.getRemaining();
System.out.print("(Remaining API calls: " + remaining + ")");
int remainingTime = rateLimit.getSecondsUntilReset();
if (remaining <= NUM_TWEETS / 200 + 1) {
System.out.println("Waiting " + remainingTime + " seconds");
Thread.sleep(remainingTime * 1000);
} else bWait = false;
} catch (Exception te) {
if (te.toString().toLowerCase().contains("rate limit")
&& !te.toString().toLowerCase().contains("bad authentication data")) {
System.out.println("Waiting 60s");
Thread.sleep(60 * 1000);
} else {
bWait = false;
}
}
} while (bWait);
try {
Detector detector = DetectorFactory.create();
List<Status> statuses;
int iPage = 1;
int iTweets = 0;
do {
int iPageSize = 0;
if (iTweets + 200 < NUM_TWEETS) {
iPageSize = 200;
} else {
iPageSize = NUM_TWEETS - iTweets;
}
statuses = twitter.getUserTimeline(user, new Paging(iPage, iPageSize));
for (Status status : statuses) {
String sStatusId = "-1";
try {
if ((status.getRetweetedStatus() != null)
&& (status.getRetweetedStatus().getUser() != null)) {
continue;
}
try {
detector.append(Simplify(status.getText()));
if (detector.detect().equalsIgnoreCase("es")) {
String sStatusJSON = DataObjectFactory.getRawJSON(status);
Tweets.add(sStatusJSON);
}
} catch (Exception exl) {
}
} catch (Exception ex) {
System.out.println("ERROR in status id " + sStatusId);
}
iTweets++;
}
iPage++;
} while (statuses.size() > 0 && iTweets < NUM_TWEETS);
} catch (TwitterException te) {
te.printStackTrace();
System.out.println("Failed to get timeline: " + te.getMessage());
} catch (Exception ex) {
}
System.out.println("..." + Tweets.size() + " tweets.");
return Tweets;
}