public void testSearchUser() throws TwitterException {
ResponseList<User> users = twitter1.searchUsers("Doug Williams", 1);
assertTrue(4 < users.size());
assertNotNull(DataObjectFactory.getRawJSON(users.get(0)));
assertEquals(
users.get(0), DataObjectFactory.createUser(DataObjectFactory.getRawJSON(users.get(0))));
assertNotNull(DataObjectFactory.getRawJSON(users));
}
public void testAccountProfileImageUpdates() throws Exception {
User user = twitter1.updateProfileImage(new FileInputStream(getRandomlyChosenFile()));
assertNotNull(DataObjectFactory.getRawJSON(user));
// tile randomly
User user2 =
twitter1.updateProfileBackgroundImage(
getRandomlyChosenFile(), (5 < System.currentTimeMillis() % 5));
assertNotNull(DataObjectFactory.getRawJSON(user2));
assertEquals(user2, DataObjectFactory.createUser(DataObjectFactory.getRawJSON(user2)));
}
public void testLookupUsers() throws TwitterException {
ResponseList<User> users = twitter1.lookupUsers(new String[] {id1.screenName, id2.screenName});
assertEquals(2, users.size());
assertContains(users, id1);
assertContains(users, id2);
users = twitter1.lookupUsers(new long[] {id1.id, id2.id});
assertEquals(2, users.size());
assertContains(users, id1);
assertContains(users, id2);
assertNotNull(DataObjectFactory.getRawJSON(users.get(0)));
assertEquals(
users.get(0), DataObjectFactory.createUser(DataObjectFactory.getRawJSON(users.get(0))));
assertNotNull(DataObjectFactory.getRawJSON(users));
}
public void testShowUser() throws Exception {
User user = twitter1.showUser(id1.screenName);
assertEquals(id1.screenName, user.getScreenName());
assertNotNull(user.getLocation());
assertNotNull(user.getDescription());
assertNotNull(user.getProfileImageURL());
assertNotNull(user.getURL());
assertFalse(user.isProtected());
assertTrue(0 <= user.getFavouritesCount());
assertTrue(0 <= user.getFollowersCount());
assertTrue(0 <= user.getFriendsCount());
assertNotNull(user.getCreatedAt());
assertNotNull(user.getTimeZone());
assertNotNull(user.getProfileBackgroundImageUrl());
assertTrue(0 <= user.getStatusesCount());
assertNotNull(user.getProfileBackgroundColor());
assertNotNull(user.getProfileTextColor());
assertNotNull(user.getProfileLinkColor());
assertNotNull(user.getProfileSidebarBorderColor());
assertNotNull(user.getProfileSidebarFillColor());
assertNotNull(user.getProfileTextColor());
assertTrue(1 < user.getFollowersCount());
if (user.getStatus() != null) {
assertNotNull(user.getStatus().getCreatedAt());
assertNotNull(user.getStatus().getText());
assertNotNull(user.getStatus().getSource());
assertFalse(user.getStatus().isFavorited());
assertEquals(-1, user.getStatus().getInReplyToStatusId());
assertEquals(-1, user.getStatus().getInReplyToUserId());
assertFalse(user.getStatus().isFavorited());
assertNull(user.getStatus().getInReplyToScreenName());
}
assertTrue(1 <= user.getListedCount());
assertFalse(user.isFollowRequestSent());
// test case for TFJ-91 null pointer exception getting user detail on users with no statuses
// http://yusuke.homeip.net/jira/browse/TFJ-91
twitter1.showUser("twit4jnoupdate");
user = twitter1.showUser("tigertest");
User previousUser = user;
assertNotNull(DataObjectFactory.getRawJSON(user));
user = twitter1.showUser(numberId);
assertEquals(numberIdId, user.getId());
assertNull(DataObjectFactory.getRawJSON(previousUser));
assertNotNull(DataObjectFactory.getRawJSON(user));
assertEquals(user, DataObjectFactory.createUser(DataObjectFactory.getRawJSON(user)));
previousUser = user;
user = twitter1.showUser(numberIdId);
assertEquals(numberIdId, user.getId());
assertNotNull(DataObjectFactory.getRawJSON(user));
assertEquals(user, DataObjectFactory.createUser(DataObjectFactory.getRawJSON(user)));
}
/** Synchronize lists between what is available in the db and on twitter. */
private void syncLists() {
TwitterHelper th = new TwitterHelper(this, account);
TweetDB tdb = new TweetDB(this, accountId);
if (account.getServerType().equalsIgnoreCase("twitter")) {
List<UserList> userLists = th.getUserLists();
Map<String, Integer> storedLists = tdb.getLists();
// Check for lists to add
for (UserList userList : userLists) {
if (!storedLists.containsValue(userList.getId())) {
tdb.addList(userList.getName(), userList.getId(), DataObjectFactory.getRawJSON(userList));
}
}
// check for outdated lists and remove them
for (Entry<String, Integer> entry : storedLists.entrySet()) {
Integer id = entry.getValue();
boolean found = false;
for (UserList userList2 : userLists) {
if (userList2.getId() == id) {
found = true;
break;
}
}
if (!found) {
tdb.removeList(id);
}
}
syncSearches(th, tdb);
}
}
public static Status getStatusTweet(String JSONString) {
if (JSONString == null) return null;
Status tweet = null;
try {
// parse json to object type
tweet = DataObjectFactory.createStatus(JSONString);
} catch (TwitterException e) {
System.err.println("error parsing tweet object");
return null;
}
return tweet;
}
public void testUnparsable() throws Exception {
String str;
str = "";
try {
DataObjectFactory.createStatus(str);
fail("should fail");
} catch (TwitterException expected) {
} catch (Error notExpected) {
fail("failed" + notExpected.getMessage());
}
try {
DataObjectFactory.createStatus(str);
fail("should fail");
} catch (TwitterException expected) {
} catch (Error notExpected) {
fail("failed" + notExpected.getMessage());
}
str =
"{\"in_reply_to_status_id_str\":null,\"place\":null,\"in_reply_to_user_id\":null,\"text\":\"working\",\"contributors\":null,\"retweet_count\":0,\"in_reply_to_user_id_str\":null,\"retweeted\":false,\"id_str\":\"794626207\",\"source\":\"\\u003Ca href=\\\"http:\\/\\/twitterhelp.blogspot.com\\/2008\\/05\\/twitter-via-mobile-web-mtwittercom.html\\\" rel=\\\"nofollow\\\"\\u003Emobile web\\u003C\\/a\\u003E\",\"truncated\":false,\"geo\":null,\"in_reply_to_status_id\":null,\"favorited\":false,\"user\":{\"show_all_inline_media\":false,\"geo_enabled\":false,\"profile_background_tile\":false,\"time_zone\":null,\"favourites_count\":0,\"description\":null,\"friends_count\":0,\"profile_link_color\":\"0084B4\",\"location\":null,\"profile_sidebar_border_color\":\"C0DEED\",\"id_str\":\"14481043\",\"url\":null,\"follow_request_sent\":false,\"statuses_count\":1,\"profile_use_background_image\":true,\"lang\":\"en\",\"profile_background_color\":\"C0DEED\",\"profile_image_url\":\"http:\\/\\/a3.twimg.com\\/a\\/1292975674\\/images\\/default_profile_3_normal.png\",\"profile_background_image_url\":\"http:\\/\\/a3.twimg.com\\/a\\/1292975674\\/images\\/themes\\/theme1\\/bg.png\",\"followers_count\":44,\"protected\":false,\"contributors_enabled\":false,\"notifications\":false,\"screen_name\":\"Yusuke\",\"name\":\"Yusuke\",\"is_translator\":false,\"listed_count\":1,\"following\":false,\"verified\":false,\"profile_text_color\":\"333333\",\"id\":14481043,\"utc_offset\":null,\"created_at\":\"Tue Apr 22 21:49:13 +0000 2008\",\"profile_sidebar_fill_color\":\"DDEEF6\"},\"id\":794626207,\"coordinates\":null,\"in_reply_to_screen_name\":null,\"created_at\":\"Tue Apr 2200 21:49:34 +0000 2008\"";
try {
DataObjectFactory.createCategory(str);
fail("should fail");
} catch (TwitterException expected) {
expected.printStackTrace();
} catch (Error notExpected) {
fail("failed" + notExpected.getMessage());
}
try {
DataObjectFactory.createCategory(str);
fail("should fail");
} catch (TwitterException expected) {
} catch (Error notExpected) {
fail("failed" + notExpected.getMessage());
}
}
public void onStatus(twitter4j.Status status) {
Logger.info(status.getUser().getName() + " : " + status.getText());
Tweet tweet = new Tweet(status);
tweet.conformsToTerms = checkMatch(status);
tweet.save();
if (tweet.conformsToTerms && esClient != null) {
String json = DataObjectFactory.getRawJSON(status);
json =
json.replaceAll(
"(\"geo\":\\{\"type\":\"Point\",\"coordinates\":)\\[([-0-9.,]*)\\]", "$1\"$2\"");
// Logger.debug("geo mangled json");
// Logger.debug(json);
IndexResponse response =
esClient.prepareIndex("twitter", "tweet").setSource(json).execute().actionGet();
}
}
/**
* Performing the searching operation for the given Geo Query criteria.
*
* @param twitter
* @param query
* @return
* @throws XMLStreamException
* @throws TwitterException
* @throws JSONException
* @throws IOException
*/
private OMElement performSearchMessages(List<User> users)
throws XMLStreamException, TwitterException, JSONException, IOException {
OMElement resultElement = AXIOMUtil.stringToOM("<jsonObject><followers/></jsonObject>");
OMElement childElment = resultElement.getFirstElement();
for (User user : users) {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append("{ \"follower\" : ");
String json = DataObjectFactory.getRawJSON(user);
stringBuilder.append(json);
stringBuilder.append("} ");
OMElement element = super.parseJsonToXml(stringBuilder.toString());
childElment.addChild(element.getFirstOMChild());
}
if (users.size() == 0) {
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append("{ \"follower\" : {}");
stringBuilder.append("} ");
OMElement element = super.parseJsonToXml(stringBuilder.toString());
resultElement.addChild(element);
}
return resultElement;
}
public void testAccountMethods() throws Exception {
User original = twitter1.verifyCredentials();
assertNotNull(DataObjectFactory.getRawJSON(original));
assertEquals(original, DataObjectFactory.createUser(DataObjectFactory.getRawJSON(original)));
if (original.getScreenName().endsWith("new") || original.getName().endsWith("new")) {
original =
twitter1.updateProfile(
"twit4j",
"http://yusuke.homeip.net/twitter4j/",
"location:",
"Hi there, I do test a lot!new");
}
String newName, newURL, newLocation, newDescription;
String neu = "new";
newName = original.getName() + neu;
newURL = original.getURL() + neu;
newLocation = new Date().toString();
newDescription = original.getDescription() + neu;
User altered = twitter1.updateProfile(newName, newURL, newLocation, newDescription);
assertNotNull(DataObjectFactory.getRawJSON(altered));
assertEquals(original, DataObjectFactory.createUser(DataObjectFactory.getRawJSON(original)));
assertEquals(altered, DataObjectFactory.createUser(DataObjectFactory.getRawJSON(altered)));
twitter1.updateProfile(
original.getName(),
original.getURL().toString(),
original.getLocation(),
original.getDescription());
assertEquals(newName, altered.getName());
assertEquals(newURL, altered.getURL().toString());
assertEquals(newLocation, altered.getLocation());
assertEquals(newDescription, altered.getDescription());
User eu;
eu = twitter1.updateProfileColors("f00", "f0f", "0ff", "0f0", "f0f");
assertNotNull(DataObjectFactory.getRawJSON(eu));
assertEquals(eu, DataObjectFactory.createUser(DataObjectFactory.getRawJSON(eu)));
assertEquals("f00", eu.getProfileBackgroundColor());
assertEquals("f0f", eu.getProfileTextColor());
assertEquals("0ff", eu.getProfileLinkColor());
assertEquals("0f0", eu.getProfileSidebarFillColor());
assertEquals("f0f", eu.getProfileSidebarBorderColor());
assertTrue(eu.isProfileUseBackgroundImage());
assertFalse(eu.isShowAllInlineMedia());
assertTrue(0 <= eu.getListedCount());
eu = twitter1.updateProfileColors("87bc44", "9ae4e8", "000000", "0000ff", "e0ff92");
assertNotNull(DataObjectFactory.getRawJSON(eu));
assertEquals(eu, DataObjectFactory.createUser(DataObjectFactory.getRawJSON(eu)));
assertEquals("87bc44", eu.getProfileBackgroundColor());
assertEquals("9ae4e8", eu.getProfileTextColor());
assertEquals("000000", eu.getProfileLinkColor());
assertEquals("0000ff", eu.getProfileSidebarFillColor());
assertEquals("e0ff92", eu.getProfileSidebarBorderColor());
AccountSettings settings = twitter1.getAccountSettings();
assertTrue(settings.isSleepTimeEnabled());
assertTrue(settings.isGeoEnabled());
assertEquals("en", settings.getLanguage());
assertEquals("Rome", settings.getTimeZone().getName());
assertTrue(settings.isAlwaysUseHttps());
assertFalse(settings.isDiscoverableByEmail());
Location[] locations = settings.getTrendLocations();
assertTrue(0 < locations.length);
AccountSettings intermSettings =
twitter1.updateAccountSettings(1 /* GLOBAL */, true, "23", "08", "Helsinki", "it");
assertTrue(intermSettings.isSleepTimeEnabled());
assertEquals(intermSettings.getSleepStartTime(), "23");
assertEquals(intermSettings.getSleepEndTime(), "8");
assertTrue(intermSettings.isGeoEnabled());
assertEquals("it", intermSettings.getLanguage());
assertTrue(intermSettings.isAlwaysUseHttps());
assertFalse(intermSettings.isDiscoverableByEmail());
assertEquals("Helsinki", intermSettings.getTimeZone().getName());
Location[] intermLocations = intermSettings.getTrendLocations();
assertTrue(0 < intermLocations.length);
AccountSettings lastSettings =
twitter1.updateAccountSettings(
settings.getTrendLocations()[0].getWoeid(),
settings.isSleepTimeEnabled(),
settings.getSleepStartTime(),
settings.getSleepStartTime(),
settings.getTimeZone().getName(),
settings.getLanguage());
assertEquals(settings.getLanguage(), lastSettings.getLanguage());
assertEquals(settings.isSleepTimeEnabled(), lastSettings.isSleepTimeEnabled());
assertEquals(settings.getTimeZone().getName(), lastSettings.getTimeZone().getName());
assertEquals(settings.getSleepEndTime(), lastSettings.getSleepEndTime());
}
public void testBlockMethods() throws Exception {
User user1 = twitter2.createBlock(id1.screenName);
assertNotNull(DataObjectFactory.getRawJSON(user1));
assertEquals(user1, DataObjectFactory.createUser(DataObjectFactory.getRawJSON(user1)));
User user2 = twitter2.destroyBlock(id1.screenName);
assertNotNull(DataObjectFactory.getRawJSON(user2));
assertEquals(user2, DataObjectFactory.createUser(DataObjectFactory.getRawJSON(user2)));
assertFalse(twitter1.existsBlock("twit4j2"));
assertTrue(twitter1.existsBlock("twit4jblock"));
List<User> users = twitter1.getBlockingUsers();
assertNotNull(DataObjectFactory.getRawJSON(users));
assertEquals(
users.get(0), DataObjectFactory.createUser(DataObjectFactory.getRawJSON(users.get(0))));
assertEquals(1, users.size());
assertEquals(39771963, users.get(0).getId());
users = twitter1.getBlockingUsers(1);
assertNotNull(DataObjectFactory.getRawJSON(users));
assertEquals(
users.get(0), DataObjectFactory.createUser(DataObjectFactory.getRawJSON(users.get(0))));
assertEquals(1, users.size());
assertEquals(39771963, users.get(0).getId());
IDs ids = twitter1.getBlockingUsersIDs();
assertNull(DataObjectFactory.getRawJSON(users));
assertNotNull(DataObjectFactory.getRawJSON(ids));
assertEquals(1, ids.getIDs().length);
assertEquals(39771963, ids.getIDs()[0]);
}
private static ArrayList<String> ProcessTimeLine(String user)
throws InterruptedException, TwitterException {
ArrayList<String> Tweets = new ArrayList<String>();
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setDebugEnabled(true)
.setOAuthConsumerKey(KEY)
.setOAuthConsumerSecret(SECRET)
.setOAuthAccessToken(ACCESSTOKEN)
.setOAuthAccessTokenSecret(ACCESSSECRET);
cb.setJSONStoreEnabled(true);
// gets Twitter instance with default credentials
boolean bWait = true;
Twitter twitter = new TwitterFactory(cb.build()).getInstance();
do {
try {
Map<String, RateLimitStatus> oRT = twitter.getRateLimitStatus();
RateLimitStatus rateLimit = oRT.get("/statuses/user_timeline");
int remaining = rateLimit.getRemaining();
System.out.print("(Remaining API calls: " + remaining + ")");
int remainingTime = rateLimit.getSecondsUntilReset();
if (remaining <= NUM_TWEETS / 200 + 1) {
System.out.println("Waiting " + remainingTime + " seconds");
Thread.sleep(remainingTime * 1000);
} else bWait = false;
} catch (Exception te) {
if (te.toString().toLowerCase().contains("rate limit")
&& !te.toString().toLowerCase().contains("bad authentication data")) {
System.out.println("Waiting 60s");
Thread.sleep(60 * 1000);
} else {
bWait = false;
}
}
} while (bWait);
try {
Detector detector = DetectorFactory.create();
List<Status> statuses;
int iPage = 1;
int iTweets = 0;
do {
int iPageSize = 0;
if (iTweets + 200 < NUM_TWEETS) {
iPageSize = 200;
} else {
iPageSize = NUM_TWEETS - iTweets;
}
statuses = twitter.getUserTimeline(user, new Paging(iPage, iPageSize));
for (Status status : statuses) {
String sStatusId = "-1";
try {
if ((status.getRetweetedStatus() != null)
&& (status.getRetweetedStatus().getUser() != null)) {
continue;
}
try {
detector.append(Simplify(status.getText()));
if (detector.detect().equalsIgnoreCase("es")) {
String sStatusJSON = DataObjectFactory.getRawJSON(status);
Tweets.add(sStatusJSON);
}
} catch (Exception exl) {
}
} catch (Exception ex) {
System.out.println("ERROR in status id " + sStatusId);
}
iTweets++;
}
iPage++;
} while (statuses.size() > 0 && iTweets < NUM_TWEETS);
} catch (TwitterException te) {
te.printStackTrace();
System.out.println("Failed to get timeline: " + te.getMessage());
} catch (Exception ex) {
}
System.out.println("..." + Tweets.size() + " tweets.");
return Tweets;
}
public static void main(String argv[]) throws IOException {
argv[0] = "-mis";
argv[1] = "/Users/Indri/Eclipse_workspace/GeoNames/cities1000.txt";
argv[2] = "/Users/Indri/Eclipse_workspace/GazIndex";
argv[3] = "SampleInput/jsonTweets.txt";
argv[4] = "-json";
argv[5] = "SampleOutput/jsonTweets.out.csv";
boolean misspell = argv[0].equals("-mis") ? true : false;
String dicPath = argv[1]; // = "GeoNames/allCountries.txt";// gazetteer from geonames
String indexPath = argv[2]; // index path
String input = argv[3]; // = "tweet.csv";//to be determined.// test file path
String type = argv[4]; // -json or -text
String output = argv[5]; // = "output2.csv"; //output file path
CollaborativeIndex ci =
new CollaborativeIndex()
.config("GazIndex/StringIndex", "GazIndex/InfoIndex", "mmap", "mmap")
.open();
EnglishParser enparser = new EnglishParser("res/", ci, false);
ContextDisamb c = new ContextDisamb();
LangDetector lang = new LangDetector("res/langdetect.profile");
BufferedReader reader = GetReader.getUTF8FileReader(argv[3]);
CsvWriter writer = new CsvWriter(output, ',', Charset.forName("utf-8")); // write
writer.writeRecord(new String[] {"SPANISH TWEETS", "LOCATIONS"});
String line = null;
while ((line = reader.readLine()) != null) {
line = line.trim();
if (line.length() == 0) continue;
Tweet t = new Tweet();
String text = null;
if (argv[4].equals("-text")) text = line;
else
try {
text = (DataObjectFactory.createStatus(line.trim()).getText());
} catch (TwitterException e) {
// TODO Auto-generated catch block
System.err.println("JSON format corrupted, or no content.");
continue;
}
t.setText(text);
List<String> match = enparser.parse(t);
// Generate Matches
if (match == null || match.size() == 0) {
/** write blank result and the line itself if no match found. */
writer.writeRecord(new String[] {text, ""});
continue;
}
HashSet<String> reducedmatch = new HashSet<String>();
for (String s : match) reducedmatch.add(s.substring(3, s.length() - 3));
// Disambiguate topo
HashMap<String, String[]> result = c.returnBestTopo(ci, reducedmatch);
if (result == null) {
System.out.println("No GPS for any location is found.");
} else {
System.out.println("The grounded location(s) are:");
String topoStr = "";
for (String topo : result.keySet())
topoStr +=
"["
+ (topo
+ ": "
+ result.get(topo)[2]
+ " "
+ result.get(topo)[0]
+ " "
+ result.get(topo)[1])
+ "] ";
writer.writeRecord(new String[] {text, topoStr});
}
}
reader.close();
writer.close();
}
public JSONTweet readLine() {
String line;
try {
line = br.readLine();
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
return null;
}
if (line == null) return null;
Status tweet = null;
try {
// parse json to object type
tweet = DataObjectFactory.createStatus(line);
} catch (TwitterException e) {
System.err.println("error parsing tweet object");
return null;
}
jsontweet.JSON = line;
jsontweet.id = tweet.getId();
jsontweet.source = tweet.getSource();
jsontweet.text = tweet.getText();
jsontweet.createdat = tweet.getCreatedAt();
jsontweet.tweetgeolocation = tweet.getGeoLocation();
User user;
if ((user = tweet.getUser()) != null) {
jsontweet.userdescription = user.getDescription();
jsontweet.userid = user.getId();
jsontweet.userlanguage = user.getLang();
jsontweet.userlocation = user.getLocation();
jsontweet.username = user.getName();
jsontweet.usertimezone = user.getTimeZone();
jsontweet.usergeoenabled = user.isGeoEnabled();
if (user.getURL() != null) {
String url = user.getURL().toString();
jsontweet.userurl = url;
String addr = url.substring(7).split("/")[0];
String[] countrysuffix = addr.split("[.]");
String suffix = countrysuffix[countrysuffix.length - 1];
jsontweet.userurlsuffix = suffix;
try {
InetAddress address = null; // InetAddress.getByName(user.getURL().getHost());
String generate_URL
// =
// "http://www.geobytes.com/IpLocator.htm?GetLocation&template=php3.txt&IpAddress="
= "http://www.geoplugin.net/php.gp?ip=" + address.getHostAddress();
URL data = new URL(generate_URL);
URLConnection yc = data.openConnection();
BufferedReader in = new BufferedReader(new InputStreamReader(yc.getInputStream()));
String inputLine;
String temp = "";
while ((inputLine = in.readLine()) != null) {
temp += inputLine + "\n";
}
temp = temp.split("s:2:\"")[1].split("\"")[0];
jsontweet.userurllocation = temp;
} catch (Exception uhe) {
// uhe.printStackTrace();
jsontweet.userurllocation = null;
}
}
}
return jsontweet;
}