/**
* Transforms a twitter4j.Status object into a JSONObject
*
* @param pStatus a twitter4j.Status object
* @return JSONObject
*/
@SuppressWarnings("unchecked")
public static JSONObject getJson(Status pStatus) {
JSONObject jsonObj = null;
if (pStatus != null) {
jsonObj = new JSONObject();
jsonObj.put("createdAt", pStatus.getCreatedAt());
jsonObj.put("id", pStatus.getId());
jsonObj.put("text", pStatus.getText());
jsonObj.put("source", pStatus.getSource());
jsonObj.put("isTruncated", pStatus.isTruncated());
jsonObj.put("inReplyToStatusId", pStatus.getInReplyToStatusId());
jsonObj.put("inReplyToUserId", pStatus.getInReplyToUserId());
jsonObj.put("isFavorited", pStatus.isFavorited());
jsonObj.put("isRetweeted", pStatus.isRetweeted());
jsonObj.put("favoriteCount", pStatus.getFavoriteCount());
jsonObj.put("inReplyToScreenName", pStatus.getInReplyToScreenName());
jsonObj.put("geoLocation", pStatus.getGeoLocation());
jsonObj.put("place", pStatus.getPlace());
jsonObj.put("retweetCount", pStatus.getRetweetCount());
jsonObj.put("isPossiblySensitive", pStatus.isPossiblySensitive());
jsonObj.put("isoLanguageCode", pStatus.getIsoLanguageCode());
jsonObj.put("contributorsIDs", pStatus.getContributors());
jsonObj.put("retweetedStatus", pStatus.getRetweetedStatus());
jsonObj.put("userMentionEntities", pStatus.getUserMentionEntities());
jsonObj.put("urlEntities", pStatus.getURLEntities());
jsonObj.put("hashtagEntities", pStatus.getHashtagEntities());
jsonObj.put("mediaEntities", pStatus.getMediaEntities());
jsonObj.put("currentUserRetweetId", pStatus.getCurrentUserRetweetId());
jsonObj.put("user", pStatus.getUser());
}
return jsonObj;
}
public static Tweet fromStatus(Status status) {
Tweet tweet = new Tweet();
tweet.setUser(fromUser(status.getUser()));
if (status.getPlace() != null) {
tweet.setLocation(fromPlace(status.getPlace()));
}
tweet.setTimeOfTweet(status.getCreatedAt().toString());
tweet.setRetweetCount(status.getRetweetCount());
ArrayList<User> lusers = new ArrayList<User>();
String statusText = status.getText();
for (UserMentionEntity entity : status.getUserMentionEntities()) {
lusers.add(fromUserMentionEntity(entity));
statusText = stripMentionedUserFromStatus(statusText, entity);
}
tweet.setMessage(statusText);
tweet.setMentioned(lusers.toArray(new User[] {}));
return tweet;
}
public static void printTweet(Status tweet, boolean isStream) {
String time;
if (isStream) {
time = "";
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException ie) {
ie.printStackTrace();
}
} else {
time = getTimeForm(tweet) + " ";
}
System.out.println(SEPARATOR);
String uName = tweet.getUser().getScreenName();
String text = tweet.getText();
Integer retweets = tweet.getRetweetCount();
if (tweet.isRetweet()) {
Pattern myPattern = Pattern.compile("RT @([^ ]*): (.*)");
Matcher m = myPattern.matcher(text);
m.find();
String uRTName = m.group(1);
text = m.group(2);
System.out.println(time + "@" + uName + ": ретвитнул @" + uRTName + ": " + text);
} else {
System.out.println(time + "@" + uName + ": " + text + retweetsForm(retweets));
}
}
public static void insertTweet(Status tweet) throws SQLException {
try {
System.out.println(tweet.getCreatedAt() + ": " + tweet.getText());
if (tweet.getRetweetedStatus() != null) {
insertRetweet(tweet);
return;
}
PreparedStatement insertTweetStmt = SqlHelper.getInsertTweetStmt();
insertTweetStmt.setLong(1, tweet.getId());
if (tweet.getGeoLocation() != null) {
insertTweetStmt.setFloat(2, (float) tweet.getGeoLocation().getLatitude());
insertTweetStmt.setFloat(3, (float) tweet.getGeoLocation().getLongitude());
} else {
insertTweetStmt.setFloat(2, 0);
insertTweetStmt.setFloat(3, 0);
}
insertTweetStmt.setString(4, tweet.getText());
insertTweetStmt.setInt(5, (int) tweet.getRetweetCount());
insertTweetStmt.setTimestamp(6, new Timestamp(tweet.getCreatedAt().getTime()));
insertTweetStmt.setLong(7, tweet.getInReplyToStatusId());
insertTweetStmt.setLong(8, tweet.getUser().getId());
insertTweetStmt.executeUpdate();
} catch (MySQLIntegrityConstraintViolationException e) {
// ignore, this is just a duplicate tweet entry, that's rather normal
}
}
private String getTweetInfo(Status tweet) {
return "Tweet by: "
+ tweet.getUser().getName()
+ " Tweet: "
+ tweet.getText()
+ " Posted at: "
+ tweet.getCreatedAt()
+ " Source: "
+ tweet.getSource()
+ " RT Count: "
+ tweet.getRetweetCount()
+ " Hashtag counts "
+ tweet.getHashtagEntities().length;
}
@Override
public void storeData(Status status) throws InterruptedException {
BasicDBObject basicObj = new BasicDBObject();
// if(status.getGeoLocation().equals(true))
// if(status.getGeoLocation().getLatitude()!=0.0d)
// {
if (status.getGeoLocation() != null) {
try {
System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText());
basicObj.put("user_name", status.getUser().getScreenName());
basicObj.put("retweet_count", status.getRetweetCount());
// basicObj.put("tweet_followers_count",
// status.getUser().getFollowersCount());
basicObj.put("source", status.getSource());
// basicObj.put("coordinates",status.getGeoLocation());
basicObj.put("latitude", status.getGeoLocation().getLatitude());
basicObj.put("longitude", status.getGeoLocation().getLongitude());
// UserMentionEntity[] mentioned =
// status.getUserMentionEntities();
// basicObj.put("tweet_mentioned_count", mentioned.length);
basicObj.put("tweet_ID", status.getId());
basicObj.put("tweet_text", status.getText());
// i++;
} catch (Exception e) {
System.out.println(e);
}
try {
items.insert(basicObj);
} catch (Exception e) {
System.out.println("MongoDB Connection Error : " + e.getMessage());
}
}
// else
// throw new InterruptedException();
}
public void persistirTweets(
List<Status> tweets, String query, String polaridadIndicada, String codFinalidadData) {
logger.debug("<==== Inicio Method persistirTweet ====>");
StatusTweet dto = null;
HashTag ht = null;
Url url = null;
UserMention um = null;
Media md = null;
List<UserMention> ums = null;
List<HashTag> hts = null;
List<Url> urls = null;
List<Media> mds = null;
UserMentionEntity[] ume = null;
URLEntity[] urle = null;
HashtagEntity[] hte = null;
MediaEntity[] mes = null;
for (Status tweet : tweets) {
ums = null;
hts = null;
urls = null;
mds = null;
ume = null;
urle = null;
hte = null;
mes = null;
dto = null;
ht = null;
url = null;
um = null;
md = null;
logger.debug(
"tweet completo: {}",
ToStringBuilder.reflectionToString(tweet, ToStringStyle.MULTI_LINE_STYLE));
dto = new StatusTweet();
dto.setId(tweet.getId());
dto.setQuery(query);
dto.setMensaje(tweet.getText());
if (tweet.getUser() != null) {
dto.setScreenNameUser(tweet.getUser().getScreenName());
}
dto.setContadorRetweets((int) tweet.getRetweetCount());
dto.setCodOpinionInicial(polaridadIndicada);
dto.setFechaHoraRegistro(new Date());
dto.setFlagEnProceso(FLAG_POR_PROCESAR);
dto.setCodIdioma(COD_LENGUAJE_CONSULTA_ESPANOL);
dto.setCodFinalidadData(codFinalidadData);
if (!ArrayUtils.isEmpty(tweet.getUserMentionEntities())) {
ums = new ArrayList<UserMention>();
ume = tweet.getUserMentionEntities();
int size = ume.length;
for (int i = 0; i < size; i++) {
um = new UserMention();
um.setScreenName(ume[i].getScreenName());
um.setIdTweet(tweet.getId());
ums.add(um);
}
}
if (!ArrayUtils.isEmpty(tweet.getURLEntities())) {
urls = new ArrayList<Url>();
urle = tweet.getURLEntities();
int size = urle.length;
for (int i = 0; i < size; i++) {
url = new Url();
url.setIdTweet(tweet.getId());
url.setUrlCorta(urle[i].getURL());
url.setUrlLarga(urle[i].getExpandedURL());
url.setUrlMostrada(urle[i].getDisplayURL());
url.setFlagEnProceso(FLAG_POR_PROCESAR);
urls.add(url);
}
}
if (!ArrayUtils.isEmpty(tweet.getMediaEntities())) {
mds = new ArrayList<Media>();
mes = tweet.getMediaEntities();
int size = mes.length;
for (int i = 0; i < size; i++) {
md = new Media();
md.setIdTweet(tweet.getId());
md.setUrl(mes[i].getURL());
md.setMediaUrl(mes[i].getMediaURL());
md.setMediaUrlHttps(mes[i].getMediaURLHttps());
md.setExpandedUrl(mes[i].getExpandedURL());
mds.add(md);
}
}
if (!ArrayUtils.isEmpty(tweet.getHashtagEntities())) {
hts = new ArrayList<HashTag>();
hte = tweet.getHashtagEntities();
int size = hte.length;
for (int i = 0; i < size; i++) {
ht = new HashTag();
ht.setTextoTag(hte[i].getText());
ht.setIdTweet(tweet.getId());
hts.add(ht);
}
}
dto.setUrls(urls);
dto.setUsuariosMencionados(ums);
dto.setHashTags(hts);
dto.setMedias(mds);
persistirTweet(dto);
}
}
private void getAndStoreTweetsOnTopic(String topic) {
// Configuring twitter
ConfigurationBuilder configBuilder = new ConfigurationBuilder();
configBuilder.setOAuthAccessToken(ACCESS_TOKEN);
configBuilder.setOAuthAccessTokenSecret(ACCESS_TOKEN_SECRET);
configBuilder.setOAuthConsumerKey(CONSUMER_KEY);
configBuilder.setOAuthConsumerSecret(CONSUMER_KEY_SECRET);
Configuration configuration = configBuilder.build();
TwitterFactory factory = new TwitterFactory(configuration);
Twitter twitter = factory.getInstance();
Query query = new Query(topic);
// number of tweets to return per page
query.setCount(100);
try {
// search for tweets on specifc topics
QueryResult search = twitter.search(query);
List<Status> tweets = search.getTweets();
// initialize Mongo DB
Mongo mongo = new Mongo(MONGO_DB_HOST, MONGO_DB_PORT);
DB db = mongo.getDB(MONGO_DB_NAME);
DBCollection collection = db.getCollection(MONGO_DB_COLLECTION);
for (Status tweet : tweets) {
System.out.println(getTweetInfo(tweet));
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
Date dateWithoutTime = sdf.parse(sdf.format(tweet.getCreatedAt()));
// create a document
BasicDBObject document = new BasicDBObject();
document.put("createdAt", dateWithoutTime);
document.put("user", tweet.getUser().getName());
document.put("rTCount", tweet.getRetweetCount());
document.put("tweet", tweet.getText());
document.put("source", tweet.getSource());
// store it in database
collection.insert(document);
}
/**
* // Find all records from collection DBCursor cursorDoc = collection.find();
*
* <p>// ... and display while (cursorDoc.hasNext()) { System.out.println(cursorDoc.next());
* }*
*/
} catch (TwitterException e) {
e.printStackTrace();
} catch (UnknownHostException e) {
e.printStackTrace();
} catch (ParseException e) {
e.printStackTrace();
}
}
/*
* (non-Javadoc)
*
* @see edu.umd.cs.dmonner.tweater.StatusEater#persist(java.util.List, twitter4j.Status)
*/
public void persist(final List<QueryItem> matches, final Status status) {
final User user = status.getUser();
// get location information
final GeoLocation loc = status.getGeoLocation();
final double lat = loc == null ? 0D : loc.getLatitude();
final double lon = loc == null ? 0D : loc.getLongitude();
final String locStr = user.getLocation() == null ? "" : scrub(user.getLocation());
final String lang = user.getLang() == null ? "" : scrub(user.getLang());
// get retweet information
final boolean rt = status.isRetweet();
final long rtct = rt ? status.getRetweetCount() : 0;
final long rtid = rt ? status.getRetweetedStatus().getId() : -1;
synchronized (outfile) {
this.outfile.println(
user.getId()
+ ",\""
+ //
user.getScreenName()
+ "\",\""
+ //
locStr
+ "\","
+ //
user.getFollowersCount()
+ ","
+ //
user.getFriendsCount()
+ ","
+ //
status.getId()
+ ","
+ //
status.getCreatedAt()
+ ",\""
+ //
scrub(status.getText())
+ "\","
+ //
rt
+ ","
+ //
rtid
+ ","
+ //
rtct
+ ","
+ //
lat
+ ","
+ //
lon
+ ","
+ //
user.getCreatedAt()
+ ","
+ //
user.getStatusesCount()
+ ","
+ //
user.getListedCount()
+ ","
+ //
user.isVerified()
+ ",\""
+ //
lang
+ "\","
+ //
user.getUtcOffset() / 3600
+ ",\""
+ //
matches.get(0).toString()
+ "\"");
this.outfile.flush();
}
}
public static Record buildTweet(Schema schema, Status status) {
GenericRecordBuilder builderTweet = new GenericRecordBuilder(schema);
builderTweet.set("created_at", status.getCreatedAt().getTime());
builderTweet.set("favorite_count", status.getFavoriteCount());
builderTweet.set("favorited", status.isFavorited());
builderTweet.set("id", status.getId());
builderTweet.set("in_reply_to_screen_name", status.getInReplyToScreenName());
if (status.getInReplyToStatusId() != -1)
builderTweet.set("in_reply_to_status_id", status.getInReplyToStatusId());
if (status.getInReplyToUserId() != -1)
builderTweet.set("in_reply_to_user_id", status.getInReplyToUserId());
builderTweet.set("lang", status.getLang());
builderTweet.set("possibly_sensitive", status.isPossiblySensitive());
builderTweet.set("retweet_count", status.getRetweetCount());
builderTweet.set("retweeted", status.isRetweeted());
builderTweet.set("source", status.getSource());
builderTweet.set("text", status.getText());
builderTweet.set("truncated", status.isTruncated());
if (status.getWithheldInCountries() != null)
builderTweet.set("withheld_in_countries", Arrays.asList(status.getWithheldInCountries()));
if (status.getGeoLocation() != null)
builderTweet.set(
"coordinates",
Arrays.asList(
status.getGeoLocation().getLatitude(), status.getGeoLocation().getLongitude()));
builderTweet.set("entities", buildEntities(schema.getField("entities").schema(), status));
if (status.getPlace() != null)
builderTweet.set(
"place",
buildPlace(schema.getField("place").schema().getTypes().get(1), status.getPlace()));
User user = status.getUser();
if (user != null && schema.getField("user") != null) {
Schema schemaUser = schema.getField("user").schema();
GenericRecordBuilder builderUser = new GenericRecordBuilder(schemaUser);
builderUser.set("contributors_enabled", user.isContributorsEnabled());
builderUser.set("created_at", user.getCreatedAt().getTime());
builderUser.set("default_profile", user.isDefaultProfile());
builderUser.set("default_profile_image", user.isDefaultProfileImage());
builderUser.set("description", user.getDescription());
builderUser.set(
"entities",
buildURLEntity(schemaUser.getField("entities").schema(), user.getURLEntity()));
builderUser.set("favourites_count", user.getFavouritesCount());
builderUser.set("followers_count", user.getFollowersCount());
builderUser.set("friends_count", user.getFriendsCount());
builderUser.set("geo_enabled", user.isGeoEnabled());
builderUser.set("id", user.getId());
builderUser.set("is_translator", user.isTranslator());
builderUser.set("lang", user.getLang());
builderUser.set("listed_count", user.getListedCount());
builderUser.set("location", user.getLocation());
builderUser.set("name", user.getName());
builderUser.set("screen_name", user.getScreenName());
builderUser.set("profile_background_color", user.getProfileBackgroundColor());
builderUser.set("profile_background_image_url", user.getProfileBackgroundImageURL());
builderUser.set(
"profile_background_image_url_https", user.getProfileBackgroundImageUrlHttps());
builderUser.set("profile_background_tile", user.isProfileBackgroundTiled());
builderUser.set("profile_banner_url", user.getProfileBannerURL());
builderUser.set("profile_image_url", user.getProfileImageURL());
builderUser.set("profile_image_url_https", user.getProfileBackgroundImageUrlHttps());
builderUser.set("profile_link_color", user.getProfileLinkColor());
builderUser.set("profile_sidebar_border_color", user.getProfileSidebarBorderColor());
builderUser.set("profile_sidebar_fill_color", user.getProfileSidebarFillColor());
builderUser.set("profile_text_color", user.getProfileTextColor());
builderUser.set("profile_use_background_image", user.isProfileUseBackgroundImage());
builderUser.set("protected", user.isProtected());
builderUser.set("show_all_inline_media", user.isShowAllInlineMedia());
builderUser.set("statuses_count", user.getStatusesCount());
builderUser.set("time_zone", user.getTimeZone());
builderUser.set("url", user.getURL());
builderUser.set("utc_offset", user.getUtcOffset());
builderUser.set("verified", user.isVerified());
if (user.getStatus() != null && schemaUser.getField("status") != null)
builderUser.set(
"status",
buildTweet(schemaUser.getField("status").schema().getTypes().get(1), user.getStatus()));
if (user.getWithheldInCountries() != null)
builderUser.set("withheld_in_countries", Arrays.asList(user.getWithheldInCountries()));
builderTweet.set("user", builderUser.build());
}
if (status.getQuotedStatus() != null && schema.getField("quoted_status") != null)
builderTweet.set(
"quoted_status",
buildTweet(
schema.getField("quoted_status").schema().getTypes().get(1),
status.getQuotedStatus()));
if (status.getRetweetedStatus() != null && schema.getField("retweeted_status") != null)
builderTweet.set(
"retweeted_status",
buildTweet(
schema.getField("retweeted_status").schema().getTypes().get(1),
status.getRetweetedStatus()));
return builderTweet.build();
}