@SuppressWarnings("unused")
private void fetchTweets(String company) {
Twitter twitter = new TwitterFactory().getInstance();
Query query = new Query(company);
query.setLang("en");
query.setRpp(100);
try {
List<TweetDTO> tweets = new ArrayList<TweetDTO>();
for (int i = 1; i < 10; i++) {
query.setPage(i);
QueryResult result = twitter.search(query);
for (Tweet tweet : result.getTweets()) {
TweetDTO dto = new TweetDTO();
dto.setTwitterId(Long.toString(tweet.getId()));
dto.setText(tweet.getText());
dto.setDate(tweet.getCreatedAt());
dto.setCompanies(company);
tweets.add(dto);
}
}
new GoogleTweetDAO().storeTweet(tweets);
} catch (TwitterException e) {
}
}
public void grab(Date timeLimit) {
log.info("Twitter grabber started...");
Twitter twitter = new TwitterFactory().getInstance();
List<Brand> brandList = handler.getBrandProvider().getAllBrands();
ArticleProvider articleProvider = handler.getArticleProvider();
for (Brand b : brandList) {
Query query = new Query(b.getName());
query.setRpp(PAGE_SIZE);
query.setLang("ru");
query.setResultType(Query.MIXED);
List<Tweet> resultTweets = new LinkedList<Tweet>();
QueryResult queryResult;
int pageNumber = 1;
try {
do {
query.setPage(pageNumber);
queryResult = twitter.search(query);
resultTweets.addAll(queryResult.getTweets());
pageNumber++;
log.info(pageNumber);
} while (ISSUANCE_SIZE > resultTweets.size());
} catch (TwitterException e) {
throw new RuntimeException(e);
}
log.info("tweets in da111y: " + resultTweets.size());
twitter = null;
twitter = new TwitterFactory().getInstance();
for (Tweet t : resultTweets) {
articleProvider.writeArticleToDataStore(
new Article(-1, b.getId(), 2, "", t.getText(), "", getSimpleTime(t.getCreatedAt()), 1));
}
}
log.info("twitter grabber finished succesful.");
}
/** @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) */
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
// Get the query and resultLimit parameters we are going to use for fetching the tweets.
Object queryObj = "google";
Object resultsLimit = "3";
// if(queryObj == null) {
// return;
// }
int limit = 5;
if (resultsLimit != null) {
try {
limit = Integer.parseInt((String) resultsLimit);
} catch (NumberFormatException e) {
// no need for action
}
}
String query = (String) queryObj;
Twitter twitter = new TwitterFactory().getInstance();
try {
// Search for the result tweets using the parameter query.
Query queryTweet = new Query(query);
QueryResult searchResult = twitter.search(queryTweet);
List<Tweet> tweets = searchResult.getTweets();
if (tweets.size() >= limit) {
tweets = tweets.subList(0, limit);
}
Long lastTweetId = null;
try {
lastTweetId = Long.parseLong((String) request.getParameter("lastTweetId"));
} catch (NumberFormatException e) {
lastTweetId = 0L;
}
// Iterate over the tweets and extract the information from each of them.
List<TwitterHolder> tweetTexts = new ArrayList<TwitterHolder>();
for (Tweet tweet : tweets) {
if (lastTweetId > 0 && tweet.getId() <= lastTweetId.longValue()) {
continue;
}
String text = tweet.getText();
// Find URLs in the tweet text and replace them with a html link to the founded URL.
Matcher matcher = pattern.matcher(text);
while (matcher.find()) {
String group = matcher.group();
StringBuilder replacedUrl =
new StringBuilder("<a href=\"")
.append(group)
.append("\">")
.append(group)
.append("</a>");
text = text.replaceAll(Pattern.quote(group), replacedUrl.toString());
}
Date publishedDate = tweet.getCreatedAt();
StringBuffer authorLink =
new StringBuffer("http://twitter.com/#!/").append(tweet.getFromUser());
/* Create TweetHolder instance and add it to the result list of TweetHolder instances.
* That's what is going to the template for rendering. */
tweetTexts.add(
new TwitterHolder(
text,
tweet.getFromUser(),
authorLink.toString(),
new PrettyTime(new Locale("en")).format(publishedDate),
tweet.getProfileImageUrl(),
tweet.getSource(),
String.valueOf(tweet.getId())));
}
Collections.reverse(tweetTexts);
String json = new Gson().toJson(tweetTexts);
// Add the result collection of TweetHolder instances to the response.
response.setCharacterEncoding("UTF-8");
response.setContentType("application/json");
response.getWriter().append(json);
} catch (TwitterException e) {
response.setCharacterEncoding("UTF-8");
response.setContentType("application/json");
response.getWriter().append("[]");
e.printStackTrace();
// return true;
return;
}
// return true;
}
public static int searchForTwits(int id, String text, Connection conn, String getDate)
throws TwitterException, SQLException, LangDetectException, ParseException {
Twitter twitter = new TwitterFactory().getInstance();
int countTweets = 0;
int pageNumber = 1;
int n = 0;
do {
Query query = new Query(text).rpp(100).page(pageNumber);
QueryResult result = twitter.search(query);
for (Tweet tweet : result.getTweets()) {
java.util.Date date = tweet.getCreatedAt();
Format formatter;
formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String newDate = formatter.format(date);
DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
java.util.Date oldDate = df.parse(getDate);
Statement input = conn.createStatement();
countTweets++;
if (date.after(oldDate)) {
Detector detector = DetectorFactory.create();
detector.append(tweet.getText());
String lang;
try {
lang = detector.detect();
if (lang.equals("lv") || lang.equals("ru")) {
try {
input.executeUpdate(
"INSERT INTO tweet "
+ "VALUES (null, '"
+ tweet.getId()
+ "', '"
+ tweet.getFromUser()
+ "', '"
+ tweet.getText().replace("'", "’")
+ "', '"
+ newDate
+ "', null, null, null)");
input.executeUpdate(
"INSERT INTO tweet_brand "
+ "VALUES (null, '"
+ tweet.getId()
+ "', '"
+ id
+ "')");
} catch (SQLException ex) {
}
n++;
} else continue;
} catch (LangDetectException ex) {
}
} else continue;
}
pageNumber++;
if (countTweets == 100) countTweets = 0;
else break;
} while (pageNumber != 16);
System.out.print(text + " ");
return n;
}