public String toString() {
return "{\"name\" : "
+ JSONObject.quote(topicName)
+ ", "
+ "\"uri\" : "
+ JSONObject.quote(topicURI)
+ ","
+ "\"score\" : "
+ score
+ "}";
}
public List<Tweet> readFile(String fileLocation, int limit, String fromTweetId) {
long startTime = System.currentTimeMillis();
int errors = 0;
int added = 0;
int ignored = 0;
int skipped = 0;
List<Tweet> tweets = new ArrayList<Tweet>();
try {
// Read gzFile
InputStream inputStream = new GZIPInputStream(new FileInputStream(fileLocation));
Reader decoder = new InputStreamReader(inputStream);
BufferedReader br = new BufferedReader(decoder);
String status;
while ((status = br.readLine()) != null) {
// Ignore garbage and log output lines, all statuses start with JSON bracket
if (!status.equals("") && status.charAt(0) == '{') {
try {
JSONObject jsonObject = new JSONObject(status);
// We use created_at as an indicator that this is a Tweet.
if (jsonObject.has("created_at")) {
Status statusObject = TwitterObjectFactory.createStatus(status);
Tweet tweet = this.getTweetObjectFromStatus(statusObject);
if (fromTweetId != null) {
if (fromTweetId.equals(tweet.getId())) {
this.log.write("StatusFileReader - Scanner pickup from " + fromTweetId);
fromTweetId = null;
} else {
skipped++;
}
continue;
}
added++;
tweets.add(tweet);
if (limit > 0 && added >= limit) {
break;
}
} else {
ignored++;
}
} catch (JSONException e) {
this.log.write(
"Exception in StatusFileReader: Json Parse Failure on: "
+ status
+ ", "
+ e.getMessage());
}
} else {
ignored++;
}
}
br.close();
decoder.close();
inputStream.close();
} catch (Exception e) {
this.log.write(
"Exception in StatusFileReader: Error Reading File: "
+ e.getClass().getName()
+ ": "
+ e.getMessage());
}
long runTimeSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - startTime);
double ops = ((double) added / (double) Math.max(runTimeSeconds, 1));
this.log.write(
"StatusFileReader - "
+ fileLocation
+ " processed in "
+ runTimeSeconds
+ "s. "
+ added
+ " ok / "
+ errors
+ " errors / "
+ ignored
+ " ignored / "
+ skipped
+ " skipped. "
+ ops
+ " ops. Limit: "
+ limit
+ ", Fetch: "
+ tweets.size());
return tweets;
}
// Save a new Twitter Search
@POST
@Produces(MediaType.APPLICATION_JSON)
@UserConstraint(functionalities = {SpagoBIConstants.CREATE_SOCIAL_ANALYSIS})
public String search(@Context HttpServletRequest req) {
logger.debug("Method search(): Start");
try {
TwitterAnalysisLauncher twitterLauncher = null;
TwitterSearchScheduler twitterSearchScheduler = null;
TwitterMonitorScheduler twitterMonitorScheduler = null;
TwitterSearch twitterSearch = new TwitterSearch();
String languageCode = null;
// reading the user input
// String searchType = req.getParameter("searchType");
String keywords = req.getParameter("keywords");
String links = req.getParameter("links");
String accounts = req.getParameter("accounts");
String documents = req.getParameter("documents");
String booleanOperator = req.getParameter("booleanOperator");
// set ready parameters
// twitterSearch.setLanguageCode(languageCode);
if (booleanOperator != null && !booleanOperator.equals("")) {
if (booleanOperator.equalsIgnoreCase("AND")) {
twitterSearch.setBooleanOperator(BooleanOperatorEnum.AND);
} else if (booleanOperator.equalsIgnoreCase("OR")) {
twitterSearch.setBooleanOperator(BooleanOperatorEnum.OR);
} else if (booleanOperator.equalsIgnoreCase("FREE")) {
twitterSearch.setBooleanOperator(BooleanOperatorEnum.FREE);
}
}
twitterSearch.setType(SearchTypeEnum.SEARCHAPI);
twitterSearch.setLoading(true);
twitterSearch.setKeywords(keywords);
twitterSearch.setCreationDate(GregorianCalendar.getInstance());
twitterSearch.setLastActivationTime(GregorianCalendar.getInstance());
// check if advanced option "R Analysis" is checked
if (req.getParameter("ranalysis") != null) {
twitterSearch.setrAnalysis(true);
} else {
twitterSearch.setrAnalysis(false);
}
// if user is not specifying the label, create it with the keywords
String label = req.getParameter("label");
if (label == null || label.trim().equals("")) {
logger.debug("Method search(): Blank label. Creation from keywords");
String[] keywordsArr = keywords.split(",");
for (int i = 0; i < keywordsArr.length; i++) {
String tempKeyword = keywordsArr[i].trim();
if (i == keywordsArr.length - 1) {
label = label + tempKeyword;
} else {
label = label + tempKeyword + "_";
}
}
label = label + "_" + System.currentTimeMillis();
}
// set search label
twitterSearch.setLabel(label);
String repeatTypeField = req.getParameter("repeatType");
String numberRepeat = req.getParameter("numberRepeat");
if (req.getParameter("isRepeating") != null
&& numberRepeat != null
&& !numberRepeat.equals("")
&& repeatTypeField != null
&& !repeatTypeField.equals("")) {
logger.debug("Method search(): Search with scheduler");
twitterSearchScheduler = new TwitterSearchScheduler();
Calendar startingDate = GregorianCalendar.getInstance();
startingDate.set(Calendar.SECOND, 0);
startingDate.set(Calendar.MILLISECOND, 0);
int repeatFrequency = Integer.parseInt(numberRepeat);
if (repeatFrequency > 0) {
if (repeatTypeField.equalsIgnoreCase(SearchRepeatTypeEnum.Day.toString())) {
startingDate.add(Calendar.DAY_OF_MONTH, repeatFrequency);
twitterSearchScheduler.setRepeatType(SearchRepeatTypeEnum.Day);
} else if (repeatTypeField.equalsIgnoreCase(SearchRepeatTypeEnum.Hour.toString())) {
startingDate.add(Calendar.HOUR_OF_DAY, repeatFrequency);
twitterSearchScheduler.setRepeatType(SearchRepeatTypeEnum.Hour);
}
twitterSearchScheduler.setActive(true);
twitterSearchScheduler.setStartingTime(startingDate);
twitterSearchScheduler.setRepeatFrequency(repeatFrequency);
twitterSearchScheduler.setTwitterSearch(twitterSearch);
}
}
// set search scheduler
twitterSearch.setTwitterSearchScheduler(twitterSearchScheduler);
// now we take the decision abount the monitor scheduler. Check if there
// resources to monitor..
if ((links != null && !links.equals(""))
|| (accounts != null && !accounts.equals(""))
|| (documents != null && !documents.equals(""))) {
//
String numberUpTo = req.getParameter("numberUpTo");
String typeUpTo = req.getParameter("typeUpTo");
String monitorFrequencyValue = req.getParameter("monitorFrequencyValue");
String monitorFrequencyType = req.getParameter("monitorFrequencyType");
if (monitorFrequencyValidation(monitorFrequencyValue, monitorFrequencyType)
&& monitorUpToValidation(numberUpTo, typeUpTo)) {
twitterMonitorScheduler = new TwitterMonitorScheduler();
twitterMonitorScheduler.setRepeatFrequency(Integer.parseInt(monitorFrequencyValue));
if (monitorFrequencyType.equalsIgnoreCase(MonitorRepeatTypeEnum.Day.toString())) {
twitterMonitorScheduler.setRepeatType(MonitorRepeatTypeEnum.Day);
} else if (monitorFrequencyType.equalsIgnoreCase(MonitorRepeatTypeEnum.Hour.toString())) {
twitterMonitorScheduler.setRepeatType(MonitorRepeatTypeEnum.Hour);
}
twitterMonitorScheduler.setUpToValue(Integer.parseInt(numberUpTo));
if (typeUpTo.equalsIgnoreCase(UpToTypeEnum.Day.toString())) {
twitterMonitorScheduler.setUpToType(UpToTypeEnum.Day);
} else if (typeUpTo.equalsIgnoreCase(UpToTypeEnum.Week.toString())) {
twitterMonitorScheduler.setUpToType(UpToTypeEnum.Week);
} else if (typeUpTo.equalsIgnoreCase(UpToTypeEnum.Month.toString())) {
twitterMonitorScheduler.setUpToType(UpToTypeEnum.Month);
}
if (twitterSearch.getTwitterSearchScheduler() != null) {
twitterMonitorScheduler.setActiveSearch(true);
twitterMonitorScheduler.setActive(true);
} else {
twitterMonitorScheduler.setActiveSearch(false);
if (Integer.parseInt(numberUpTo) <= 0) {
twitterMonitorScheduler.setActive(false);
} else {
twitterMonitorScheduler.setActive(true);
}
}
twitterMonitorScheduler.setAccounts(accounts);
twitterMonitorScheduler.setLinks(links);
twitterMonitorScheduler.setDocuments(documents);
twitterMonitorScheduler.setLastActivationTime(GregorianCalendar.getInstance());
twitterMonitorScheduler.setStartingTime(GregorianCalendar.getInstance());
twitterMonitorScheduler.setEndingTime(
this.setMonitorSchedulerEndingDate(twitterMonitorScheduler));
twitterMonitorScheduler.setTwitterSearch(twitterSearch);
}
}
// set monitor scheduler
twitterSearch.setTwitterMonitorScheduler(twitterMonitorScheduler);
String numberStartingFrom = req.getParameter("numberStartingFrom");
if (req.getParameter("isStartingFrom") != null
&& numberStartingFrom != null
&& !numberStartingFrom.equals("")) {
logger.debug("Method search(): Search with a starting date");
Calendar actualDate = GregorianCalendar.getInstance();
// Manage starting time with the user's input
Calendar sinceDate = actualDate;
sinceDate.add(Calendar.DAY_OF_MONTH, -Integer.parseInt(numberStartingFrom));
twitterSearch.setDaysAgo(Integer.parseInt(numberStartingFrom));
twitterLauncher = new TwitterAnalysisLauncher(twitterSearch);
twitterLauncher.setLanguageCode(languageCode);
twitterLauncher.setSinceCalendar(sinceDate);
} else {
logger.debug("Method search(): Search without dates (except API limits)");
twitterLauncher = new TwitterAnalysisLauncher(twitterSearch);
twitterLauncher.setLanguageCode(languageCode);
}
long searchID = twitterLauncher.createhistoricalSearch();
JSONObject resObj = new JSONObject();
if (searchID > 0) {
resObj.put("success", true);
resObj.put("msg", "Twitter Search \"" + label + "\" inserted. Loading results..");
} else {
resObj.put("failure", true);
resObj.put("msg", "Failure starting new search ");
}
logger.debug("Method search(): End");
return resObj.toString();
} catch (Throwable th) {
throw new SpagoBIRuntimeException(
"Method search(): An error occurred in Twitter Historical Search API for REST service search ",
th);
}
}