public static void closeConnection() {
if (twitter != null) {
Logger.info("Closing down Twitter stream...");
twitter.cleanUp();
if (esClient != null) esClient.close();
} else {
Logger.info("No twitter stream found");
}
}
public TweetListener(List<String> terms, Client esClient, String esIndex, String esType) {
this.terms = terms;
this.esClient = esClient;
StringBuilder query = new StringBuilder();
for (int i = 0; i < terms.size(); i++) {
if (i > 0) query.append("|");
// query.append("(").append(Pattern.quote(terms.get(i))).append(")");
// query.append("(").append(terms.get(i)).append(")");
query.append(terms.get(i));
}
Logger.info("Query match pattern: " + query.toString());
this.matchPattern = Pattern.compile(query.toString(), Pattern.CASE_INSENSITIVE);
Logger.info("Query match pattern: " + matchPattern.toString());
}
private static void startStream(List<String> terms) throws TwitterException {
if (twitter != null) {
twitter.cleanUp();
}
if (esClient != null) {
esClient.close();
esClient = null;
}
play.Configuration pconf = Play.application().configuration();
String elasticSearchCluster = pconf.getString("tweet.elasticsearch.cluster.name");
if (elasticSearchCluster != null) {
Logger.info("Configuring ElasticSearch...");
Settings settings =
ImmutableSettings.settingsBuilder().put("cluster.name", elasticSearchCluster).build();
esClient =
new TransportClient(settings)
.addTransportAddress(
new InetSocketTransportAddress(
pconf.getString("tweet.elasticsearch.transport.host"),
pconf.getInt("tweet.elasticsearch.transport.port")));
} else {
esClient = null;
}
twitter4j.conf.Configuration tconf = Application.getTwitterConfiguration();
TwitterStreamFactory tf = new TwitterStreamFactory(tconf);
twitter = tf.getInstance();
StatusListener l =
new TweetListener(
terms,
esClient,
pconf.getString("tweet.elasticsearch.index"),
pconf.getString("tweet.elasticsearch.type"));
twitter.addListener(l);
String[] tracks = new String[terms.size()];
StringBuffer termsString = new StringBuffer();
for (int i = 0; i < terms.size(); i++) {
tracks[i] = terms.get(i);
if (i != 0) termsString.append(",");
termsString.append(terms.get(i));
}
FilterQuery q = new FilterQuery().track(tracks);
twitter.filter(q);
Logger.info("Starting listening for tweets using terms " + termsString.toString() + "...");
}
public static Result start() {
java.util.Map<String, String[]> map = request().body().asFormUrlEncoded();
List<String> terms = new ArrayList<>(map.size());
for (int i = 0; i < map.size(); i++) {
String key = "terms[" + i + "]";
if (map.containsKey(key)) {
String[] values = map.get(key);
if ((values != null) && (values.length >= 1)) {
terms.add(values[0]);
}
}
}
StreamConfig config = getConfig();
config.putTerms(terms);
config.update();
StringBuilder sb = new StringBuilder();
for (String t : terms) {
sb.append(t);
sb.append(", ");
}
sb.delete(sb.length() - 2, sb.length());
try {
startStream(terms);
flash("success", "Twitter stream started (" + sb.toString() + ")");
} catch (TwitterException e) {
Logger.info("Error starting twitter stream", e);
flash("error", "Error starting Twitter stream" + e.getMessage());
}
return redirect(routes.Streams.listAll());
}
public static void startConnection() {
StreamConfig config = getConfig();
try {
startStream(config.listTerms());
} catch (TwitterException e) {
Logger.info("Error starting twitter stream", e);
}
}
/**
* Validates fields from the registration form and either creates a new user or communicates any
* validation errors.
*/
public static Result submit() {
Form<User> filledForm = signupForm.bindFromRequest();
// Check accept conditions
if (!"true".equals(filledForm.field("accept").value())) {
filledForm.reject("accept", "You must accept the terms and conditions");
}
// Check repeated password
if (!filledForm.field("password").valueOr("").isEmpty()) {
if (!filledForm
.field("password")
.valueOr("")
.equals(filledForm.field("repeatPassword").value())) {
filledForm.reject("repeatPassword", "Passwords do not match");
}
}
// Check if the username and email are valid
if (!filledForm.hasErrors()) {
String un = filledForm.get().username;
String email = filledForm.get().email;
if (un.equals("admin") || un.equals("guest")) {
filledForm.reject("username", "This username is already taken");
}
try {
Logger.debug("Finding user " + email);
User.findByEmail(email);
filledForm.reject(
"email", "There is already an account associated with this email address.");
} catch (Exception e) {
// continue - the user does not exist
}
}
// Return validation results to user or save user
if (filledForm.hasErrors()) {
return badRequest(form.render(filledForm));
} else {
User user = filledForm.get(); /* create an object from a form */
User svUser =
new User(user.username, user.email, user.password); /* recreate to get save group info */
svUser.save();
return ok(summary.render(svUser));
}
}
public void onStatus(twitter4j.Status status) {
Logger.info(status.getUser().getName() + " : " + status.getText());
Tweet tweet = new Tweet(status);
tweet.conformsToTerms = checkMatch(status);
tweet.save();
if (tweet.conformsToTerms && esClient != null) {
String json = DataObjectFactory.getRawJSON(status);
json =
json.replaceAll(
"(\"geo\":\\{\"type\":\"Point\",\"coordinates\":)\\[([-0-9.,]*)\\]", "$1\"$2\"");
// Logger.debug("geo mangled json");
// Logger.debug(json);
IndexResponse response =
esClient.prepareIndex("twitter", "tweet").setSource(json).execute().actionGet();
}
}
private static StreamConfig getConfig() {
List<StreamConfig> configs = StreamConfig.find.findList();
if (configs.size() > 1) {
Logger.error("Multiple stream configurations present!");
}
StreamConfig config;
if (configs.isEmpty()) {
String[] tracks = new String[3];
tracks[0] = "nl-alert";
tracks[1] = "nlalert";
tracks[2] = "\"nl alert\"";
config = new StreamConfig(tracks);
config.save();
} else {
config = configs.get(0);
}
return config;
}
public F.Promise<Result> call(Http.Context ctx) throws Throwable {
try {
return delegate.call(ctx);
} catch (Exception e) {
e.printStackTrace();
StringBuilder sb = new StringBuilder();
sb.append("Error for request at " + ctx.request().uri() + "\n");
sb.append("Headers: \n");
Map<String, String[]> headers = ctx.request().headers();
for (String key : headers.keySet()) {
sb.append(" " + key + " --> ");
for (String val : headers.get(key)) {
sb.append(val + "|||");
}
sb.append("\n");
}
sb.append("Cookies: \n");
for (Http.Cookie cookie : ctx.request().cookies()) {
sb.append(" " + cookie.name() + " --> " + cookie.value() + "\n");
}
Http.RequestBody body = ctx.request().body();
Map<String, String[]> body_vals = body.asFormUrlEncoded();
if (body_vals != null) {
sb.append("Body (as form URL encoded): \n");
for (String key : body_vals.keySet()) {
sb.append(" " + key + " --> ");
for (String val : body_vals.get(key)) {
sb.append(val + "|||");
}
sb.append("\n");
}
}
Logger.error(sb.toString());
throw e;
}
}
public static Result download() {
Exporter exporter = new Exporter("/tmp/tweets.xlsx");
List<Tweet> tweets = Tweet.find.all();
try {
for (Tweet t : tweets) {
SimpleTweet simple = new SimpleTweet();
simple.id = t.id;
simple.createdAt = t.date;
simple.userName = t.fromUser;
simple.userId = t.fromUserId;
simple.text = t.text;
simple.inReplyToName = t.inReplyTo;
simple.latitude = t.latitude;
simple.longitude = t.longitude;
exporter.addTweet(simple);
}
exporter.write();
} catch (FileNotFoundException e) {
flash("error", "Bestand niet gevonden");
Logger.info("Bestand niet gevonden", e);
return ok("Exception opening file");
}
// try {
response().setContentType("application/x-download");
response().setHeader("Content-disposition", "attachment; filename=tweets.xlsx");
return ok(new File("/tmp/tweets.xlsx"));
/*
} catch (IOException e) {
Logger.info("Exception sending zipfile", e);
return ok("Exception opening file");
*/
// }
}
public void onException(java.lang.Exception ex) {
// ex.printStackTrace();
Logger.warn("Exception in Twitter Stream", ex);
}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
Logger.info("Track limitation, missed " + numberOfLimitedStatuses);
}
public void onStallWarning(StallWarning warning) {
Logger.info("Stall warning");
}
public void onScrubGeo(long userId, long upToStatusId) {
Logger.info("Scrub geo");
}
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
Logger.info("Deletion notice");
}
protected boolean checkMatch(twitter4j.Status status) {
boolean result = false;
if (matchPattern.matcher(status.getText()).find()) result = true;
if (result) {
Logger.debug("Terms found in text");
Logger.debug(" \"" + status.getText() + "\"");
return result;
}
for (URLEntity ue : status.getURLEntities()) {
if (matchPattern.matcher(ue.getDisplayURL()).find()) result = true;
if (matchPattern.matcher(ue.getExpandedURL()).find()) result = true;
}
if (result) {
Logger.debug("Terms found in URL entities");
for (URLEntity ue : status.getURLEntities()) {
Logger.debug(" " + ue.getDisplayURL());
Logger.debug(" " + ue.getExpandedURL());
}
return result;
}
for (URLEntity ue : status.getMediaEntities()) {
if (matchPattern.matcher(ue.getDisplayURL()).find()) result = true;
if (matchPattern.matcher(ue.getExpandedURL()).find()) result = true;
}
if (result) {
Logger.debug("Terms found in Media entities");
for (URLEntity ue : status.getMediaEntities()) {
Logger.debug(" " + ue.getDisplayURL());
Logger.debug(" " + ue.getExpandedURL());
}
return result;
}
for (HashtagEntity he : status.getHashtagEntities()) {
if (matchPattern.matcher(he.getText()).find()) result = true;
}
if (result) {
Logger.debug("Terms found in Hashtag entities");
for (HashtagEntity he : status.getHashtagEntities()) {
Logger.debug(" " + he.getText());
}
return result;
}
for (UserMentionEntity me : status.getUserMentionEntities()) {
if (matchPattern.matcher(me.getScreenName()).find()) result = true;
}
if (result) {
Logger.debug("Terms found in User mention entities");
for (UserMentionEntity me : status.getUserMentionEntities()) {
Logger.debug(" " + me.getScreenName());
}
return result;
}
Logger.debug("Terms NOT FOUND");
Logger.debug(" Terms not found in URL entities");
for (URLEntity ue : status.getURLEntities()) {
Logger.debug(" " + ue.getDisplayURL());
Logger.debug(" " + ue.getExpandedURL());
}
Logger.debug(" Terms not found in Media entities");
for (URLEntity ue : status.getMediaEntities()) {
Logger.debug(" " + ue.getDisplayURL());
Logger.debug(" " + ue.getExpandedURL());
}
Logger.debug(" Terms not found in Hashtag entities");
for (HashtagEntity he : status.getHashtagEntities()) {
Logger.debug(" " + he.getText());
}
Logger.debug(" Terms not found in User mention entities");
for (UserMentionEntity me : status.getUserMentionEntities()) {
Logger.debug(" " + me.getScreenName());
}
return result;
}
