public static void closeConnection() {
if (twitter != null) {
Logger.info("Closing down Twitter stream...");
twitter.cleanUp();
if (esClient != null) esClient.close();
} else {
Logger.info("No twitter stream found");
}
}
public TweetListener(List<String> terms, Client esClient, String esIndex, String esType) {
this.terms = terms;
this.esClient = esClient;
StringBuilder query = new StringBuilder();
for (int i = 0; i < terms.size(); i++) {
if (i > 0) query.append("|");
// query.append("(").append(Pattern.quote(terms.get(i))).append(")");
// query.append("(").append(terms.get(i)).append(")");
query.append(terms.get(i));
}
Logger.info("Query match pattern: " + query.toString());
this.matchPattern = Pattern.compile(query.toString(), Pattern.CASE_INSENSITIVE);
Logger.info("Query match pattern: " + matchPattern.toString());
}
private static void startStream(List<String> terms) throws TwitterException {
if (twitter != null) {
twitter.cleanUp();
}
if (esClient != null) {
esClient.close();
esClient = null;
}
play.Configuration pconf = Play.application().configuration();
String elasticSearchCluster = pconf.getString("tweet.elasticsearch.cluster.name");
if (elasticSearchCluster != null) {
Logger.info("Configuring ElasticSearch...");
Settings settings =
ImmutableSettings.settingsBuilder().put("cluster.name", elasticSearchCluster).build();
esClient =
new TransportClient(settings)
.addTransportAddress(
new InetSocketTransportAddress(
pconf.getString("tweet.elasticsearch.transport.host"),
pconf.getInt("tweet.elasticsearch.transport.port")));
} else {
esClient = null;
}
twitter4j.conf.Configuration tconf = Application.getTwitterConfiguration();
TwitterStreamFactory tf = new TwitterStreamFactory(tconf);
twitter = tf.getInstance();
StatusListener l =
new TweetListener(
terms,
esClient,
pconf.getString("tweet.elasticsearch.index"),
pconf.getString("tweet.elasticsearch.type"));
twitter.addListener(l);
String[] tracks = new String[terms.size()];
StringBuffer termsString = new StringBuffer();
for (int i = 0; i < terms.size(); i++) {
tracks[i] = terms.get(i);
if (i != 0) termsString.append(",");
termsString.append(terms.get(i));
}
FilterQuery q = new FilterQuery().track(tracks);
twitter.filter(q);
Logger.info("Starting listening for tweets using terms " + termsString.toString() + "...");
}
public static Result start() {
java.util.Map<String, String[]> map = request().body().asFormUrlEncoded();
List<String> terms = new ArrayList<>(map.size());
for (int i = 0; i < map.size(); i++) {
String key = "terms[" + i + "]";
if (map.containsKey(key)) {
String[] values = map.get(key);
if ((values != null) && (values.length >= 1)) {
terms.add(values[0]);
}
}
}
StreamConfig config = getConfig();
config.putTerms(terms);
config.update();
StringBuilder sb = new StringBuilder();
for (String t : terms) {
sb.append(t);
sb.append(", ");
}
sb.delete(sb.length() - 2, sb.length());
try {
startStream(terms);
flash("success", "Twitter stream started (" + sb.toString() + ")");
} catch (TwitterException e) {
Logger.info("Error starting twitter stream", e);
flash("error", "Error starting Twitter stream" + e.getMessage());
}
return redirect(routes.Streams.listAll());
}
public static void startConnection() {
StreamConfig config = getConfig();
try {
startStream(config.listTerms());
} catch (TwitterException e) {
Logger.info("Error starting twitter stream", e);
}
}
public void onStatus(twitter4j.Status status) {
Logger.info(status.getUser().getName() + " : " + status.getText());
Tweet tweet = new Tweet(status);
tweet.conformsToTerms = checkMatch(status);
tweet.save();
if (tweet.conformsToTerms && esClient != null) {
String json = DataObjectFactory.getRawJSON(status);
json =
json.replaceAll(
"(\"geo\":\\{\"type\":\"Point\",\"coordinates\":)\\[([-0-9.,]*)\\]", "$1\"$2\"");
// Logger.debug("geo mangled json");
// Logger.debug(json);
IndexResponse response =
esClient.prepareIndex("twitter", "tweet").setSource(json).execute().actionGet();
}
}
private static StreamConfig getConfig() {
List<StreamConfig> configs = StreamConfig.find.findList();
if (configs.size() > 1) {
Logger.error("Multiple stream configurations present!");
}
StreamConfig config;
if (configs.isEmpty()) {
String[] tracks = new String[3];
tracks[0] = "nl-alert";
tracks[1] = "nlalert";
tracks[2] = "\"nl alert\"";
config = new StreamConfig(tracks);
config.save();
} else {
config = configs.get(0);
}
return config;
}
public static Result download() {
Exporter exporter = new Exporter("/tmp/tweets.xlsx");
List<Tweet> tweets = Tweet.find.all();
try {
for (Tweet t : tweets) {
SimpleTweet simple = new SimpleTweet();
simple.id = t.id;
simple.createdAt = t.date;
simple.userName = t.fromUser;
simple.userId = t.fromUserId;
simple.text = t.text;
simple.inReplyToName = t.inReplyTo;
simple.latitude = t.latitude;
simple.longitude = t.longitude;
exporter.addTweet(simple);
}
exporter.write();
} catch (FileNotFoundException e) {
flash("error", "Bestand niet gevonden");
Logger.info("Bestand niet gevonden", e);
return ok("Exception opening file");
}
// try {
response().setContentType("application/x-download");
response().setHeader("Content-disposition", "attachment; filename=tweets.xlsx");
return ok(new File("/tmp/tweets.xlsx"));
/*
} catch (IOException e) {
Logger.info("Exception sending zipfile", e);
return ok("Exception opening file");
*/
// }
}
public void onException(java.lang.Exception ex) {
// ex.printStackTrace();
Logger.warn("Exception in Twitter Stream", ex);
}
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
Logger.info("Track limitation, missed " + numberOfLimitedStatuses);
}
public void onStallWarning(StallWarning warning) {
Logger.info("Stall warning");
}
public void onScrubGeo(long userId, long upToStatusId) {
Logger.info("Scrub geo");
}
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
Logger.info("Deletion notice");
}
protected boolean checkMatch(twitter4j.Status status) {
boolean result = false;
if (matchPattern.matcher(status.getText()).find()) result = true;
if (result) {
Logger.debug("Terms found in text");
Logger.debug(" \"" + status.getText() + "\"");
return result;
}
for (URLEntity ue : status.getURLEntities()) {
if (matchPattern.matcher(ue.getDisplayURL()).find()) result = true;
if (matchPattern.matcher(ue.getExpandedURL()).find()) result = true;
}
if (result) {
Logger.debug("Terms found in URL entities");
for (URLEntity ue : status.getURLEntities()) {
Logger.debug(" " + ue.getDisplayURL());
Logger.debug(" " + ue.getExpandedURL());
}
return result;
}
for (URLEntity ue : status.getMediaEntities()) {
if (matchPattern.matcher(ue.getDisplayURL()).find()) result = true;
if (matchPattern.matcher(ue.getExpandedURL()).find()) result = true;
}
if (result) {
Logger.debug("Terms found in Media entities");
for (URLEntity ue : status.getMediaEntities()) {
Logger.debug(" " + ue.getDisplayURL());
Logger.debug(" " + ue.getExpandedURL());
}
return result;
}
for (HashtagEntity he : status.getHashtagEntities()) {
if (matchPattern.matcher(he.getText()).find()) result = true;
}
if (result) {
Logger.debug("Terms found in Hashtag entities");
for (HashtagEntity he : status.getHashtagEntities()) {
Logger.debug(" " + he.getText());
}
return result;
}
for (UserMentionEntity me : status.getUserMentionEntities()) {
if (matchPattern.matcher(me.getScreenName()).find()) result = true;
}
if (result) {
Logger.debug("Terms found in User mention entities");
for (UserMentionEntity me : status.getUserMentionEntities()) {
Logger.debug(" " + me.getScreenName());
}
return result;
}
Logger.debug("Terms NOT FOUND");
Logger.debug(" Terms not found in URL entities");
for (URLEntity ue : status.getURLEntities()) {
Logger.debug(" " + ue.getDisplayURL());
Logger.debug(" " + ue.getExpandedURL());
}
Logger.debug(" Terms not found in Media entities");
for (URLEntity ue : status.getMediaEntities()) {
Logger.debug(" " + ue.getDisplayURL());
Logger.debug(" " + ue.getExpandedURL());
}
Logger.debug(" Terms not found in Hashtag entities");
for (HashtagEntity he : status.getHashtagEntities()) {
Logger.debug(" " + he.getText());
}
Logger.debug(" Terms not found in User mention entities");
for (UserMentionEntity me : status.getUserMentionEntities()) {
Logger.debug(" " + me.getScreenName());
}
return result;
}
public static Result index() {
LOG.info("this is home:" + request().toString());
UUIDGenerator uuid = UUIDGenerator.findByUDID(request().username());
return ok(home.render(uuid.getUser().getUsername()));
}