public static void closeConnection() { if (twitter != null) { Logger.info("Closing down Twitter stream..."); twitter.cleanUp(); if (esClient != null) esClient.close(); } else { Logger.info("No twitter stream found"); } }
public TweetListener(List<String> terms, Client esClient, String esIndex, String esType) { this.terms = terms; this.esClient = esClient; StringBuilder query = new StringBuilder(); for (int i = 0; i < terms.size(); i++) { if (i > 0) query.append("|"); // query.append("(").append(Pattern.quote(terms.get(i))).append(")"); // query.append("(").append(terms.get(i)).append(")"); query.append(terms.get(i)); } Logger.info("Query match pattern: " + query.toString()); this.matchPattern = Pattern.compile(query.toString(), Pattern.CASE_INSENSITIVE); Logger.info("Query match pattern: " + matchPattern.toString()); }
private static void startStream(List<String> terms) throws TwitterException { if (twitter != null) { twitter.cleanUp(); } if (esClient != null) { esClient.close(); esClient = null; } play.Configuration pconf = Play.application().configuration(); String elasticSearchCluster = pconf.getString("tweet.elasticsearch.cluster.name"); if (elasticSearchCluster != null) { Logger.info("Configuring ElasticSearch..."); Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", elasticSearchCluster).build(); esClient = new TransportClient(settings) .addTransportAddress( new InetSocketTransportAddress( pconf.getString("tweet.elasticsearch.transport.host"), pconf.getInt("tweet.elasticsearch.transport.port"))); } else { esClient = null; } twitter4j.conf.Configuration tconf = Application.getTwitterConfiguration(); TwitterStreamFactory tf = new TwitterStreamFactory(tconf); twitter = tf.getInstance(); StatusListener l = new TweetListener( terms, esClient, pconf.getString("tweet.elasticsearch.index"), pconf.getString("tweet.elasticsearch.type")); twitter.addListener(l); String[] tracks = new String[terms.size()]; StringBuffer termsString = new StringBuffer(); for (int i = 0; i < terms.size(); i++) { tracks[i] = terms.get(i); if (i != 0) termsString.append(","); termsString.append(terms.get(i)); } FilterQuery q = new FilterQuery().track(tracks); twitter.filter(q); Logger.info("Starting listening for tweets using terms " + termsString.toString() + "..."); }
public static Result start() { java.util.Map<String, String[]> map = request().body().asFormUrlEncoded(); List<String> terms = new ArrayList<>(map.size()); for (int i = 0; i < map.size(); i++) { String key = "terms[" + i + "]"; if (map.containsKey(key)) { String[] values = map.get(key); if ((values != null) && (values.length >= 1)) { terms.add(values[0]); } } } StreamConfig config = getConfig(); config.putTerms(terms); config.update(); StringBuilder sb = new StringBuilder(); for (String t : terms) { sb.append(t); sb.append(", "); } sb.delete(sb.length() - 2, sb.length()); try { startStream(terms); flash("success", "Twitter stream started (" + sb.toString() + ")"); } catch (TwitterException e) { Logger.info("Error starting twitter stream", e); flash("error", "Error starting Twitter stream" + e.getMessage()); } return redirect(routes.Streams.listAll()); }
public static void startConnection() { StreamConfig config = getConfig(); try { startStream(config.listTerms()); } catch (TwitterException e) { Logger.info("Error starting twitter stream", e); } }
public void onStatus(twitter4j.Status status) { Logger.info(status.getUser().getName() + " : " + status.getText()); Tweet tweet = new Tweet(status); tweet.conformsToTerms = checkMatch(status); tweet.save(); if (tweet.conformsToTerms && esClient != null) { String json = DataObjectFactory.getRawJSON(status); json = json.replaceAll( "(\"geo\":\\{\"type\":\"Point\",\"coordinates\":)\\[([-0-9.,]*)\\]", "$1\"$2\""); // Logger.debug("geo mangled json"); // Logger.debug(json); IndexResponse response = esClient.prepareIndex("twitter", "tweet").setSource(json).execute().actionGet(); } }
private static StreamConfig getConfig() { List<StreamConfig> configs = StreamConfig.find.findList(); if (configs.size() > 1) { Logger.error("Multiple stream configurations present!"); } StreamConfig config; if (configs.isEmpty()) { String[] tracks = new String[3]; tracks[0] = "nl-alert"; tracks[1] = "nlalert"; tracks[2] = "\"nl alert\""; config = new StreamConfig(tracks); config.save(); } else { config = configs.get(0); } return config; }
public static Result download() { Exporter exporter = new Exporter("/tmp/tweets.xlsx"); List<Tweet> tweets = Tweet.find.all(); try { for (Tweet t : tweets) { SimpleTweet simple = new SimpleTweet(); simple.id = t.id; simple.createdAt = t.date; simple.userName = t.fromUser; simple.userId = t.fromUserId; simple.text = t.text; simple.inReplyToName = t.inReplyTo; simple.latitude = t.latitude; simple.longitude = t.longitude; exporter.addTweet(simple); } exporter.write(); } catch (FileNotFoundException e) { flash("error", "Bestand niet gevonden"); Logger.info("Bestand niet gevonden", e); return ok("Exception opening file"); } // try { response().setContentType("application/x-download"); response().setHeader("Content-disposition", "attachment; filename=tweets.xlsx"); return ok(new File("/tmp/tweets.xlsx")); /* } catch (IOException e) { Logger.info("Exception sending zipfile", e); return ok("Exception opening file"); */ // } }
public void onException(java.lang.Exception ex) { // ex.printStackTrace(); Logger.warn("Exception in Twitter Stream", ex); }
public void onTrackLimitationNotice(int numberOfLimitedStatuses) { Logger.info("Track limitation, missed " + numberOfLimitedStatuses); }
public void onStallWarning(StallWarning warning) { Logger.info("Stall warning"); }
public void onScrubGeo(long userId, long upToStatusId) { Logger.info("Scrub geo"); }
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { Logger.info("Deletion notice"); }
protected boolean checkMatch(twitter4j.Status status) { boolean result = false; if (matchPattern.matcher(status.getText()).find()) result = true; if (result) { Logger.debug("Terms found in text"); Logger.debug(" \"" + status.getText() + "\""); return result; } for (URLEntity ue : status.getURLEntities()) { if (matchPattern.matcher(ue.getDisplayURL()).find()) result = true; if (matchPattern.matcher(ue.getExpandedURL()).find()) result = true; } if (result) { Logger.debug("Terms found in URL entities"); for (URLEntity ue : status.getURLEntities()) { Logger.debug(" " + ue.getDisplayURL()); Logger.debug(" " + ue.getExpandedURL()); } return result; } for (URLEntity ue : status.getMediaEntities()) { if (matchPattern.matcher(ue.getDisplayURL()).find()) result = true; if (matchPattern.matcher(ue.getExpandedURL()).find()) result = true; } if (result) { Logger.debug("Terms found in Media entities"); for (URLEntity ue : status.getMediaEntities()) { Logger.debug(" " + ue.getDisplayURL()); Logger.debug(" " + ue.getExpandedURL()); } return result; } for (HashtagEntity he : status.getHashtagEntities()) { if (matchPattern.matcher(he.getText()).find()) result = true; } if (result) { Logger.debug("Terms found in Hashtag entities"); for (HashtagEntity he : status.getHashtagEntities()) { Logger.debug(" " + he.getText()); } return result; } for (UserMentionEntity me : status.getUserMentionEntities()) { if (matchPattern.matcher(me.getScreenName()).find()) result = true; } if (result) { Logger.debug("Terms found in User mention entities"); for (UserMentionEntity me : status.getUserMentionEntities()) { Logger.debug(" " + me.getScreenName()); } return result; } Logger.debug("Terms NOT FOUND"); Logger.debug(" Terms not found in URL entities"); for (URLEntity ue : status.getURLEntities()) { Logger.debug(" " + ue.getDisplayURL()); Logger.debug(" " + ue.getExpandedURL()); } Logger.debug(" Terms not found in Media entities"); for (URLEntity ue : status.getMediaEntities()) { Logger.debug(" " + ue.getDisplayURL()); Logger.debug(" " + ue.getExpandedURL()); } Logger.debug(" Terms not found in Hashtag entities"); for (HashtagEntity he : status.getHashtagEntities()) { Logger.debug(" " + he.getText()); } Logger.debug(" Terms not found in User mention entities"); for (UserMentionEntity me : status.getUserMentionEntities()) { Logger.debug(" " + me.getScreenName()); } return result; }
public static Result index() { LOG.info("this is home:" + request().toString()); UUIDGenerator uuid = UUIDGenerator.findByUDID(request().username()); return ok(home.render(uuid.getUser().getUsername())); }