Пример #1
0
 public static void closeConnection() {
   if (twitter != null) {
     Logger.info("Closing down Twitter stream...");
     twitter.cleanUp();
     if (esClient != null) esClient.close();
   } else {
     Logger.info("No twitter stream found");
   }
 }
Пример #2
0
 public TweetListener(List<String> terms, Client esClient, String esIndex, String esType) {
   this.terms = terms;
   this.esClient = esClient;
   StringBuilder query = new StringBuilder();
   for (int i = 0; i < terms.size(); i++) {
     if (i > 0) query.append("|");
     // query.append("(").append(Pattern.quote(terms.get(i))).append(")");
     // query.append("(").append(terms.get(i)).append(")");
     query.append(terms.get(i));
   }
   Logger.info("Query match pattern: " + query.toString());
   this.matchPattern = Pattern.compile(query.toString(), Pattern.CASE_INSENSITIVE);
   Logger.info("Query match pattern: " + matchPattern.toString());
 }
Пример #3
0
  private static void startStream(List<String> terms) throws TwitterException {
    if (twitter != null) {
      twitter.cleanUp();
    }
    if (esClient != null) {
      esClient.close();
      esClient = null;
    }

    play.Configuration pconf = Play.application().configuration();
    String elasticSearchCluster = pconf.getString("tweet.elasticsearch.cluster.name");
    if (elasticSearchCluster != null) {
      Logger.info("Configuring ElasticSearch...");
      Settings settings =
          ImmutableSettings.settingsBuilder().put("cluster.name", elasticSearchCluster).build();

      esClient =
          new TransportClient(settings)
              .addTransportAddress(
                  new InetSocketTransportAddress(
                      pconf.getString("tweet.elasticsearch.transport.host"),
                      pconf.getInt("tweet.elasticsearch.transport.port")));
    } else {
      esClient = null;
    }

    twitter4j.conf.Configuration tconf = Application.getTwitterConfiguration();
    TwitterStreamFactory tf = new TwitterStreamFactory(tconf);
    twitter = tf.getInstance();
    StatusListener l =
        new TweetListener(
            terms,
            esClient,
            pconf.getString("tweet.elasticsearch.index"),
            pconf.getString("tweet.elasticsearch.type"));
    twitter.addListener(l);

    String[] tracks = new String[terms.size()];
    StringBuffer termsString = new StringBuffer();
    for (int i = 0; i < terms.size(); i++) {
      tracks[i] = terms.get(i);
      if (i != 0) termsString.append(",");
      termsString.append(terms.get(i));
    }
    FilterQuery q = new FilterQuery().track(tracks);
    twitter.filter(q);
    Logger.info("Starting listening for tweets using terms " + termsString.toString() + "...");
  }
Пример #4
0
  public static Result start() {
    java.util.Map<String, String[]> map = request().body().asFormUrlEncoded();

    List<String> terms = new ArrayList<>(map.size());
    for (int i = 0; i < map.size(); i++) {
      String key = "terms[" + i + "]";
      if (map.containsKey(key)) {
        String[] values = map.get(key);
        if ((values != null) && (values.length >= 1)) {
          terms.add(values[0]);
        }
      }
    }

    StreamConfig config = getConfig();
    config.putTerms(terms);
    config.update();

    StringBuilder sb = new StringBuilder();
    for (String t : terms) {
      sb.append(t);
      sb.append(", ");
    }
    sb.delete(sb.length() - 2, sb.length());

    try {
      startStream(terms);
      flash("success", "Twitter stream started (" + sb.toString() + ")");
    } catch (TwitterException e) {
      Logger.info("Error starting twitter stream", e);
      flash("error", "Error starting Twitter stream" + e.getMessage());
    }
    return redirect(routes.Streams.listAll());
  }
Пример #5
0
 public static void startConnection() {
   StreamConfig config = getConfig();
   try {
     startStream(config.listTerms());
   } catch (TwitterException e) {
     Logger.info("Error starting twitter stream", e);
   }
 }
Пример #6
0
  /**
   * Validates fields from the registration form and either creates a new user or communicates any
   * validation errors.
   */
  public static Result submit() {
    Form<User> filledForm = signupForm.bindFromRequest();

    // Check accept conditions
    if (!"true".equals(filledForm.field("accept").value())) {
      filledForm.reject("accept", "You must accept the terms and conditions");
    }

    // Check repeated password
    if (!filledForm.field("password").valueOr("").isEmpty()) {
      if (!filledForm
          .field("password")
          .valueOr("")
          .equals(filledForm.field("repeatPassword").value())) {
        filledForm.reject("repeatPassword", "Passwords do not match");
      }
    }

    // Check if the username and email are valid
    if (!filledForm.hasErrors()) {

      String un = filledForm.get().username;
      String email = filledForm.get().email;

      if (un.equals("admin") || un.equals("guest")) {
        filledForm.reject("username", "This username is already taken");
      }

      try {
        Logger.debug("Finding user " + email);
        User.findByEmail(email);
        filledForm.reject(
            "email", "There is already an account associated with this email address.");
      } catch (Exception e) {
        // continue - the user does not exist
      }
    }

    // Return validation results to user or save user
    if (filledForm.hasErrors()) {
      return badRequest(form.render(filledForm));
    } else {
      User user = filledForm.get(); /* create an object from a form */
      User svUser =
          new User(user.username, user.email, user.password); /* recreate to get save group info */
      svUser.save();
      return ok(summary.render(svUser));
    }
  }
Пример #7
0
 public void onStatus(twitter4j.Status status) {
   Logger.info(status.getUser().getName() + " : " + status.getText());
   Tweet tweet = new Tweet(status);
   tweet.conformsToTerms = checkMatch(status);
   tweet.save();
   if (tweet.conformsToTerms && esClient != null) {
     String json = DataObjectFactory.getRawJSON(status);
     json =
         json.replaceAll(
             "(\"geo\":\\{\"type\":\"Point\",\"coordinates\":)\\[([-0-9.,]*)\\]", "$1\"$2\"");
     // Logger.debug("geo mangled json");
     // Logger.debug(json);
     IndexResponse response =
         esClient.prepareIndex("twitter", "tweet").setSource(json).execute().actionGet();
   }
 }
Пример #8
0
 private static StreamConfig getConfig() {
   List<StreamConfig> configs = StreamConfig.find.findList();
   if (configs.size() > 1) {
     Logger.error("Multiple stream configurations present!");
   }
   StreamConfig config;
   if (configs.isEmpty()) {
     String[] tracks = new String[3];
     tracks[0] = "nl-alert";
     tracks[1] = "nlalert";
     tracks[2] = "\"nl alert\"";
     config = new StreamConfig(tracks);
     config.save();
   } else {
     config = configs.get(0);
   }
   return config;
 }
Пример #9
0
  public F.Promise<Result> call(Http.Context ctx) throws Throwable {
    try {
      return delegate.call(ctx);
    } catch (Exception e) {
      e.printStackTrace();
      StringBuilder sb = new StringBuilder();

      sb.append("Error for request at " + ctx.request().uri() + "\n");
      sb.append("Headers: \n");
      Map<String, String[]> headers = ctx.request().headers();
      for (String key : headers.keySet()) {
        sb.append("  " + key + " --> ");
        for (String val : headers.get(key)) {
          sb.append(val + "|||");
        }
        sb.append("\n");
      }

      sb.append("Cookies: \n");
      for (Http.Cookie cookie : ctx.request().cookies()) {
        sb.append("  " + cookie.name() + " --> " + cookie.value() + "\n");
      }

      Http.RequestBody body = ctx.request().body();
      Map<String, String[]> body_vals = body.asFormUrlEncoded();
      if (body_vals != null) {
        sb.append("Body (as form URL encoded): \n");
        for (String key : body_vals.keySet()) {
          sb.append("  " + key + " --> ");
          for (String val : body_vals.get(key)) {
            sb.append(val + "|||");
          }
          sb.append("\n");
        }
      }

      Logger.error(sb.toString());
      throw e;
    }
  }
Пример #10
0
  public static Result download() {
    Exporter exporter = new Exporter("/tmp/tweets.xlsx");

    List<Tweet> tweets = Tweet.find.all();

    try {
      for (Tweet t : tweets) {
        SimpleTweet simple = new SimpleTweet();
        simple.id = t.id;
        simple.createdAt = t.date;
        simple.userName = t.fromUser;
        simple.userId = t.fromUserId;
        simple.text = t.text;
        simple.inReplyToName = t.inReplyTo;
        simple.latitude = t.latitude;
        simple.longitude = t.longitude;

        exporter.addTweet(simple);
      }

      exporter.write();
    } catch (FileNotFoundException e) {
      flash("error", "Bestand niet gevonden");
      Logger.info("Bestand niet gevonden", e);
      return ok("Exception opening file");
    }

    //      try {
    response().setContentType("application/x-download");
    response().setHeader("Content-disposition", "attachment; filename=tweets.xlsx");
    return ok(new File("/tmp/tweets.xlsx"));
    /*
      } catch (IOException e) {
      Logger.info("Exception sending zipfile", e);
      return ok("Exception opening file");
    */
    //      }
  }
Пример #11
0
 public void onException(java.lang.Exception ex) {
   // ex.printStackTrace();
   Logger.warn("Exception in Twitter Stream", ex);
 }
Пример #12
0
 public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
   Logger.info("Track limitation, missed " + numberOfLimitedStatuses);
 }
Пример #13
0
 public void onStallWarning(StallWarning warning) {
   Logger.info("Stall warning");
 }
Пример #14
0
 public void onScrubGeo(long userId, long upToStatusId) {
   Logger.info("Scrub geo");
 }
Пример #15
0
 public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
   Logger.info("Deletion notice");
 }
Пример #16
0
 protected boolean checkMatch(twitter4j.Status status) {
   boolean result = false;
   if (matchPattern.matcher(status.getText()).find()) result = true;
   if (result) {
     Logger.debug("Terms found in text");
     Logger.debug("    \"" + status.getText() + "\"");
     return result;
   }
   for (URLEntity ue : status.getURLEntities()) {
     if (matchPattern.matcher(ue.getDisplayURL()).find()) result = true;
     if (matchPattern.matcher(ue.getExpandedURL()).find()) result = true;
   }
   if (result) {
     Logger.debug("Terms found in URL entities");
     for (URLEntity ue : status.getURLEntities()) {
       Logger.debug("    " + ue.getDisplayURL());
       Logger.debug("    " + ue.getExpandedURL());
     }
     return result;
   }
   for (URLEntity ue : status.getMediaEntities()) {
     if (matchPattern.matcher(ue.getDisplayURL()).find()) result = true;
     if (matchPattern.matcher(ue.getExpandedURL()).find()) result = true;
   }
   if (result) {
     Logger.debug("Terms found in Media entities");
     for (URLEntity ue : status.getMediaEntities()) {
       Logger.debug("    " + ue.getDisplayURL());
       Logger.debug("    " + ue.getExpandedURL());
     }
     return result;
   }
   for (HashtagEntity he : status.getHashtagEntities()) {
     if (matchPattern.matcher(he.getText()).find()) result = true;
   }
   if (result) {
     Logger.debug("Terms found in Hashtag entities");
     for (HashtagEntity he : status.getHashtagEntities()) {
       Logger.debug("    " + he.getText());
     }
     return result;
   }
   for (UserMentionEntity me : status.getUserMentionEntities()) {
     if (matchPattern.matcher(me.getScreenName()).find()) result = true;
   }
   if (result) {
     Logger.debug("Terms found in User mention entities");
     for (UserMentionEntity me : status.getUserMentionEntities()) {
       Logger.debug("    " + me.getScreenName());
     }
     return result;
   }
   Logger.debug("Terms NOT FOUND");
   Logger.debug("    Terms not found in URL entities");
   for (URLEntity ue : status.getURLEntities()) {
     Logger.debug("    " + ue.getDisplayURL());
     Logger.debug("    " + ue.getExpandedURL());
   }
   Logger.debug("    Terms not found in Media entities");
   for (URLEntity ue : status.getMediaEntities()) {
     Logger.debug("    " + ue.getDisplayURL());
     Logger.debug("    " + ue.getExpandedURL());
   }
   Logger.debug("    Terms not found in Hashtag entities");
   for (HashtagEntity he : status.getHashtagEntities()) {
     Logger.debug("    " + he.getText());
   }
   Logger.debug("    Terms not found in User mention entities");
   for (UserMentionEntity me : status.getUserMentionEntities()) {
     Logger.debug("    " + me.getScreenName());
   }
   return result;
 }