public static void main(String[] args) throws Exception {

    Context context = new Context("conf/producer.conf");
    Tweety tp = new Tweety();

    try {
      Class.forName("com.mysql.jdbc.Driver");
      String url = "jdbc:mysql://172.20.95.137/mongo_test?" + "user=manoj&password=qwerty1!";
      Connection conn = DriverManager.getConnection(url);
      System.out.println(url);
      Statement stmt = conn.createStatement();

      ConfigurationBuilder cb = new ConfigurationBuilder();
      cb.setOAuthConsumerKey("ujS6V0corQ4v8K4dp6bIlVYsq");
      cb.setOAuthConsumerSecret("XhmTLcS4BI6zy9fZ0xIOy1ZZkAnQYZ1jFrlHojWuWpx2RCWH9H");
      cb.setOAuthAccessToken("30427118-YNLB3WfviD4j7MvNgU4gZ7SPQbCDBXWh4kPZcHXrY");
      cb.setOAuthAccessTokenSecret("LkhcZUJbj33RBc1DTXrxJps4ELgog8WxoRS6gCReemOHc");
      cb.setJSONStoreEnabled(true);
      cb.setIncludeEntitiesEnabled(true);
      cb.setHttpProxyHost("proxy.tcs.com");
      cb.setHttpProxyPort(8080);
      cb.setHttpProxyUser("467098");
      cb.setHttpProxyPassword("See@0615");
      Twitter twitter = new TwitterFactory(cb.build()).getInstance();
      ResponseList<Location> locations;
      locations = twitter.getAvailableTrends();
      System.out.println("Showing available trends");
      Trends trends = twitter.getPlaceTrends(2295386);
      stmt.executeUpdate("drop table hashtag");
      stmt.executeUpdate("create table hashtag(name varchar(50),count int)");
      for (int i = 0; i <= 5; i++) {
        System.out.println(trends.getTrends()[i].getName());
        hashtags[i] = trends.getTrends()[i].getName();
        stmt.executeUpdate("insert into hashtag values (\'" + hashtags[i] + "\'" + "," + 0 + ")");
      }
      System.out.println("done.");

    } catch (Exception te) {
      te.printStackTrace();
      System.out.println("Failed to get trends: " + te.getMessage());
      System.exit(-1);
    }
    tp.start(context);
  }
  /**
   * The initialization method for the Source. The context contains all the Flume configuration
   * info, and can be used to retrieve any configuration values necessary to set up the Source.
   */
  @Override
  public void configure(Context context) {
    consumerKey = context.getString(TwitterSourceConstants.CONSUMER_KEY_KEY);
    consumerSecret = context.getString(TwitterSourceConstants.CONSUMER_SECRET_KEY);
    accessToken = context.getString(TwitterSourceConstants.ACCESS_TOKEN_KEY);
    accessTokenSecret = context.getString(TwitterSourceConstants.ACCESS_TOKEN_SECRET_KEY);

    String keywordString = context.getString(TwitterSourceConstants.KEYWORDS_KEY, "");
    keywords = keywordString.split(",");
    for (int i = 0; i < keywords.length; i++) {
      keywords[i] = keywords[i].trim();
    }

    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setOAuthConsumerKey(consumerKey);
    cb.setOAuthConsumerSecret(consumerSecret);
    cb.setOAuthAccessToken(accessToken);
    cb.setOAuthAccessTokenSecret(accessTokenSecret);
    cb.setJSONStoreEnabled(true);
    cb.setIncludeEntitiesEnabled(true);

    twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
  }
Exemplo n.º 3
0
  private void start(final Context context) {

    /** Producer properties * */
    Properties props = new Properties();
    props.put("metadata.broker.list", context.getString(BROKER_LIST));
    props.put("serializer.class", context.getString(SERIALIZER));
    props.put("request.required.acks", context.getString(REQUIRED_ACKS));

    ProducerConfig config = new ProducerConfig(props);
    final Producer<String, String> producer = new Producer<String, String>(config);

    /** Twitter properties * */
    consumerKey = context.getString(CONSUMER_KEY_KEY);
    consumerSecret = context.getString(CONSUMER_SECRET_KEY);
    accessToken = context.getString(ACCESS_TOKEN_KEY);
    accessTokenSecret = context.getString(ACCESS_TOKEN_SECRET_KEY);

    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setOAuthConsumerKey(consumerKey);
    cb.setOAuthConsumerSecret(consumerSecret);
    cb.setOAuthAccessToken(accessToken);
    cb.setOAuthAccessTokenSecret(accessTokenSecret);
    cb.setJSONStoreEnabled(true);
    cb.setIncludeEntitiesEnabled(true);

    twitterStream = new TwitterStreamFactory(cb.build()).getInstance();
    final boolean shouldPrintTweetsOnScreen =
        Boolean.parseBoolean(context.getString(printTweetsOnScreen));
    final boolean shouldSendTweetsToKafka =
        Boolean.parseBoolean(context.getString(sendTweetsToKafka));

    final StatusListener listener =
        new StatusListener() {
          // The onStatus method is executed every time a new tweet comes
          // in.
          public void onStatus(Status status) {
            // The EventBuilder is used to build an event using the
            // the raw JSON of a tweet
            if (shouldPrintTweetsOnScreen) {
              logger.info(status.getUser().getScreenName() + ": " + status.getText());
            }

            if (shouldSendTweetsToKafka) {
              KeyedMessage<String, String> data =
                  new KeyedMessage<String, String>(
                      context.getString(KAFKA_TOPIC), TwitterObjectFactory.getRawJSON(status));
              producer.send(data);
            }
          }

          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          public void onScrubGeo(long userId, long upToStatusId) {}

          public void onException(Exception ex) {
            logger.info("Shutting down Twitter sample stream...");
            twitterStream.shutdown();
          }

          public void onStallWarning(StallWarning warning) {}
        };

    twitterStream.addListener(listener);

    twitterStream.filter(new FilterQuery().track(context.getString(keywords).split(",")));
  }
  private void start(final Context context) throws IOException {

    // Producer properties
    Properties props = new Properties();
    props.put("metadata.broker.list", context.getString(TwitterSourceConstant.BROKER_LIST));
    props.put("serializer.class", context.getString(TwitterSourceConstant.SERIALIZER));
    props.put("partitioner.class", context.getString(TwitterSourceConstant.PARTITIONER));
    props.put("request.required.acks", context.getString(TwitterSourceConstant.REQUIRED_ACKS));

    ProducerConfig config = new ProducerConfig(props);

    final Producer<String, String> producer = new Producer<String, String>(config);

    /** Twitter properties * */
    consumerKey = context.getString(TwitterSourceConstant.CONSUMER_KEY_KEY);
    consumerSecret = context.getString(TwitterSourceConstant.CONSUMER_SECRET_KEY);
    accessToken = context.getString(TwitterSourceConstant.ACCESS_TOKEN_KEY);
    accessTokenSecret = context.getString(TwitterSourceConstant.ACCESS_TOKEN_SECRET_KEY);

    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setOAuthConsumerKey(consumerKey);
    cb.setOAuthConsumerSecret(consumerSecret);
    cb.setOAuthAccessToken(accessToken);
    cb.setOAuthAccessTokenSecret(accessTokenSecret);
    cb.setJSONStoreEnabled(true);
    cb.setIncludeEntitiesEnabled(true);
    cb.setHttpProxyHost("proxy.tcs.com");
    cb.setHttpProxyPort(8080);
    cb.setHttpProxyUser("876216");
    cb.setHttpProxyPassword("Apple@123");
    twitterStream = new TwitterStreamFactory(cb.build()).getInstance();

    final Map<String, String> headers = new HashMap<String, String>();

    /** Twitter listener * */
    StatusListener listener =
        new StatusListener() {
          // The onStatus method is executed every time a new tweet comes
          // in.
          public void onStatus(Status status) {
            // The EventBuilder is used to build an event using the
            // the raw JSON of a tweet

            System.out.println("Listening :");
            KeyedMessage<String, String> data =
                new KeyedMessage<String, String>(
                    "testing1", TwitterObjectFactory.getRawJSON(status));

            producer.send(data);
            System.out.println(data);
          }

          public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {}

          public void onTrackLimitationNotice(int numberOfLimitedStatuses) {}

          public void onScrubGeo(long userId, long upToStatusId) {}

          public void onException(Exception ex) {
            logger.info("ShutDown");
            twitterStream.shutdown();
          }

          public void onStallWarning(StallWarning warning) {}
        };

    twitterStream.addListener(listener);
    /** GOGOGO * */
    twitterStream.sample();
    FilterQuery query =
        new FilterQuery()
            .track(
                Tweety.hashtags[0],
                Tweety.hashtags[1],
                Tweety.hashtags[2],
                Tweety.hashtags[3],
                Tweety.hashtags[4]);
    twitterStream.filter(query);
    /** Bind the listener * */
  }
Exemplo n.º 5
0
  private static ArrayList<String> ProcessTimeLine(String user)
      throws InterruptedException, TwitterException {
    ArrayList<String> Tweets = new ArrayList<String>();

    ConfigurationBuilder cb = new ConfigurationBuilder();
    cb.setDebugEnabled(true)
        .setOAuthConsumerKey(KEY)
        .setOAuthConsumerSecret(SECRET)
        .setOAuthAccessToken(ACCESSTOKEN)
        .setOAuthAccessTokenSecret(ACCESSSECRET);
    cb.setJSONStoreEnabled(true);

    // gets Twitter instance with default credentials
    boolean bWait = true;
    Twitter twitter = new TwitterFactory(cb.build()).getInstance();
    do {
      try {
        Map<String, RateLimitStatus> oRT = twitter.getRateLimitStatus();
        RateLimitStatus rateLimit = oRT.get("/statuses/user_timeline");
        int remaining = rateLimit.getRemaining();
        System.out.print("(Remaining API calls: " + remaining + ")");
        int remainingTime = rateLimit.getSecondsUntilReset();

        if (remaining <= NUM_TWEETS / 200 + 1) {
          System.out.println("Waiting " + remainingTime + " seconds");
          Thread.sleep(remainingTime * 1000);
        } else bWait = false;

      } catch (Exception te) {
        if (te.toString().toLowerCase().contains("rate limit")
            && !te.toString().toLowerCase().contains("bad authentication data")) {
          System.out.println("Waiting 60s");
          Thread.sleep(60 * 1000);
        } else {
          bWait = false;
        }
      }
    } while (bWait);

    try {
      Detector detector = DetectorFactory.create();
      List<Status> statuses;

      int iPage = 1;
      int iTweets = 0;
      do {

        int iPageSize = 0;
        if (iTweets + 200 < NUM_TWEETS) {
          iPageSize = 200;
        } else {
          iPageSize = NUM_TWEETS - iTweets;
        }
        statuses = twitter.getUserTimeline(user, new Paging(iPage, iPageSize));

        for (Status status : statuses) {

          String sStatusId = "-1";
          try {
            if ((status.getRetweetedStatus() != null)
                && (status.getRetweetedStatus().getUser() != null)) {
              continue;
            }

            try {
              detector.append(Simplify(status.getText()));
              if (detector.detect().equalsIgnoreCase("es")) {
                String sStatusJSON = DataObjectFactory.getRawJSON(status);
                Tweets.add(sStatusJSON);
              }
            } catch (Exception exl) {
            }
          } catch (Exception ex) {
            System.out.println("ERROR in status id " + sStatusId);
          }

          iTweets++;
        }
        iPage++;
      } while (statuses.size() > 0 && iTweets < NUM_TWEETS);

    } catch (TwitterException te) {
      te.printStackTrace();
      System.out.println("Failed to get timeline: " + te.getMessage());
    } catch (Exception ex) {

    }

    System.out.println("..." + Tweets.size() + " tweets.");

    return Tweets;
  }