public void grab(Date timeLimit) {
    log.info("Twitter grabber started...");
    Twitter twitter = new TwitterFactory().getInstance();

    List<Brand> brandList = handler.getBrandProvider().getAllBrands();
    ArticleProvider articleProvider = handler.getArticleProvider();

    for (Brand b : brandList) {
      Query query = new Query(b.getName());
      query.setRpp(PAGE_SIZE);
      query.setLang("ru");
      query.setResultType(Query.MIXED);

      List<Tweet> resultTweets = new LinkedList<Tweet>();
      QueryResult queryResult;
      int pageNumber = 1;

      try {
        do {
          query.setPage(pageNumber);
          queryResult = twitter.search(query);
          resultTweets.addAll(queryResult.getTweets());
          pageNumber++;
          log.info(pageNumber);
        } while (ISSUANCE_SIZE > resultTweets.size());
      } catch (TwitterException e) {
        throw new RuntimeException(e);
      }
      log.info("tweets in da111y: " + resultTweets.size());
      twitter = null;
      twitter = new TwitterFactory().getInstance();

      for (Tweet t : resultTweets) {
        articleProvider.writeArticleToDataStore(
            new Article(-1, b.getId(), 2, "", t.getText(), "", getSimpleTime(t.getCreatedAt()), 1));
      }
    }
    log.info("twitter grabber finished succesful.");
  }
Exemple #2
0
  // this test not work while not add base of articles
  public static void main(String[] args) {

    BasicDataSource ds = new BasicDataSource();
    ds.setDriverClassName("com.mysql.jdbc.Driver");
    ds.setUrl("jdbc:mysql://localhost:3306/BAdirty?useUnicode=true&amp;characterEncoding=utf8");
    ds.setUsername("root");
    ds.setPassword("root");
    ds.setValidationQuery("select 1");
    SimpleJdbcTemplate jdbcTemplate = new SimpleJdbcTemplate(ds);
    BrandProvider dataStore = new BrandProvider(jdbcTemplate);

    dataStore.cleanDataStore();
    Brand b1 = new Brand(4, "Gazprom", "Gazprom is russian gasoline gaint", "www.gazprom.ru", 0);
    dataStore.writeBrandToDataStore(b1);
    b1 = new Brand(1, "Microsoft", "Microsoft makes bad software", "www.microsoft.com", 0);
    dataStore.writeBrandToDataStore(b1);
    b1 =
        new Brand(
            3,
            "Apple",
            "Apple makes software too and i-production. На русском",
            "www.apple.com",
            0);
    dataStore.writeBrandToDataStore(b1);
    b1 = new Brand(2, "Google", "Google is better than other search machines", "www.google.com", 0);
    dataStore.writeBrandToDataStore(b1);

    MySQLArticleProvider dataStore2 = new MySQLArticleProvider(jdbcTemplate);
    Article a1 =
        new Article(
            4,
            1,
            1,
            "Gazprom crashed",
            "Apple делает хорошую продукцию",
            "www.lenta.ru",
            new Timestamp(90, 0, 0, 0, 0, 0, 0),
            0);
    dataStore2.writeArticleToDataStore(a1);
    a1 =
        new Article(
            2,
            1,
            1,
            "Microsoft crashed",
            "Apple не делает хорошую продукцию",
            "www.lenta.ru",
            new Timestamp(90, 0, 0, 0, 0, 0, 0),
            0);
    dataStore2.writeArticleToDataStore(a1);

    Indexer ind = new Indexer();
    ind.setDirectoryBrand("index_brand/");
    ind.setDirectoryArticle(
        "index_article/"); // while not work's sorry... can add base of articles...
    ind.setJdbcTemplate(jdbcTemplate); // set base of brand and base of articles
    try {
      ind.afterPropertiesSet();
    } catch (Exception e) {
      e.printStackTrace();
    }

    Searcher searcher = new Searcher();
    searcher.setIndexDirArticle("index_article/");
    searcher.setIndexDirBrand("index_brand/");
    try {
      searcher.getReadyForSearch();
    } catch (Exception e) {
      e.printStackTrace();
    }

    try {
      List<Article> lst = searcher.searchArticleByContent("делает");
      for (Article b : lst) {
        System.out.println(b.getContent());
      }
    } catch (Exception e) {
      e.printStackTrace();
    }
    try {
      List<Brand> lst = searcher.searchBrandByDescription("software");
      for (Brand b : lst) {
        System.out.println(b.getDescription());
      }
    } catch (Exception e) {
      e.printStackTrace();
    }
  }