public void grab(Date timeLimit) {
log.info("Twitter grabber started...");
Twitter twitter = new TwitterFactory().getInstance();
List<Brand> brandList = handler.getBrandProvider().getAllBrands();
ArticleProvider articleProvider = handler.getArticleProvider();
for (Brand b : brandList) {
Query query = new Query(b.getName());
query.setRpp(PAGE_SIZE);
query.setLang("ru");
query.setResultType(Query.MIXED);
List<Tweet> resultTweets = new LinkedList<Tweet>();
QueryResult queryResult;
int pageNumber = 1;
try {
do {
query.setPage(pageNumber);
queryResult = twitter.search(query);
resultTweets.addAll(queryResult.getTweets());
pageNumber++;
log.info(pageNumber);
} while (ISSUANCE_SIZE > resultTweets.size());
} catch (TwitterException e) {
throw new RuntimeException(e);
}
log.info("tweets in da111y: " + resultTweets.size());
twitter = null;
twitter = new TwitterFactory().getInstance();
for (Tweet t : resultTweets) {
articleProvider.writeArticleToDataStore(
new Article(-1, b.getId(), 2, "", t.getText(), "", getSimpleTime(t.getCreatedAt()), 1));
}
}
log.info("twitter grabber finished succesful.");
}
// this test not work while not add base of articles
public static void main(String[] args) {
BasicDataSource ds = new BasicDataSource();
ds.setDriverClassName("com.mysql.jdbc.Driver");
ds.setUrl("jdbc:mysql://localhost:3306/BAdirty?useUnicode=true&characterEncoding=utf8");
ds.setUsername("root");
ds.setPassword("root");
ds.setValidationQuery("select 1");
SimpleJdbcTemplate jdbcTemplate = new SimpleJdbcTemplate(ds);
BrandProvider dataStore = new BrandProvider(jdbcTemplate);
dataStore.cleanDataStore();
Brand b1 = new Brand(4, "Gazprom", "Gazprom is russian gasoline gaint", "www.gazprom.ru", 0);
dataStore.writeBrandToDataStore(b1);
b1 = new Brand(1, "Microsoft", "Microsoft makes bad software", "www.microsoft.com", 0);
dataStore.writeBrandToDataStore(b1);
b1 =
new Brand(
3,
"Apple",
"Apple makes software too and i-production. На русском",
"www.apple.com",
0);
dataStore.writeBrandToDataStore(b1);
b1 = new Brand(2, "Google", "Google is better than other search machines", "www.google.com", 0);
dataStore.writeBrandToDataStore(b1);
MySQLArticleProvider dataStore2 = new MySQLArticleProvider(jdbcTemplate);
Article a1 =
new Article(
4,
1,
1,
"Gazprom crashed",
"Apple делает хорошую продукцию",
"www.lenta.ru",
new Timestamp(90, 0, 0, 0, 0, 0, 0),
0);
dataStore2.writeArticleToDataStore(a1);
a1 =
new Article(
2,
1,
1,
"Microsoft crashed",
"Apple не делает хорошую продукцию",
"www.lenta.ru",
new Timestamp(90, 0, 0, 0, 0, 0, 0),
0);
dataStore2.writeArticleToDataStore(a1);
Indexer ind = new Indexer();
ind.setDirectoryBrand("index_brand/");
ind.setDirectoryArticle(
"index_article/"); // while not work's sorry... can add base of articles...
ind.setJdbcTemplate(jdbcTemplate); // set base of brand and base of articles
try {
ind.afterPropertiesSet();
} catch (Exception e) {
e.printStackTrace();
}
Searcher searcher = new Searcher();
searcher.setIndexDirArticle("index_article/");
searcher.setIndexDirBrand("index_brand/");
try {
searcher.getReadyForSearch();
} catch (Exception e) {
e.printStackTrace();
}
try {
List<Article> lst = searcher.searchArticleByContent("делает");
for (Article b : lst) {
System.out.println(b.getContent());
}
} catch (Exception e) {
e.printStackTrace();
}
try {
List<Brand> lst = searcher.searchBrandByDescription("software");
for (Brand b : lst) {
System.out.println(b.getDescription());
}
} catch (Exception e) {
e.printStackTrace();
}
}