public List<WeiboEntityBean> getWeiboEntity(int pageNumber, String cookie) throws Exception {
weiboEntityBeans = new ArrayList<>();
HttpRequesterImpl myRequester = (HttpRequesterImpl) this.getHttpRequester();
myRequester.setCookie(cookie);
for (int i = 1; i <= pageNumber; i++) {
this.addSeed(CommonValues.WEIBO_BASE_URL + "/index/feed?format=cards&page=" + i);
}
this.start(1);
return weiboEntityBeans;
}
public MoviceJsonCrawler(String crawlPath) {
super(crawlPath);
this.crawlPath = crawlPath;
HttpRequesterImpl requester = (HttpRequesterImpl) this.getHttpRequester();
requester.setHeader(
"Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
requester.setHeader("Accept-Encoding", "gzip, deflate, sdch");
requester.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6");
requester.setHeader("Cache-Control", "max-age=0");
requester.setHeader("Connection", "keep-alive");
requester.setHeader("Host", "www.douban.com");
requester.setHeader(
"User-Agent",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36");
requester.setCookie(
"bid=\"kurlmstJI8o\"; ll=\"108309\"; ap=1; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1439121430%2C%22http%3A%2F%2Fmovie.douban.com%2Ftag%2F%22%5D; _pk_id.100001.8cb4=c07960c45819019e.1403859829.8.1439121430.1439111681.; _pk_ses.100001.8cb4=*; __utmt=1; __utma=30149280.2001245108.1375193550.1439111224.1439121431.12; __utmb=30149280.2.9.1439121431; __utmc=30149280; __utmz=30149280.1439111224.11.6.utmcsr=douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/tag/%E7%88%B1%E6%83%85/");
requester.setMethod("GET");
}