Java IndeedDao.getTargetUrls Examples

Example #1

File: IndeedCrawler.java Project: gyiang/MyCrawlerHub

  // 详情页爬取
  public void crawlDeatil() {
    while (true) {

      List<String> l = indeedDao.getTargetUrls();
      if (l == null) {
        break;
      }
      OOSpider.create(site, indeedDetailPipeline, IndeedDetail.class)
          .startUrls(l)
          .thread(threadNum)
          .setSpawnUrl(false)
          .run();
    }
  }