public static void getBlogs() throws Exception {

    BlockingQueue<String> queue = new ArrayBlockingQueue<String>(numCrawler * 4);

    CrawlerC[] crawler = new CrawlerC[numCrawler];
    for (int i = 0; i < crawler.length; i++) {
      crawler[i] = new CrawlerC(queue);
      crawler[i].start();
    }

    ResultSet rs = null;
    int offset = 1;
    while (true) {
      offset += 100;
      myStm.executeQuery("SELECT blogID from blogs where country = 'BR' LIMIT " + offset + ",100");
      System.out.println("\n---" + offset + "---");

      rs = myStm.getResultSet();
      try {
        if (!rs.first()) break;
        if (false) break;
        while (rs.next()) {
          // System.out.println(rs.getString("blogID"));
          if (!queue.offer(rs.getString("blogID"), 60, TimeUnit.SECONDS)) {
            System.out.println("Offer.Timeout");
          }
        }
      } catch (Exception e) {
      }
    }

    queue.clear();
    for (int i = 0; i < crawler.length; i++) queue.put(CrawlerC.NO_MORE_WORK);
    for (int i = 0; i < crawler.length; i++) crawler[i].join();
  }
 public void submitCompletedDownload(DownloadRequest request) {
   completedRequests.offer(request);
   if (isCollectingRequests) {
     interrupt();
   }
 }
Example #3
0
 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "RV_RETURN_VALUE_IGNORED_BAD_PRACTICE")
 private void sendInterruptMarker() {
   LOG.debug("Sending interrupt marker in order to interrupt feed consumer");
   changes.offer(INTERRUPT_MARKER);
 }