public static void getBlogs() throws Exception {
BlockingQueue<String> queue = new ArrayBlockingQueue<String>(numCrawler * 4);
CrawlerC[] crawler = new CrawlerC[numCrawler];
for (int i = 0; i < crawler.length; i++) {
crawler[i] = new CrawlerC(queue);
crawler[i].start();
}
ResultSet rs = null;
int offset = 1;
while (true) {
offset += 100;
myStm.executeQuery("SELECT blogID from blogs where country = 'BR' LIMIT " + offset + ",100");
System.out.println("\n---" + offset + "---");
rs = myStm.getResultSet();
try {
if (!rs.first()) break;
if (false) break;
while (rs.next()) {
// System.out.println(rs.getString("blogID"));
if (!queue.offer(rs.getString("blogID"), 60, TimeUnit.SECONDS)) {
System.out.println("Offer.Timeout");
}
}
} catch (Exception e) {
}
}
queue.clear();
for (int i = 0; i < crawler.length; i++) queue.put(CrawlerC.NO_MORE_WORK);
for (int i = 0; i < crawler.length; i++) crawler[i].join();
}
/**
* Cancels and clears the queue of all tasks that should not be run due to shutdown policy.
* Invoked within super.shutdown.
*/
@Override
void onShutdown() {
BlockingQueue<Runnable> q = super.getQueue();
boolean keepDelayed = getExecuteExistingDelayedTasksAfterShutdownPolicy();
boolean keepPeriodic = getContinueExistingPeriodicTasksAfterShutdownPolicy();
if (!keepDelayed && !keepPeriodic) {
for (Object e : q.toArray())
if (e instanceof RunnableScheduledFuture<?>) ((RunnableScheduledFuture<?>) e).cancel(false);
q.clear();
} else {
// Traverse snapshot to avoid iterator exceptions
for (Object e : q.toArray()) {
if (e instanceof RunnableScheduledFuture) {
RunnableScheduledFuture<?> t = (RunnableScheduledFuture<?>) e;
if ((t.isPeriodic() ? !keepPeriodic : !keepDelayed)
|| t.isCancelled()) { // also remove if already cancelled
if (q.remove(t)) t.cancel(false);
}
}
}
}
tryTerminate();
}
public final synchronized void init() {
queue.clear();
state = true;
}
