public static void getBlogs() throws Exception {
BlockingQueue<String> queue = new ArrayBlockingQueue<String>(numCrawler * 4);
CrawlerC[] crawler = new CrawlerC[numCrawler];
for (int i = 0; i < crawler.length; i++) {
crawler[i] = new CrawlerC(queue);
crawler[i].start();
}
ResultSet rs = null;
int offset = 1;
while (true) {
offset += 100;
myStm.executeQuery("SELECT blogID from blogs where country = 'BR' LIMIT " + offset + ",100");
System.out.println("\n---" + offset + "---");
rs = myStm.getResultSet();
try {
if (!rs.first()) break;
if (false) break;
while (rs.next()) {
// System.out.println(rs.getString("blogID"));
if (!queue.offer(rs.getString("blogID"), 60, TimeUnit.SECONDS)) {
System.out.println("Offer.Timeout");
}
}
} catch (Exception e) {
}
}
queue.clear();
for (int i = 0; i < crawler.length; i++) queue.put(CrawlerC.NO_MORE_WORK);
for (int i = 0; i < crawler.length; i++) crawler[i].join();
}
public final void put(T item) {
if (!state) {
throw new RuntimeException("Putting Forbidden");
} else {
try {
queue.put(item);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
/*package*/ List<O> doJob(List<I> batch) {
List<O> generatedBatch;
assert lastBatchSent == false;
if (batch == POISON_PILL) {
lastBatch = true;
synchronized (name) {
pendingJobs--;
}
// System.out.println(name + " - lastBatch");
generatedBatch = Collections.emptyList();
} else {
EXECUTOR task = taskQueue.poll();
boolean nextNodesAvailable = true;
for (Node<O, ?, ?> node : nodes) {
nextNodesAvailable &= node.isAvailable();
}
if (task == null) { // No available task
resubmit(batch);
generatedBatch = null;
} else if (!nextNodesAvailable) { // Next nodes have to many batches.
try {
taskQueue.put(task);
} catch (InterruptedException e) {
e.printStackTrace();
}
resubmit(batch);
generatedBatch = null;
} else { // Execute
generatedBatch = execute(task, batch);
// System.out.println(name + " - end job - " + generatedBatch.size());
for (Node<O, ?, ?> node : nodes) {
node.submit(generatedBatch);
}
try {
taskQueue.put(task);
} catch (InterruptedException e) {
e.printStackTrace();
}
synchronized (name) {
pendingJobs--;
}
// System.out.println(name + " - pendingJobs " + pendingJobs);
}
}
if (isFinished()) {
if (!lastBatchSent) {
for (Node<O, ?, ?> node : nodes) {
node.submit(POISON_PILL);
}
lastBatchSent = true;
}
System.out.println("Node '" + name + "' is finished");
synchronized (syncObject) {
syncObject.notify();
}
} else {
System.out.println("Node '" + name + "' pendingJobs " + pendingJobs);
}
return generatedBatch;
}
private void handleChange(String line)
throws IOException, InterruptedException, JsonParseException, JsonMappingException {
changes.put(new StdDocumentChange(OBJECT_MAPPER.readTree(line)));
}
