public static void getBlogs() throws Exception {
BlockingQueue<String> queue = new ArrayBlockingQueue<String>(numCrawler * 4);
CrawlerC[] crawler = new CrawlerC[numCrawler];
for (int i = 0; i < crawler.length; i++) {
crawler[i] = new CrawlerC(queue);
crawler[i].start();
}
ResultSet rs = null;
int offset = 1;
while (true) {
offset += 100;
myStm.executeQuery("SELECT blogID from blogs where country = 'BR' LIMIT " + offset + ",100");
System.out.println("\n---" + offset + "---");
rs = myStm.getResultSet();
try {
if (!rs.first()) break;
if (false) break;
while (rs.next()) {
// System.out.println(rs.getString("blogID"));
if (!queue.offer(rs.getString("blogID"), 60, TimeUnit.SECONDS)) {
System.out.println("Offer.Timeout");
}
}
} catch (Exception e) {
}
}
queue.clear();
for (int i = 0; i < crawler.length; i++) queue.put(CrawlerC.NO_MORE_WORK);
for (int i = 0; i < crawler.length; i++) crawler[i].join();
}
private int pollCompletedDownloads() {
int tasks = 0;
for (int i = 0; i < completedRequests.size(); i++) {
parserService.submit(new FeedParserTask(completedRequests.poll()));
tasks++;
}
return tasks;
}
public final T take() {
if (!state) {
return queue.poll();
} else {
try {
return queue.take();
} catch (InterruptedException e) {
e.printStackTrace();
}
return null;
}
}
@Test
public void testIssue292() throws Exception {
final BlockingQueue qResponse = new ArrayBlockingQueue(1);
createSingleNodeExecutorService("testIssue292")
.submit(
new MemberCheck(),
new ExecutionCallback<Member>() {
public void onResponse(Member response) {
qResponse.offer(response);
}
public void onFailure(Throwable t) {}
});
Object response = qResponse.poll(10, TimeUnit.SECONDS);
assertNotNull(response);
assertTrue(response instanceof Member);
}
/**
* Waits for completed requests. Once the first request has been taken, the method will wait
* WAIT_TIMEOUT ms longer to collect more completed requests.
*
* @return Collected feeds or null if the method has been interrupted during the first waiting
* period.
*/
private List<Feed> collectCompletedRequests() {
List<Feed> results = new LinkedList<Feed>();
DownloadRequester requester = DownloadRequester.getInstance();
int tasks = 0;
try {
DownloadRequest request = completedRequests.take();
parserService.submit(new FeedParserTask(request));
tasks++;
} catch (InterruptedException e) {
return null;
}
tasks += pollCompletedDownloads();
isCollectingRequests = true;
if (requester.isDownloadingFeeds()) {
// wait for completion of more downloads
long startTime = System.currentTimeMillis();
long currentTime = startTime;
while (requester.isDownloadingFeeds() && (currentTime - startTime) < WAIT_TIMEOUT) {
try {
if (BuildConfig.DEBUG)
Log.d(TAG, "Waiting for " + (startTime + WAIT_TIMEOUT - currentTime) + " ms");
sleep(startTime + WAIT_TIMEOUT - currentTime);
} catch (InterruptedException e) {
if (BuildConfig.DEBUG) Log.d(TAG, "interrupted while waiting for more downloads");
tasks += pollCompletedDownloads();
} finally {
currentTime = System.currentTimeMillis();
}
}
tasks += pollCompletedDownloads();
}
isCollectingRequests = false;
for (int i = 0; i < tasks; i++) {
try {
Feed f = parserService.take().get();
if (f != null) {
results.add(f);
}
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}
return results;
}
public final void put(T item) {
if (!state) {
throw new RuntimeException("Putting Forbidden");
} else {
try {
queue.put(item);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
public void run() {
try {
while (!Thread.interrupted()) {
// Blocks until a course is ready
Plate plate = filledOrders.take();
print(this + "received " + plate + " delivering to " + plate.getOrder().getCustomer());
plate.getOrder().getCustomer().deliver(plate);
}
} catch (InterruptedException e) {
print(this + " interrupted");
}
print(this + " off duty");
}
/**
* Cancels and clears the queue of all tasks that should not be run due to shutdown policy.
* Invoked within super.shutdown.
*/
@Override
void onShutdown() {
BlockingQueue<Runnable> q = super.getQueue();
boolean keepDelayed = getExecuteExistingDelayedTasksAfterShutdownPolicy();
boolean keepPeriodic = getContinueExistingPeriodicTasksAfterShutdownPolicy();
if (!keepDelayed && !keepPeriodic) {
for (Object e : q.toArray())
if (e instanceof RunnableScheduledFuture<?>) ((RunnableScheduledFuture<?>) e).cancel(false);
q.clear();
} else {
// Traverse snapshot to avoid iterator exceptions
for (Object e : q.toArray()) {
if (e instanceof RunnableScheduledFuture) {
RunnableScheduledFuture<?> t = (RunnableScheduledFuture<?>) e;
if ((t.isPeriodic() ? !keepPeriodic : !keepDelayed)
|| t.isCancelled()) { // also remove if already cancelled
if (q.remove(t)) t.cancel(false);
}
}
}
}
tryTerminate();
}
public void run() {
while (true) {
try {
// System.out.println(r+": Take(wait)");
// String[] info = q.take();
String blogID = q.poll(60, TimeUnit.SECONDS);
if (blogID == null) {
System.out.println("Poll.Timeout");
continue;
}
// System.out.println(r+": Take(get) : "+blogID);
if (blogID == NO_MORE_WORK) {
break;
}
URL feedUrl = new URL("http://www.blogger.com/feeds/" + blogID + "/comments/default");
Query myQuery = new Query(feedUrl);
myQuery.setMaxResults(25);
System.out.print(r + "+,");
Feed resultFeed = myService.query(myQuery, Feed.class);
for (Entry entry : resultFeed.getEntries()) {
if (entry.getAuthors().get(0).getUri() != null) {
String profileID = entry.getAuthors().get(0).getUri().replaceAll("[^\\d]", "");
if (profileID.length() == 20) {
try {
myStm.executeUpdate(
"INSERT IGNORE INTO author SET profileID = '" + profileID + "'");
// System.out.print(r+"+,");
} catch (Exception e) {
}
}
}
}
} catch (Exception e) {
System.out.print(r + "ERR,");
}
}
System.out.println("Bye(" + r + ")");
try {
myStm.close();
} catch (Exception e) {
}
}
/*package*/ List<O> doJob(List<I> batch) {
List<O> generatedBatch;
assert lastBatchSent == false;
if (batch == POISON_PILL) {
lastBatch = true;
synchronized (name) {
pendingJobs--;
}
// System.out.println(name + " - lastBatch");
generatedBatch = Collections.emptyList();
} else {
EXECUTOR task = taskQueue.poll();
boolean nextNodesAvailable = true;
for (Node<O, ?, ?> node : nodes) {
nextNodesAvailable &= node.isAvailable();
}
if (task == null) { // No available task
resubmit(batch);
generatedBatch = null;
} else if (!nextNodesAvailable) { // Next nodes have to many batches.
try {
taskQueue.put(task);
} catch (InterruptedException e) {
e.printStackTrace();
}
resubmit(batch);
generatedBatch = null;
} else { // Execute
generatedBatch = execute(task, batch);
// System.out.println(name + " - end job - " + generatedBatch.size());
for (Node<O, ?, ?> node : nodes) {
node.submit(generatedBatch);
}
try {
taskQueue.put(task);
} catch (InterruptedException e) {
e.printStackTrace();
}
synchronized (name) {
pendingJobs--;
}
// System.out.println(name + " - pendingJobs " + pendingJobs);
}
}
if (isFinished()) {
if (!lastBatchSent) {
for (Node<O, ?, ?> node : nodes) {
node.submit(POISON_PILL);
}
lastBatchSent = true;
}
System.out.println("Node '" + name + "' is finished");
synchronized (syncObject) {
syncObject.notify();
}
} else {
System.out.println("Node '" + name + "' pendingJobs " + pendingJobs);
}
return generatedBatch;
}
public int queueSize() {
return changes.size();
}
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "RV_RETURN_VALUE_IGNORED_BAD_PRACTICE")
private void sendInterruptMarker() {
LOG.debug("Sending interrupt marker in order to interrupt feed consumer");
changes.offer(INTERRUPT_MARKER);
}
private void checkIfInterrupted(DocumentChange c) throws InterruptedException {
if (c == INTERRUPT_MARKER || (!shouldRun && changes.isEmpty())) {
throw new InterruptedException();
}
}
public DocumentChange next(long timeout, TimeUnit unit) throws InterruptedException {
assertRunningState();
DocumentChange c = changes.poll(timeout, unit);
checkIfInterrupted(c);
return c;
}
public void removeItem(RecordingEpisode episode) {
if (!RecordingMaps.remove(episode)) {
Log.getInstance()
.write(Log.LOGLEVEL_ERROR, "DT: Failed to remove episode from RecordingMaps.");
}
}
public final synchronized void init() {
queue.clear();
state = true;
}
public final int size() {
return queue.size();
}
/**
* Returns the number of items in the download queue. Does NOT include the currently downloading
* item, if any.
*
* <p>
*
* @return The number of items in the download queue.
*/
public Integer getNumberOfQueuedItems() {
Log.getInstance().write(Log.LOGLEVEL_TRACE, "DT: getNumberOfItems.");
return RecordingMaps.size();
}
/**
* Add an item to be downloaded. Details to follow....
*
* <p>
*
* @param info an array of strings ....
* @return true if it succeeded, false otherwise.
*/
public boolean addItem(RecordingEpisode episode) {
Log.getInstance().write(Log.LOGLEVEL_TRACE, "DT: addItem.");
return RecordingMaps.add(episode);
}
/** The main thread that does all of the downloading. */
@Override
public void run() {
Log.getInstance().write(Log.LOGLEVEL_TRACE, "DT: Starting.");
Thread.currentThread().setName("DownloadThread");
while (!stop) {
// Get the first item in the queue and then remove it from the queue.
try {
CurrentlyRecording = RecordingMaps.take();
} catch (InterruptedException e) {
Log.getInstance().write(Log.LOGLEVEL_WARN, "DT: Interrupted. Terminating.");
return;
}
Log.getInstance().write(Log.LOGLEVEL_TRACE, "DT: Have work to do.");
showCurrentlyRecording(CurrentlyRecording);
// Make sure we have enough parameters.
if (!CurrentlyRecording.isComplete()) {
Log.getInstance().write(Log.LOGLEVEL_ERROR, "DT: Not enough parameters.");
CurrentlyRecording.failed();
continue;
}
DownloadManager.getInstance().setCurrentlyRecordingID(CurrentlyRecording.getRequestID());
// Get all of the RSSItems for the Feed Context.
List<RSSItem> RSSItems = CurrentlyRecording.getRSSItems();
if (RSSItems == null) {
Log.getInstance().write(Log.LOGLEVEL_ERROR, "DT: null RSSItems.");
CurrentlyRecording.failed();
continue;
}
Log.getInstance()
.write(Log.LOGLEVEL_TRACE, "DT: Found episodes for podcast = " + RSSItems.size());
if (RSSItems.isEmpty()) {
Log.getInstance().write(Log.LOGLEVEL_ERROR, "DT: No RSSItems.");
CurrentlyRecording.failed();
continue;
}
// Get the one ChanItem (RSSItem) we are interested in.
RSSItem ChanItem =
CurrentlyRecording.getItemForID(RSSItems, CurrentlyRecording.getEpisodeID());
if (ChanItem == null) {
Log.getInstance().write(Log.LOGLEVEL_ERROR, "DT: null ChanItem.");
CurrentlyRecording.failed();
continue;
}
// Set the ChanItem.
CurrentlyRecording.setChanItem(ChanItem);
// Set the fileExt instance variable.
CurrentlyRecording.setFileExt();
// Create the tempfile where the episode will be downloaded to.
if (!CurrentlyRecording.setTempFile()) {
Log.getInstance().write(Log.LOGLEVEL_ERROR, "DT: Failed to setTempFile.");
CurrentlyRecording.failed();
continue;
}
// Download the episode to the tempfile.
if (!CurrentlyRecording.download()) {
Log.getInstance().write(Log.LOGLEVEL_ERROR, "DT: download failed.");
CurrentlyRecording.failed();
continue;
}
// Check for 0 size download.
if (CurrentlyRecording.isZeroSizeDownload()) {
Log.getInstance().write(Log.LOGLEVEL_WARN, "DT: File is 0 bytes long.");
CurrentlyRecording.failed();
continue;
}
// Move the tempfile to the final location and rename it to the final name.
if (!CurrentlyRecording.moveToFinalLocation()) {
Log.getInstance().write(Log.LOGLEVEL_ERROR, "DT: moveToFinalLocation failed.");
CurrentlyRecording.failed();
continue;
}
// Import the episode into the Sage database as an imported media file.
if (CurrentlyRecording.importAsAiring() == null) {
Log.getInstance().write(Log.LOGLEVEL_ERROR, "DT: importAsMediaFile failed.");
CurrentlyRecording.failed();
continue;
}
// Force Episode to update it's Airing information.
int AiringID = CurrentlyRecording.getAiringID();
// It worked.
Log.getInstance().write(Log.LOGLEVEL_TRACE, "DT: Completed successfully.");
CurrentlyRecording.completed();
CurrentlyRecording = null;
} // While !stop
Log.getInstance().write(Log.LOGLEVEL_TRACE, "DT: Fatal error. Ending.");
} // Run
public DocumentChange poll() throws InterruptedException {
assertRunningState();
DocumentChange c = changes.poll();
checkIfInterrupted(c);
return c;
}
public Iterator<T> iterator() {
return queue.iterator();
}
public void submitCompletedDownload(DownloadRequest request) {
completedRequests.offer(request);
if (isCollectingRequests) {
interrupt();
}
}
private void handleChange(String line)
throws IOException, InterruptedException, JsonParseException, JsonMappingException {
changes.put(new StdDocumentChange(OBJECT_MAPPER.readTree(line)));
}