/** * Process specific urls without url discovering. * * @param urls urls to process */ public void test(String... urls) { checkComponent(); if (urls.length > 0) { for (String url : urls) { processRequest(new Request(url)); } } }
@Override public void run() { if (!stat.compareAndSet(STAT_INIT, STAT_RUNNING)) { throw new IllegalStateException("Spider is already running!"); } checkComponent(); if (startUrls != null) { for (String startUrl : startUrls) { scheduler.push(new Request(startUrl), this); } } Request request = scheduler.poll(this); // singel thread if (executorService == null) { while (request != null) { processRequest(request); request = scheduler.poll(this); } } else { // multi thread final AtomicInteger threadAlive = new AtomicInteger(0); while (true) { if (request == null) { // when no request found but some thread is alive, sleep a while. try { Thread.sleep(100); } catch (InterruptedException e) { } } else { final Request requestFinal = request; threadAlive.incrementAndGet(); executorService.execute( new Runnable() { @Override public void run() { processRequest(requestFinal); threadAlive.decrementAndGet(); } }); } request = scheduler.poll(this); if (threadAlive.get() == 0) { request = scheduler.poll(this); if (request == null) { break; } } } executorService.shutdown(); } stat.compareAndSet(STAT_RUNNING, STAT_STOPPED); // release some resources destroy(); }