/**
* Process specific urls without url discovering.
*
* @param urls urls to process
*/
public void test(String... urls) {
checkComponent();
if (urls.length > 0) {
for (String url : urls) {
processRequest(new Request(url));
}
}
}
@Override
public void run() {
if (!stat.compareAndSet(STAT_INIT, STAT_RUNNING)) {
throw new IllegalStateException("Spider is already running!");
}
checkComponent();
if (startUrls != null) {
for (String startUrl : startUrls) {
scheduler.push(new Request(startUrl), this);
}
}
Request request = scheduler.poll(this);
// singel thread
if (executorService == null) {
while (request != null) {
processRequest(request);
request = scheduler.poll(this);
}
} else {
// multi thread
final AtomicInteger threadAlive = new AtomicInteger(0);
while (true) {
if (request == null) {
// when no request found but some thread is alive, sleep a while.
try {
Thread.sleep(100);
} catch (InterruptedException e) {
}
} else {
final Request requestFinal = request;
threadAlive.incrementAndGet();
executorService.execute(
new Runnable() {
@Override
public void run() {
processRequest(requestFinal);
threadAlive.decrementAndGet();
}
});
}
request = scheduler.poll(this);
if (threadAlive.get() == 0) {
request = scheduler.poll(this);
if (request == null) {
break;
}
}
}
executorService.shutdown();
}
stat.compareAndSet(STAT_RUNNING, STAT_STOPPED);
// release some resources
destroy();
}