/**
* start with more than one threads
*
* @param threadNum
* @return this
*/
public Spider thread(int threadNum) {
checkIfNotRunning();
this.threadNum = threadNum;
if (threadNum <= 0) {
throw new IllegalArgumentException("threadNum should be more than one!");
}
if (threadNum == 1) {
return this;
}
synchronized (this) {
this.executorService = ThreadUtils.newFixedThreadPool(threadNum);
}
return this;
}
/**
* set the downloader of spider
*
* @see Downloader
* @param downloader
* @return this
*/
public Spider setDownloader(Downloader downloader) {
checkIfNotRunning();
this.downloader = downloader;
return this;
}
/**
* set scheduler for Spider
*
* @param scheduler
* @return this
* @since 0.2.1
* @see Scheduler
*/
public Spider setScheduler(Scheduler scheduler) {
checkIfNotRunning();
this.scheduler = scheduler;
return this;
}
/**
* add a pipeline for Spider
*
* @param pipeline
* @return this
* @since 0.2.1
* @see Pipeline
*/
public Spider addPipeline(Pipeline pipeline) {
checkIfNotRunning();
this.pipelines.add(pipeline);
return this;
}
/**
* Set startUrls of Spider.<br>
* Prior to startUrls of Site.
*
* @param startUrls
* @return this
*/
public Spider startUrls(List<String> startUrls) {
checkIfNotRunning();
this.startUrls = startUrls;
return this;
}