public static void init(ScraperConfig scraperConfig) throws TalesException {
try {
// inits the services
talesDB =
new TalesDB(scraperConfig.getConnection(), scraperConfig.getTemplate().getMetadata());
tasksDB = new TasksDB(scraperConfig);
// starts the task machine with the template
taskWorker = new TaskWorker(scraperConfig);
taskWorker.init();
loopReferenceTime = talesDB.getMostRecentCrawledDocuments(1).get(0).getLastUpdate();
while (!taskWorker.hasFailover()) {
// adds tasks
if ((tasksDB.count() + taskWorker.getTasksRunning().size()) < Config.getMinTasks()) {
ArrayList<Task> tasks = getTasks();
if (tasks.size() > 0) {
Logger.log(new Throwable(), "adding tasks to \"" + scraperConfig.getTaskName() + "\"");
tasksDB.add(tasks);
if (!taskWorker.isWorkerActive() && !taskWorker.hasFailover()) {
taskWorker = new TaskWorker(scraperConfig);
taskWorker.init();
}
}
}
// if no tasks means we are finished
if ((tasksDB.count() + taskWorker.getTasksRunning().size()) == 0) {
break;
}
try {
Thread.sleep(1000);
} catch (Exception e) {
new TalesException(new Throwable(), e);
}
}
} catch (Exception e) {
throw new TalesException(new Throwable(), e);
}
}
