Exemple #1
0
  public static void main(String[] args) throws TalesException {

    try {

      Options options = new Options();
      options.addOption("template", true, "template class path");
      options.addOption("threads", true, "number of templates");
      CommandLineParser parser = new PosixParser();
      CommandLine cmd = parser.parse(options, args);

      String templatePath = cmd.getOptionValue("template");
      int threads = Integer.parseInt(cmd.getOptionValue("threads"));

      // when app is killed
      Runtime.getRuntime()
          .addShutdownHook(
              new Thread() {

                public void run() {

                  if (taskWorker != null) {
                    taskWorker.stop();
                  }

                  Logger.log(new Throwable(), "---> bye...");
                }
              });

      // monitors the app performance
      AppMonitor.init();

      // reflection / new template
      TemplateInterface template = (TemplateInterface) Class.forName(templatePath).newInstance();

      // connection
      Connection connection = new Connection();
      connection.setConnectionsNumber(threads);

      // scraper config
      ScraperConfig scraperConfig = new ScraperConfig();
      scraperConfig.setScraperName("LoopScraper");
      scraperConfig.setTemplate(template);
      scraperConfig.setConnection(connection);

      // scraper
      LoopScraper.init(scraperConfig);

      // stop
      AppMonitor.stop();
      System.exit(0);

    } catch (Exception e) {
      AppMonitor.stop();
      System.exit(0);
      throw new TalesException(new Throwable(), e);
    }
  }
Exemple #2
0
  public static void init(ScraperConfig scraperConfig) throws TalesException {

    try {

      // inits the services
      talesDB =
          new TalesDB(scraperConfig.getConnection(), scraperConfig.getTemplate().getMetadata());
      tasksDB = new TasksDB(scraperConfig);

      // starts the task machine with the template
      taskWorker = new TaskWorker(scraperConfig);
      taskWorker.init();

      loopReferenceTime = talesDB.getMostRecentCrawledDocuments(1).get(0).getLastUpdate();

      while (!taskWorker.hasFailover()) {

        // adds tasks
        if ((tasksDB.count() + taskWorker.getTasksRunning().size()) < Config.getMinTasks()) {

          ArrayList<Task> tasks = getTasks();

          if (tasks.size() > 0) {

            Logger.log(new Throwable(), "adding tasks to \"" + scraperConfig.getTaskName() + "\"");

            tasksDB.add(tasks);

            if (!taskWorker.isWorkerActive() && !taskWorker.hasFailover()) {
              taskWorker = new TaskWorker(scraperConfig);
              taskWorker.init();
            }
          }
        }

        // if no tasks means we are finished
        if ((tasksDB.count() + taskWorker.getTasksRunning().size()) == 0) {
          break;
        }

        try {
          Thread.sleep(1000);
        } catch (Exception e) {
          new TalesException(new Throwable(), e);
        }
      }

    } catch (Exception e) {
      throw new TalesException(new Throwable(), e);
    }
  }