Exemplo n.º 1
0
  private void failover(HttpServletRequest request, HttpServletResponse response) {

    try {

      Logger.log(new Throwable(), "FAILOVER: failover...");

      // new server
      String publicDNS = newServer(request, response);

      // starts process
      String process = URLDecoder.decode(request.getParameter("process"), "UTF-8");

      String url = "http://" + publicDNS + ":" + Config.getDashbaordPort();

      while (!new Download().urlExists(url)) {
        Thread.sleep(1000);
      }

      url += "/start?process=" + process;
      new Download().urlExists(url);

      Logger.log(new Throwable(), "FAILOVER: starting process: " + url);

    } catch (Exception e) {
      new TalesException(new Throwable(), e);
    }
  }
Exemplo n.º 2
0
  private String newServer(HttpServletRequest request, HttpServletResponse response) {

    try {

      String requestProvider = request.getParameter("cloud-provider").toLowerCase();

      for (CloudProviderInterface cloudProvider : Config.getCloudProviders()) {

        if (requestProvider.equals(cloudProvider.getId().toLowerCase())) {

          // new server
          Logger.log(new Throwable(), "NEW: creating new server in: " + cloudProvider.getId());
          String publicDNS = cloudProvider.newServer(request);

          // waits for tales dashboard to be up
          Logger.log(new Throwable(), "NEW: waiting for server (" + publicDNS + ") to be up...");
          while (true) {

            if (new Download().urlExists("http://" + publicDNS + ":" + Config.getDashbaordPort())) {
              break;
            }

            Thread.sleep(1000);
          }

          Logger.log(new Throwable(), "NEW: finished");

          // http response
          JSONObject json = new JSONObject();
          json.put("dns", publicDNS);

          response.setContentType("application/json");
          response.setStatus(HttpServletResponse.SC_OK);
          response.getWriter().println(json);

          return publicDNS;
        }
      }

    } catch (Exception e) {
      new TalesException(new Throwable(), e);
    }

    return null;
  }
Exemplo n.º 3
0
  private void reboot() {

    try {

      Logger.log(new Throwable(), "REBOOT: rebooting...");
      ProcessBuilder builder = new ProcessBuilder("/bin/sh", "-c", "reboot");
      builder.start();

    } catch (Exception e) {
      new TalesException(new Throwable(), e);
    }
  }
Exemplo n.º 4
0
  private void kill(String pid) {

    try {

      Logger.log(new Throwable(), "KILL: killing pid: " + pid);
      ProcessBuilder builder = new ProcessBuilder("/bin/sh", "-c", "kill " + pid);
      builder.start();

    } catch (Exception e) {
      new TalesException(new Throwable(), e);
    }
  }
Exemplo n.º 5
0
  public static void init(ScraperConfig scraperConfig) throws TalesException {

    try {

      // inits the services
      talesDB =
          new TalesDB(scraperConfig.getConnection(), scraperConfig.getTemplate().getMetadata());
      tasksDB = new TasksDB(scraperConfig);

      // starts the task machine with the template
      taskWorker = new TaskWorker(scraperConfig);
      taskWorker.init();

      loopReferenceTime = talesDB.getMostRecentCrawledDocuments(1).get(0).getLastUpdate();

      while (!taskWorker.hasFailover()) {

        // adds tasks
        if ((tasksDB.count() + taskWorker.getTasksRunning().size()) < Config.getMinTasks()) {

          ArrayList<Task> tasks = getTasks();

          if (tasks.size() > 0) {

            Logger.log(new Throwable(), "adding tasks to \"" + scraperConfig.getTaskName() + "\"");

            tasksDB.add(tasks);

            if (!taskWorker.isWorkerActive() && !taskWorker.hasFailover()) {
              taskWorker = new TaskWorker(scraperConfig);
              taskWorker.init();
            }
          }
        }

        // if no tasks means we are finished
        if ((tasksDB.count() + taskWorker.getTasksRunning().size()) == 0) {
          break;
        }

        try {
          Thread.sleep(1000);
        } catch (Exception e) {
          new TalesException(new Throwable(), e);
        }
      }

    } catch (Exception e) {
      throw new TalesException(new Throwable(), e);
    }
  }
Exemplo n.º 6
0
  private void start(HttpServletRequest request, HttpServletResponse response) {

    try {

      String process = URLDecoder.decode(request.getParameter("process"), "UTF-8");

      process = "java -cp " + Config.getTemplatesJar() + " " + process + " >/dev/null 2>&1";
      Logger.log(new Throwable(), "START: launching \"" + process + "/");

      ProcessBuilder builder = new ProcessBuilder("/bin/sh", "-c", process);
      builder.start();

    } catch (Exception e) {
      new TalesException(new Throwable(), e);
    }
  }
Exemplo n.º 7
0
  private void forceDelete() {

    try {

      for (CloudProviderInterface cloudProvider : Config.getCloudProviders()) {
        if (cloudProvider.isApplicationRunningHere()) {
          cloudProvider.delete();
          break;
        }
      }

      Logger.log(new Throwable(), "DELETE: server deleted");

    } catch (Exception e) {
      new TalesException(new Throwable(), e);
    }
  }
Exemplo n.º 8
0
  private static synchronized void load() {

    if (!inited) {

      inited = true;
      Loader loader = new Config().new Loader();
      loader.run();

      try {
        Logger.log(
            new Throwable(),
            "reading tales-config from "
                + TalesSystem.getFolderGitBranchName("~/tales-templates/")
                + ".json");
      } catch (Exception e) {
        e.printStackTrace();
      }
    }
  }
Exemplo n.º 9
0
  private void delete() {

    try {

      // makes sure that we dont delete a server with dbs -- we ignore tales logs
      if (DBUtils.getLocalTalesDBNames().size() == 0
          || (DBUtils.getLocalTalesDBNames().size() == 1
              && DBUtils.getLocalTalesDBNames().get(0).contains(LogsDB.getDBName()))) {

        forceDelete();

      } else {
        Logger.log(
            new Throwable(),
            "DELETE: cant delete server, it contains tales databases. Delete all the tales databases before trying to delete the server.");
      }

    } catch (Exception e) {
      new TalesException(new Throwable(), e);
    }
  }