private void failover(HttpServletRequest request, HttpServletResponse response) {
try {
Logger.log(new Throwable(), "FAILOVER: failover...");
// new server
String publicDNS = newServer(request, response);
// starts process
String process = URLDecoder.decode(request.getParameter("process"), "UTF-8");
String url = "http://" + publicDNS + ":" + Config.getDashbaordPort();
while (!new Download().urlExists(url)) {
Thread.sleep(1000);
}
url += "/start?process=" + process;
new Download().urlExists(url);
Logger.log(new Throwable(), "FAILOVER: starting process: " + url);
} catch (Exception e) {
new TalesException(new Throwable(), e);
}
}
private String newServer(HttpServletRequest request, HttpServletResponse response) {
try {
String requestProvider = request.getParameter("cloud-provider").toLowerCase();
for (CloudProviderInterface cloudProvider : Config.getCloudProviders()) {
if (requestProvider.equals(cloudProvider.getId().toLowerCase())) {
// new server
Logger.log(new Throwable(), "NEW: creating new server in: " + cloudProvider.getId());
String publicDNS = cloudProvider.newServer(request);
// waits for tales dashboard to be up
Logger.log(new Throwable(), "NEW: waiting for server (" + publicDNS + ") to be up...");
while (true) {
if (new Download().urlExists("http://" + publicDNS + ":" + Config.getDashbaordPort())) {
break;
}
Thread.sleep(1000);
}
Logger.log(new Throwable(), "NEW: finished");
// http response
JSONObject json = new JSONObject();
json.put("dns", publicDNS);
response.setContentType("application/json");
response.setStatus(HttpServletResponse.SC_OK);
response.getWriter().println(json);
return publicDNS;
}
}
} catch (Exception e) {
new TalesException(new Throwable(), e);
}
return null;
}
private void reboot() {
try {
Logger.log(new Throwable(), "REBOOT: rebooting...");
ProcessBuilder builder = new ProcessBuilder("/bin/sh", "-c", "reboot");
builder.start();
} catch (Exception e) {
new TalesException(new Throwable(), e);
}
}
private void kill(String pid) {
try {
Logger.log(new Throwable(), "KILL: killing pid: " + pid);
ProcessBuilder builder = new ProcessBuilder("/bin/sh", "-c", "kill " + pid);
builder.start();
} catch (Exception e) {
new TalesException(new Throwable(), e);
}
}
public static void init(ScraperConfig scraperConfig) throws TalesException {
try {
// inits the services
talesDB =
new TalesDB(scraperConfig.getConnection(), scraperConfig.getTemplate().getMetadata());
tasksDB = new TasksDB(scraperConfig);
// starts the task machine with the template
taskWorker = new TaskWorker(scraperConfig);
taskWorker.init();
loopReferenceTime = talesDB.getMostRecentCrawledDocuments(1).get(0).getLastUpdate();
while (!taskWorker.hasFailover()) {
// adds tasks
if ((tasksDB.count() + taskWorker.getTasksRunning().size()) < Config.getMinTasks()) {
ArrayList<Task> tasks = getTasks();
if (tasks.size() > 0) {
Logger.log(new Throwable(), "adding tasks to \"" + scraperConfig.getTaskName() + "\"");
tasksDB.add(tasks);
if (!taskWorker.isWorkerActive() && !taskWorker.hasFailover()) {
taskWorker = new TaskWorker(scraperConfig);
taskWorker.init();
}
}
}
// if no tasks means we are finished
if ((tasksDB.count() + taskWorker.getTasksRunning().size()) == 0) {
break;
}
try {
Thread.sleep(1000);
} catch (Exception e) {
new TalesException(new Throwable(), e);
}
}
} catch (Exception e) {
throw new TalesException(new Throwable(), e);
}
}
private void start(HttpServletRequest request, HttpServletResponse response) {
try {
String process = URLDecoder.decode(request.getParameter("process"), "UTF-8");
process = "java -cp " + Config.getTemplatesJar() + " " + process + " >/dev/null 2>&1";
Logger.log(new Throwable(), "START: launching \"" + process + "/");
ProcessBuilder builder = new ProcessBuilder("/bin/sh", "-c", process);
builder.start();
} catch (Exception e) {
new TalesException(new Throwable(), e);
}
}
private void forceDelete() {
try {
for (CloudProviderInterface cloudProvider : Config.getCloudProviders()) {
if (cloudProvider.isApplicationRunningHere()) {
cloudProvider.delete();
break;
}
}
Logger.log(new Throwable(), "DELETE: server deleted");
} catch (Exception e) {
new TalesException(new Throwable(), e);
}
}
private static synchronized void load() {
if (!inited) {
inited = true;
Loader loader = new Config().new Loader();
loader.run();
try {
Logger.log(
new Throwable(),
"reading tales-config from "
+ TalesSystem.getFolderGitBranchName("~/tales-templates/")
+ ".json");
} catch (Exception e) {
e.printStackTrace();
}
}
}
private void delete() {
try {
// makes sure that we dont delete a server with dbs -- we ignore tales logs
if (DBUtils.getLocalTalesDBNames().size() == 0
|| (DBUtils.getLocalTalesDBNames().size() == 1
&& DBUtils.getLocalTalesDBNames().get(0).contains(LogsDB.getDBName()))) {
forceDelete();
} else {
Logger.log(
new Throwable(),
"DELETE: cant delete server, it contains tales databases. Delete all the tales databases before trying to delete the server.");
}
} catch (Exception e) {
new TalesException(new Throwable(), e);
}
}