Esempio n. 1
0
  /** Creates a new document server */
  public DocServer() {
    super();
    /* really should try and get the companion service name from the env
     * here, but not worth the effort
     */
    JsonServerSyslog templogger =
        new JsonServerSyslog(
            DEFAULT_COMPANION_SERVICE_NAME,
            JsonServerServlet.KB_DEP,
            JsonServerSyslog.LOG_LEVEL_INFO,
            false);
    if (sysLogOut != null) {
      templogger.changeOutput(sysLogOut);
    }
    // getConfig() gets the service name from the env if it exists
    config = JsonServerServlet.getConfig(DEFAULT_COMPANION_SERVICE_NAME, templogger);

    String serverName = config.get(CFG_SERVICE_NAME);
    if (serverName == null || serverName.isEmpty()) {
      serverName = DEFAULT_SERVICE_NAME;
    }
    final String dlog = config.get(CFG_DOCS_LOC);
    if (dlog == null || dlog.isEmpty()) {
      docsLoc = defaultDocsLoc;
    } else {
      if (!dlog.startsWith("/")) {
        docsLoc = "/" + dlog;
      } else {
        docsLoc = dlog;
      }
    }
    logger =
        new JsonServerSyslog(
            serverName, JsonServerServlet.KB_DEP, JsonServerSyslog.LOG_LEVEL_INFO, false);
    if (sysLogOut != null) {
      logger.changeOutput(sysLogOut);
    }
  }
Esempio n. 2
0
  @Override
  protected void doGet(final HttpServletRequest request, final HttpServletResponse response)
      throws ServletException, IOException {

    final RpcInfo rpc = JsonServerSyslog.getCurrentRpcInfo();
    rpc.setId(("" + Math.random()).substring(2));
    rpc.setIp(JsonServerServlet.getIpAddress(request, config));
    rpc.setMethod("GET");
    logHeaders(request);

    String path = request.getPathInfo();

    if (path == null) { // e.g. /docs
      handle404(request, response);
      return;
    }
    if (path.endsWith("/")) { // e.g. /docs/
      path = path + "index.html";
    }
    // the path is already normalized by the framework, so no need to
    // normalize here
    path = docsLoc + path;
    final InputStream is = getClass().getResourceAsStream(path);
    if (is == null) {
      handle404(request, response);
      return;
    }
    try {
      final byte[] page = IOUtils.toByteArray(is);
      response.getOutputStream().write(page);
    } catch (IOException ioe) {
      logger.logErr(request.getRequestURI() + " 500 " + request.getHeader(USER_AGENT));
      logger.logErr(ioe);
      response.sendError(500);
      return;
    }
    logger.logInfo(request.getRequestURI() + " 200 " + request.getHeader(USER_AGENT));
  }
Esempio n. 3
0
 public int run(
     String methodName,
     File inputFile,
     boolean stdin,
     String inputJson,
     File output,
     String tagVer,
     boolean verbose,
     boolean keepTempFiles,
     String provRefs,
     String mountPoints)
     throws Exception {
   AuthToken auth = AuthService.login(user, password).getToken();
   ////////////////////////////////// Loading image name /////////////////////////////////////
   CatalogClient client = new CatalogClient(catalogUrl);
   String moduleName = methodName.split(Pattern.quote("."))[0];
   ModuleVersion mv =
       client.getModuleVersion(
           new SelectModuleVersion()
               .withModuleName(moduleName)
               .withVersion(tagVer)
               .withIncludeCompilationReport(1L));
   if (mv.getDataVersion() != null)
     throw new IllegalStateException(
         "Reference data is required for module "
             + moduleName
             + ". This feature is not supported for local calls.");
   String dockerImage = mv.getDockerImgName();
   System.out.println("Docker image name recieved from Catalog: " + dockerImage);
   ////////////////////////////////// Standard files in run_local ////////////////////////////
   if (!runDir.exists()) runDir.mkdir();
   File runLocalSh = new File(runDir, "run_local.sh");
   File runDockerSh = new File(runDir, "run_docker.sh");
   if (!runLocalSh.exists()) {
     FileUtils.writeLines(
         runLocalSh,
         Arrays.asList(
             "#!/bin/bash",
             "sdir=\"$(cd \"$(dirname \"$(readlink -f \"$0\")\")\" && pwd)\"",
             "callback_url=$1",
             "cnt_id=$2",
             "docker_image=$3",
             "mount_points=$4",
             "$sdir/run_docker.sh run -v $sdir/workdir:/kb/module/work $mount_points "
                 + "-e \"SDK_CALLBACK_URL=$callback_url\" --name $cnt_id $docker_image async"));
     ProcessHelper.cmd("chmod", "+x", runLocalSh.getCanonicalPath()).exec(runDir);
   }
   if (!runDockerSh.exists()) {
     FileUtils.writeLines(runDockerSh, Arrays.asList("#!/bin/bash", "docker \"$@\""));
     ProcessHelper.cmd("chmod", "+x", runDockerSh.getCanonicalPath()).exec(runDir);
   }
   ////////////////////////////////// Temporary files ////////////////////////////////////////
   StringBuilder mountPointsDocker = new StringBuilder();
   if (mountPoints != null) {
     for (String part : mountPoints.split(Pattern.quote(","))) {
       String[] fromTo = part.split(Pattern.quote(":"));
       String to;
       if (fromTo.length != 2) {
         if (fromTo.length == 1) {
           to = "tmp";
         } else {
           throw new IllegalStateException("Unexpected mount point format: " + part);
         }
       } else {
         to = fromTo[1];
       }
       File fromDir = new File(fromTo[0]);
       if ((!fromDir.exists()) || (!fromDir.isDirectory()))
         throw new IllegalStateException("Mount point directory doesn't exist: " + fromDir);
       String from = fromDir.getCanonicalPath();
       if (!to.startsWith("/")) to = "/kb/module/work/" + to;
       if (mountPointsDocker.length() > 0) mountPointsDocker.append(" ");
       mountPointsDocker.append("-v ").append(from).append(":").append(to);
     }
   }
   File workDir = new File(runDir, "workdir");
   if (workDir.exists()) FileUtils.deleteDirectory(workDir);
   workDir.mkdir();
   File subjobsDir = new File(runDir, "subjobs");
   if (subjobsDir.exists()) FileUtils.deleteDirectory(subjobsDir);
   File tokenFile = new File(workDir, "token");
   try (FileWriter fw = new FileWriter(tokenFile)) {
     fw.write(auth.getToken());
   }
   String jobSrvUrl = kbaseEndpoint + "/userandjobstate";
   String wsUrl = kbaseEndpoint + "/ws";
   String shockUrl = kbaseEndpoint + "/shock-api";
   File configPropsFile = new File(workDir, "config.properties");
   PrintWriter pw = new PrintWriter(configPropsFile);
   try {
     pw.println("[global]");
     pw.println("job_service_url = " + jobSrvUrl);
     pw.println("workspace_url = " + wsUrl);
     pw.println("shock_url = " + shockUrl);
     pw.println("kbase_endpoint = " + kbaseEndpoint);
   } finally {
     pw.close();
   }
   ////////////////////////////////// Preparing input.json ///////////////////////////////////
   String jsonString;
   if (inputFile != null) {
     jsonString = FileUtils.readFileToString(inputFile);
   } else if (inputJson != null) {
     jsonString = inputJson;
   } else if (stdin) {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     IOUtils.copy(System.in, baos);
     jsonString = new String(baos.toByteArray(), Charset.forName("utf-8"));
   } else {
     throw new IllegalStateException("No one input method is used");
   }
   jsonString = jsonString.trim();
   if (!jsonString.startsWith("["))
     jsonString = "[" + jsonString + "]"; // Wrapping one argument by array
   if (verbose) System.out.println("Input parameters: " + jsonString);
   Map<String, Object> rpc = new LinkedHashMap<String, Object>();
   rpc.put("version", "1.1");
   rpc.put("method", methodName);
   List<UObject> params =
       UObject.getMapper().readValue(jsonString, new TypeReference<List<UObject>>() {});
   rpc.put("params", params);
   rpc.put("context", new LinkedHashMap<String, Object>());
   UObject.getMapper().writeValue(new File(workDir, "input.json"), rpc);
   ////////////////////////////////// Starting callback service //////////////////////////////
   int callbackPort = NetUtils.findFreePort();
   URL callbackUrl = CallbackServer.getCallbackUrl(callbackPort, callbackNetworks);
   Server jettyServer = null;
   if (callbackUrl != null) {
     if (System.getProperty("os.name").startsWith("Windows")) {
       JsonServerSyslog.setStaticUseSyslog(false);
       JsonServerSyslog.setStaticMlogFile(new File(workDir, "callback.log").getCanonicalPath());
     }
     CallbackServerConfig cfg =
         new CallbackServerConfigBuilder(
                 new URL(kbaseEndpoint),
                 callbackUrl,
                 runDir.toPath(),
                 new LineLogger() {
                   @Override
                   public void logNextLine(String line, boolean isError) {
                     if (isError) {
                       System.err.println(line);
                     } else {
                       System.out.println(line);
                     }
                   }
                 })
             .build();
     Set<String> releaseTags = new TreeSet<String>();
     if (mv.getReleaseTags() != null) releaseTags.addAll(mv.getReleaseTags());
     String requestedRelease =
         releaseTags.contains("release")
             ? "release"
             : (releaseTags.contains("beta") ? "beta" : "dev");
     final ModuleRunVersion runver =
         new ModuleRunVersion(
             new URL(mv.getGitUrl()),
             new ModuleMethod(methodName),
             mv.getGitCommitHash(),
             mv.getVersion(),
             requestedRelease);
     List<String> inputWsObjects = new ArrayList<String>();
     if (provRefs != null) {
       inputWsObjects.addAll(Arrays.asList(provRefs.split(Pattern.quote(","))));
     }
     JsonServerServlet catalogSrv =
         new SDKCallbackServer(auth, cfg, runver, params, inputWsObjects);
     jettyServer = new Server(callbackPort);
     ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
     context.setContextPath("/");
     jettyServer.setHandler(context);
     context.addServlet(new ServletHolder(catalogSrv), "/*");
     jettyServer.start();
   } else {
     if (callbackNetworks != null && callbackNetworks.length > 0) {
       throw new IllegalStateException(
           "No proper callback IP was found, "
               + "please check callback_networks parameter in configuration");
     }
     System.out.println(
         "WARNING: No callback URL was recieved "
             + "by the job runner. Local callbacks are disabled.");
   }
   ////////////////////////////////// Running Docker /////////////////////////////////////////
   final String containerName =
       "local_" + moduleName.toLowerCase() + "_" + System.currentTimeMillis();
   try {
     System.out.println();
     int exitCode =
         ProcessHelper.cmd(
                 "bash",
                 DirUtils.getFilePath(runLocalSh),
                 callbackUrl.toExternalForm(),
                 containerName,
                 dockerImage,
                 mountPointsDocker.toString())
             .exec(runDir)
             .getExitCode();
     File outputTmpFile = new File(workDir, "output.json");
     if (!outputTmpFile.exists())
       throw new IllegalStateException("Output JSON file was not found");
     FinishJobParams outObj = UObject.getMapper().readValue(outputTmpFile, FinishJobParams.class);
     if (outObj.getError() != null || outObj.getResult() == null) {
       System.out.println();
       if (outObj.getError() == null) {
         System.err.println("Unknown error (no information)");
       } else {
         System.err.println("Error: " + outObj.getError().getMessage());
         if (verbose && outObj.getError().getError() != null) {
           System.err.println("Error details: \n" + outObj.getError().getError());
         }
       }
       System.out.println();
     } else {
       String outputJson = UObject.getMapper().writeValueAsString(outObj.getResult());
       if (output != null) {
         FileUtils.writeStringToFile(output, outputJson);
         System.out.println("Output is saved to file: " + output.getCanonicalPath());
       } else {
         System.out.println();
         System.out.println("Output returned by the method:");
         System.out.println(outputJson);
         System.out.println();
       }
     }
     return exitCode;
   } finally {
     try {
       System.out.println("Deleteing docker container...");
       ProcessHelper.cmd(
               "bash", DirUtils.getFilePath(runDockerSh), "rm", "-v", "-f", containerName)
           .exec(runDir);
     } catch (Exception ex) {
       System.out.println("Error deleting container [" + containerName + "]: " + ex.getMessage());
     }
     if (jettyServer != null) {
       System.out.println("Shutting down callback server...");
       jettyServer.stop();
     }
     if (!keepTempFiles) {
       System.out.println("Deleting temporary files...");
       FileUtils.deleteDirectory(workDir);
       FileUtils.deleteDirectory(subjobsDir);
     }
   }
 }
Esempio n. 4
0
 private void logHeaders(final HttpServletRequest req) {
   final String xFF = req.getHeader(X_FORWARDED_FOR);
   if (xFF != null && !xFF.isEmpty()) {
     logger.logInfo(X_FORWARDED_FOR + ": " + xFF);
   }
 }
Esempio n. 5
0
 private void handle404(final HttpServletRequest request, final HttpServletResponse response)
     throws IOException {
   logger.logErr(request.getRequestURI() + " 404 " + request.getHeader(USER_AGENT));
   response.sendError(404);
 }