Exemplo n.º 1
0
  /** Initialize SecondaryNameNode. */
  private void initialize(Configuration conf) throws IOException {
    // initiate Java VM metrics
    JvmMetrics.init("SecondaryNameNode", conf.get("session.id"));

    // Create connection to the namenode.
    shouldRun = true;
    nameNodeAddr = NameNode.getAddress(conf);

    this.conf = conf;
    this.namenode =
        (NamenodeProtocol)
            RPC.waitForProxy(
                NamenodeProtocol.class, NamenodeProtocol.versionID, nameNodeAddr, conf);

    // initialize checkpoint directories
    fsName = getInfoServer();
    checkpointDirs = FSImage.getCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary");
    checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary");
    checkpointImage = new CheckpointStorage();
    checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs);

    // Initialize other scheduling parameters from the configuration
    checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600);
    checkpointSize = conf.getLong("fs.checkpoint.size", 4194304);

    // initialize the webserver for uploading files.
    InetSocketAddress infoSocAddr =
        NetUtils.createSocketAddr(conf.get("dfs.secondary.http.address", "0.0.0.0:50090"));
    infoBindAddress = infoSocAddr.getHostName();
    int tmpInfoPort = infoSocAddr.getPort();
    infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf);
    infoServer.setAttribute("secondary.name.node", this);
    infoServer.setAttribute("name.system.image", checkpointImage);
    this.infoServer.setAttribute("name.conf", conf);
    infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
    infoServer.start();

    // The web-server port can be ephemeral... ensure we have the correct info
    infoPort = infoServer.getPort();
    conf.set("dfs.secondary.http.address", infoBindAddress + ":" + infoPort);
    LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" + infoPort);
    LOG.warn(
        "Checkpoint Period   :"
            + checkpointPeriod
            + " secs "
            + "("
            + checkpointPeriod / 60
            + " min)");
    LOG.warn(
        "Log Size Trigger    :"
            + checkpointSize
            + " bytes "
            + "("
            + checkpointSize / 1024
            + " KB)");
  }
Exemplo n.º 2
0
 @Override
 protected void serviceStart() throws Exception {
   try {
     proxyServer = new HttpServer("proxy", bindAddress, port, port == 0, getConfig(), acl);
     proxyServer.addServlet(
         ProxyUriUtils.PROXY_SERVLET_NAME,
         ProxyUriUtils.PROXY_PATH_SPEC,
         WebAppProxyServlet.class);
     proxyServer.setAttribute(FETCHER_ATTRIBUTE, fetcher);
     proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, isSecurityEnabled);
     proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
     proxyServer.start();
   } catch (IOException e) {
     LOG.fatal("Could not start proxy web server", e);
     throw new YarnRuntimeException("Could not start proxy web server", e);
   }
   super.serviceStart();
 }
Exemplo n.º 3
0
 @Override
 public void start() {
   // Register servlet with JobTracker's Jetty server
   if (taskTrackerManager instanceof JobTracker) {
     JobTracker jobTracker = (JobTracker) taskTrackerManager;
     HttpServer infoServer = jobTracker.infoServer;
     infoServer.setAttribute("booking", this);
     infoServer.addServlet("booking", "/booking", BookSchedulerServlet.class);
   }
   super.start();
 }
  public void initializeServer() throws IOException {

    String serverAddr = conf.get(CLUSTER_BALANCER_ADDR, "localhost:9143");
    InetSocketAddress addr = NetUtils.createSocketAddr(serverAddr);
    clusterDaemonServer = RPC.getServer(this, addr.getHostName(), addr.getPort(), conf);
    clusterDaemonServer.start();

    // Http server
    String infoServerAddr = conf.get(CLUSTER_HTTP_BALANCER_ADDR, "localhost:50143");
    InetSocketAddress infoAddr = NetUtils.createSocketAddr(infoServerAddr);
    infoServer =
        new HttpServer(
            "cb", infoAddr.getHostName(), infoAddr.getPort(), infoAddr.getPort() == 0, conf);
    infoServer.setAttribute("cluster.balancer", this);
    infoServer.start();
  }
Exemplo n.º 5
0
  public void start() throws IOException {
    final String infoHost = bindAddress.getHostName();
    int infoPort = bindAddress.getPort();

    httpServer =
        new HttpServer(
            "hdfs",
            infoHost,
            infoPort,
            infoPort == 0,
            conf,
            new AccessControlList(conf.get(DFS_ADMIN, " "))) {
          {
            // Add SPNEGO support to NameNode
            if (UserGroupInformation.isSecurityEnabled()) {
              initSpnego(
                  conf,
                  DFSConfigKeys.DFS_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY,
                  DFSUtil.getSpnegoKeytabKey(conf, DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
            }
            if (WebHdfsFileSystem.isEnabled(conf, LOG)) {
              // add SPNEGO authentication filter for webhdfs
              final String name = "SPNEGO";
              final String classname = AuthFilter.class.getName();
              final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*";
              Map<String, String> params = getAuthFilterParams(conf);
              defineFilter(webAppContext, name, classname, params, new String[] {pathSpec});
              LOG.info("Added filter '" + name + "' (class=" + classname + ")");

              // add webhdfs packages
              addJerseyResourcePackage(
                  NamenodeWebHdfsMethods.class.getPackage().getName()
                      + ";"
                      + Param.class.getPackage().getName(),
                  pathSpec);
            }
          }

          private Map<String, String> getAuthFilterParams(Configuration conf) throws IOException {
            Map<String, String> params = new HashMap<String, String>();
            String principalInConf =
                conf.get(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY);
            if (principalInConf != null && !principalInConf.isEmpty()) {
              params.put(
                  DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
                  SecurityUtil.getServerPrincipal(principalInConf, bindAddress.getHostName()));
            } else if (UserGroupInformation.isSecurityEnabled()) {
              LOG.error(
                  "WebHDFS and security are enabled, but configuration property '"
                      + DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY
                      + "' is not set.");
            }
            String httpKeytab =
                conf.get(
                    DFSUtil.getSpnegoKeytabKey(conf, DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
            if (httpKeytab != null && !httpKeytab.isEmpty()) {
              params.put(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY, httpKeytab);
            } else if (UserGroupInformation.isSecurityEnabled()) {
              LOG.error(
                  "WebHDFS and security are enabled, but configuration property '"
                      + DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY
                      + "' is not set.");
            }
            return params;
          }
        };

    boolean certSSL = conf.getBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, false);
    if (certSSL) {
      boolean needClientAuth = conf.getBoolean("dfs.https.need.client.auth", false);
      InetSocketAddress secInfoSocAddr =
          NetUtils.createSocketAddr(
              infoHost
                  + ":"
                  + conf.get(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, infoHost + ":" + 0));
      Configuration sslConf = new Configuration(false);
      if (certSSL) {
        sslConf.addResource(
            conf.get(DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, "ssl-server.xml"));
      }
      httpServer.addSslListener(secInfoSocAddr, sslConf, needClientAuth);
      // assume same ssl port for all datanodes
      InetSocketAddress datanodeSslPort =
          NetUtils.createSocketAddr(
              conf.get(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, infoHost + ":" + 50475));
      httpServer.setAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY, datanodeSslPort.getPort());
    }
    httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, nn);
    httpServer.setAttribute(JspHelper.CURRENT_CONF, conf);
    setupServlets(httpServer, conf);
    httpServer.start();
    httpAddress = new InetSocketAddress(bindAddress.getAddress(), httpServer.getPort());
  }
Exemplo n.º 6
0
 /**
  * Sets startup progress of namenode for use by servlets.
  *
  * @param prog StartupProgress to set
  */
 public void setStartupProgress(StartupProgress prog) {
   httpServer.setAttribute(STARTUP_PROGRESS_ATTRIBUTE_KEY, prog);
 }
Exemplo n.º 7
0
 /**
  * Sets address of namenode for use by servlets.
  *
  * @param nameNodeAddress InetSocketAddress to set
  */
 public void setNameNodeAddress(InetSocketAddress nameNodeAddress) {
   httpServer.setAttribute(
       NAMENODE_ADDRESS_ATTRIBUTE_KEY, NetUtils.getConnectAddress(nameNodeAddress));
 }
Exemplo n.º 8
0
 /**
  * Sets fsimage for use by servlets.
  *
  * @param fsImage FSImage to set
  */
 public void setFSImage(FSImage fsImage) {
   httpServer.setAttribute(FSIMAGE_ATTRIBUTE_KEY, fsImage);
 }