/** Initialize SecondaryNameNode. */
private void initialize(Configuration conf) throws IOException {
// initiate Java VM metrics
JvmMetrics.init("SecondaryNameNode", conf.get("session.id"));
// Create connection to the namenode.
shouldRun = true;
nameNodeAddr = NameNode.getAddress(conf);
this.conf = conf;
this.namenode =
(NamenodeProtocol)
RPC.waitForProxy(
NamenodeProtocol.class, NamenodeProtocol.versionID, nameNodeAddr, conf);
// initialize checkpoint directories
fsName = getInfoServer();
checkpointDirs = FSImage.getCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary");
checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary");
checkpointImage = new CheckpointStorage();
checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs);
// Initialize other scheduling parameters from the configuration
checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600);
checkpointSize = conf.getLong("fs.checkpoint.size", 4194304);
// initialize the webserver for uploading files.
InetSocketAddress infoSocAddr =
NetUtils.createSocketAddr(conf.get("dfs.secondary.http.address", "0.0.0.0:50090"));
infoBindAddress = infoSocAddr.getHostName();
int tmpInfoPort = infoSocAddr.getPort();
infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf);
infoServer.setAttribute("secondary.name.node", this);
infoServer.setAttribute("name.system.image", checkpointImage);
this.infoServer.setAttribute("name.conf", conf);
infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
infoServer.start();
// The web-server port can be ephemeral... ensure we have the correct info
infoPort = infoServer.getPort();
conf.set("dfs.secondary.http.address", infoBindAddress + ":" + infoPort);
LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" + infoPort);
LOG.warn(
"Checkpoint Period :"
+ checkpointPeriod
+ " secs "
+ "("
+ checkpointPeriod / 60
+ " min)");
LOG.warn(
"Log Size Trigger :"
+ checkpointSize
+ " bytes "
+ "("
+ checkpointSize / 1024
+ " KB)");
}
@Override
protected void serviceStart() throws Exception {
try {
proxyServer = new HttpServer("proxy", bindAddress, port, port == 0, getConfig(), acl);
proxyServer.addServlet(
ProxyUriUtils.PROXY_SERVLET_NAME,
ProxyUriUtils.PROXY_PATH_SPEC,
WebAppProxyServlet.class);
proxyServer.setAttribute(FETCHER_ATTRIBUTE, fetcher);
proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, isSecurityEnabled);
proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
proxyServer.start();
} catch (IOException e) {
LOG.fatal("Could not start proxy web server", e);
throw new YarnRuntimeException("Could not start proxy web server", e);
}
super.serviceStart();
}
@Override
public void start() {
// Register servlet with JobTracker's Jetty server
if (taskTrackerManager instanceof JobTracker) {
JobTracker jobTracker = (JobTracker) taskTrackerManager;
HttpServer infoServer = jobTracker.infoServer;
infoServer.setAttribute("booking", this);
infoServer.addServlet("booking", "/booking", BookSchedulerServlet.class);
}
super.start();
}
public void initializeServer() throws IOException {
String serverAddr = conf.get(CLUSTER_BALANCER_ADDR, "localhost:9143");
InetSocketAddress addr = NetUtils.createSocketAddr(serverAddr);
clusterDaemonServer = RPC.getServer(this, addr.getHostName(), addr.getPort(), conf);
clusterDaemonServer.start();
// Http server
String infoServerAddr = conf.get(CLUSTER_HTTP_BALANCER_ADDR, "localhost:50143");
InetSocketAddress infoAddr = NetUtils.createSocketAddr(infoServerAddr);
infoServer =
new HttpServer(
"cb", infoAddr.getHostName(), infoAddr.getPort(), infoAddr.getPort() == 0, conf);
infoServer.setAttribute("cluster.balancer", this);
infoServer.start();
}
public void start() throws IOException {
final String infoHost = bindAddress.getHostName();
int infoPort = bindAddress.getPort();
httpServer =
new HttpServer(
"hdfs",
infoHost,
infoPort,
infoPort == 0,
conf,
new AccessControlList(conf.get(DFS_ADMIN, " "))) {
{
// Add SPNEGO support to NameNode
if (UserGroupInformation.isSecurityEnabled()) {
initSpnego(
conf,
DFSConfigKeys.DFS_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY,
DFSUtil.getSpnegoKeytabKey(conf, DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
}
if (WebHdfsFileSystem.isEnabled(conf, LOG)) {
// add SPNEGO authentication filter for webhdfs
final String name = "SPNEGO";
final String classname = AuthFilter.class.getName();
final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*";
Map<String, String> params = getAuthFilterParams(conf);
defineFilter(webAppContext, name, classname, params, new String[] {pathSpec});
LOG.info("Added filter '" + name + "' (class=" + classname + ")");
// add webhdfs packages
addJerseyResourcePackage(
NamenodeWebHdfsMethods.class.getPackage().getName()
+ ";"
+ Param.class.getPackage().getName(),
pathSpec);
}
}
private Map<String, String> getAuthFilterParams(Configuration conf) throws IOException {
Map<String, String> params = new HashMap<String, String>();
String principalInConf =
conf.get(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY);
if (principalInConf != null && !principalInConf.isEmpty()) {
params.put(
DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
SecurityUtil.getServerPrincipal(principalInConf, bindAddress.getHostName()));
} else if (UserGroupInformation.isSecurityEnabled()) {
LOG.error(
"WebHDFS and security are enabled, but configuration property '"
+ DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY
+ "' is not set.");
}
String httpKeytab =
conf.get(
DFSUtil.getSpnegoKeytabKey(conf, DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
if (httpKeytab != null && !httpKeytab.isEmpty()) {
params.put(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY, httpKeytab);
} else if (UserGroupInformation.isSecurityEnabled()) {
LOG.error(
"WebHDFS and security are enabled, but configuration property '"
+ DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY
+ "' is not set.");
}
return params;
}
};
boolean certSSL = conf.getBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, false);
if (certSSL) {
boolean needClientAuth = conf.getBoolean("dfs.https.need.client.auth", false);
InetSocketAddress secInfoSocAddr =
NetUtils.createSocketAddr(
infoHost
+ ":"
+ conf.get(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, infoHost + ":" + 0));
Configuration sslConf = new Configuration(false);
if (certSSL) {
sslConf.addResource(
conf.get(DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, "ssl-server.xml"));
}
httpServer.addSslListener(secInfoSocAddr, sslConf, needClientAuth);
// assume same ssl port for all datanodes
InetSocketAddress datanodeSslPort =
NetUtils.createSocketAddr(
conf.get(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, infoHost + ":" + 50475));
httpServer.setAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY, datanodeSslPort.getPort());
}
httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, nn);
httpServer.setAttribute(JspHelper.CURRENT_CONF, conf);
setupServlets(httpServer, conf);
httpServer.start();
httpAddress = new InetSocketAddress(bindAddress.getAddress(), httpServer.getPort());
}
/**
* Sets startup progress of namenode for use by servlets.
*
* @param prog StartupProgress to set
*/
public void setStartupProgress(StartupProgress prog) {
httpServer.setAttribute(STARTUP_PROGRESS_ATTRIBUTE_KEY, prog);
}
/**
* Sets address of namenode for use by servlets.
*
* @param nameNodeAddress InetSocketAddress to set
*/
public void setNameNodeAddress(InetSocketAddress nameNodeAddress) {
httpServer.setAttribute(
NAMENODE_ADDRESS_ATTRIBUTE_KEY, NetUtils.getConnectAddress(nameNodeAddress));
}
/**
* Sets fsimage for use by servlets.
*
* @param fsImage FSImage to set
*/
public void setFSImage(FSImage fsImage) {
httpServer.setAttribute(FSIMAGE_ATTRIBUTE_KEY, fsImage);
}