private static void setupServlets(HttpServer httpServer, Configuration conf) {
httpServer.addInternalServlet(
"startupProgress", StartupProgressServlet.PATH_SPEC, StartupProgressServlet.class);
httpServer.addInternalServlet(
"getDelegationToken",
GetDelegationTokenServlet.PATH_SPEC,
GetDelegationTokenServlet.class,
true);
httpServer.addInternalServlet(
"renewDelegationToken",
RenewDelegationTokenServlet.PATH_SPEC,
RenewDelegationTokenServlet.class,
true);
httpServer.addInternalServlet(
"cancelDelegationToken",
CancelDelegationTokenServlet.PATH_SPEC,
CancelDelegationTokenServlet.class,
true);
httpServer.addInternalServlet("fsck", "/fsck", FsckServlet.class, true);
httpServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class, true);
httpServer.addInternalServlet("listPaths", "/listPaths/*", ListPathsServlet.class, false);
httpServer.addInternalServlet("data", "/data/*", FileDataServlet.class, false);
httpServer.addInternalServlet(
"checksum", "/fileChecksum/*", FileChecksumServlets.RedirectServlet.class, false);
httpServer.addInternalServlet(
"contentSummary", "/contentSummary/*", ContentSummaryServlet.class, false);
}
/** Initialize SecondaryNameNode. */
private void initialize(Configuration conf) throws IOException {
// initiate Java VM metrics
JvmMetrics.init("SecondaryNameNode", conf.get("session.id"));
// Create connection to the namenode.
shouldRun = true;
nameNodeAddr = NameNode.getAddress(conf);
this.conf = conf;
this.namenode =
(NamenodeProtocol)
RPC.waitForProxy(
NamenodeProtocol.class, NamenodeProtocol.versionID, nameNodeAddr, conf);
// initialize checkpoint directories
fsName = getInfoServer();
checkpointDirs = FSImage.getCheckpointDirs(conf, "/tmp/hadoop/dfs/namesecondary");
checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, "/tmp/hadoop/dfs/namesecondary");
checkpointImage = new CheckpointStorage(conf);
checkpointImage.recoverCreate(checkpointDirs, checkpointEditsDirs);
// Initialize other scheduling parameters from the configuration
checkpointPeriod = conf.getLong("fs.checkpoint.period", 3600);
checkpointSize = conf.getLong("fs.checkpoint.size", 4194304);
// initialize the webserver for uploading files.
String infoAddr =
NetUtils.getServerAddress(
conf,
"dfs.secondary.info.bindAddress",
"dfs.secondary.info.port",
"dfs.secondary.http.address");
InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
infoBindAddress = infoSocAddr.getHostName();
int tmpInfoPort = infoSocAddr.getPort();
infoServer = new HttpServer("secondary", infoBindAddress, tmpInfoPort, tmpInfoPort == 0, conf);
infoServer.setAttribute("name.system.image", checkpointImage);
this.infoServer.setAttribute("name.conf", conf);
infoServer.addInternalServlet("getimage", "/getimage", GetImageServlet.class);
infoServer.start();
// The web-server port can be ephemeral... ensure we have the correct info
infoPort = infoServer.getPort();
conf.set("dfs.secondary.http.address", infoBindAddress + ":" + infoPort);
LOG.info("Secondary Web-server up at: " + infoBindAddress + ":" + infoPort);
LOG.warn(
"Checkpoint Period :"
+ checkpointPeriod
+ " secs "
+ "("
+ checkpointPeriod / 60
+ " min)");
LOG.warn(
"Log Size Trigger :"
+ checkpointSize
+ " bytes "
+ "("
+ checkpointSize / 1024
+ " KB)");
}
@Override
public void start() {
// Register servlet with JobTracker's Jetty server
if (taskTrackerManager instanceof JobTracker) {
JobTracker jobTracker = (JobTracker) taskTrackerManager;
HttpServer infoServer = jobTracker.infoServer;
infoServer.setAttribute("booking", this);
infoServer.addServlet("booking", "/booking", BookSchedulerServlet.class);
}
super.start();
}
public void initializeServer() throws IOException {
String serverAddr = conf.get(CLUSTER_BALANCER_ADDR, "localhost:9143");
InetSocketAddress addr = NetUtils.createSocketAddr(serverAddr);
clusterDaemonServer = RPC.getServer(this, addr.getHostName(), addr.getPort(), conf);
clusterDaemonServer.start();
// Http server
String infoServerAddr = conf.get(CLUSTER_HTTP_BALANCER_ADDR, "localhost:50143");
InetSocketAddress infoAddr = NetUtils.createSocketAddr(infoServerAddr);
infoServer =
new HttpServer(
"cb", infoAddr.getHostName(), infoAddr.getPort(), infoAddr.getPort() == 0, conf);
infoServer.setAttribute("cluster.balancer", this);
infoServer.start();
}
public void join() {
if (proxyServer != null) {
try {
proxyServer.join();
} catch (InterruptedException e) {
}
}
}
@Override
protected void serviceStart() throws Exception {
try {
proxyServer = new HttpServer("proxy", bindAddress, port, port == 0, getConfig(), acl);
proxyServer.addServlet(
ProxyUriUtils.PROXY_SERVLET_NAME,
ProxyUriUtils.PROXY_PATH_SPEC,
WebAppProxyServlet.class);
proxyServer.setAttribute(FETCHER_ATTRIBUTE, fetcher);
proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, isSecurityEnabled);
proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
proxyServer.start();
} catch (IOException e) {
LOG.fatal("Could not start proxy web server", e);
throw new YarnRuntimeException("Could not start proxy web server", e);
}
super.serviceStart();
}
@Override
protected void serviceStop() throws Exception {
if (proxyServer != null) {
try {
proxyServer.stop();
} catch (Exception e) {
LOG.fatal("Error stopping proxy web server", e);
throw new YarnRuntimeException("Error stopping proxy web server", e);
}
}
super.serviceStop();
}
/** Shut down this instance of the datanode. Returns only after shutdown is complete. */
public void shutdown() {
shouldRun = false;
try {
if (infoServer != null) infoServer.stop();
} catch (Exception e) {
LOG.warn("Exception shutting down SecondaryNameNode", e);
}
try {
if (checkpointImage != null) checkpointImage.close();
} catch (IOException e) {
LOG.warn(StringUtils.stringifyException(e));
}
}
public void start() throws IOException {
final String infoHost = bindAddress.getHostName();
int infoPort = bindAddress.getPort();
httpServer =
new HttpServer(
"hdfs",
infoHost,
infoPort,
infoPort == 0,
conf,
new AccessControlList(conf.get(DFS_ADMIN, " "))) {
{
// Add SPNEGO support to NameNode
if (UserGroupInformation.isSecurityEnabled()) {
initSpnego(
conf,
DFSConfigKeys.DFS_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY,
DFSUtil.getSpnegoKeytabKey(conf, DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
}
if (WebHdfsFileSystem.isEnabled(conf, LOG)) {
// add SPNEGO authentication filter for webhdfs
final String name = "SPNEGO";
final String classname = AuthFilter.class.getName();
final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*";
Map<String, String> params = getAuthFilterParams(conf);
defineFilter(webAppContext, name, classname, params, new String[] {pathSpec});
LOG.info("Added filter '" + name + "' (class=" + classname + ")");
// add webhdfs packages
addJerseyResourcePackage(
NamenodeWebHdfsMethods.class.getPackage().getName()
+ ";"
+ Param.class.getPackage().getName(),
pathSpec);
}
}
private Map<String, String> getAuthFilterParams(Configuration conf) throws IOException {
Map<String, String> params = new HashMap<String, String>();
String principalInConf =
conf.get(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY);
if (principalInConf != null && !principalInConf.isEmpty()) {
params.put(
DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
SecurityUtil.getServerPrincipal(principalInConf, bindAddress.getHostName()));
} else if (UserGroupInformation.isSecurityEnabled()) {
LOG.error(
"WebHDFS and security are enabled, but configuration property '"
+ DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY
+ "' is not set.");
}
String httpKeytab =
conf.get(
DFSUtil.getSpnegoKeytabKey(conf, DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
if (httpKeytab != null && !httpKeytab.isEmpty()) {
params.put(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY, httpKeytab);
} else if (UserGroupInformation.isSecurityEnabled()) {
LOG.error(
"WebHDFS and security are enabled, but configuration property '"
+ DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY
+ "' is not set.");
}
return params;
}
};
boolean certSSL = conf.getBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, false);
if (certSSL) {
boolean needClientAuth = conf.getBoolean("dfs.https.need.client.auth", false);
InetSocketAddress secInfoSocAddr =
NetUtils.createSocketAddr(
infoHost
+ ":"
+ conf.get(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, infoHost + ":" + 0));
Configuration sslConf = new Configuration(false);
if (certSSL) {
sslConf.addResource(
conf.get(DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, "ssl-server.xml"));
}
httpServer.addSslListener(secInfoSocAddr, sslConf, needClientAuth);
// assume same ssl port for all datanodes
InetSocketAddress datanodeSslPort =
NetUtils.createSocketAddr(
conf.get(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, infoHost + ":" + 50475));
httpServer.setAttribute(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY, datanodeSslPort.getPort());
}
httpServer.setAttribute(NAMENODE_ATTRIBUTE_KEY, nn);
httpServer.setAttribute(JspHelper.CURRENT_CONF, conf);
setupServlets(httpServer, conf);
httpServer.start();
httpAddress = new InetSocketAddress(bindAddress.getAddress(), httpServer.getPort());
}
/**
* Sets startup progress of namenode for use by servlets.
*
* @param prog StartupProgress to set
*/
public void setStartupProgress(StartupProgress prog) {
httpServer.setAttribute(STARTUP_PROGRESS_ATTRIBUTE_KEY, prog);
}
/**
* Sets address of namenode for use by servlets.
*
* @param nameNodeAddress InetSocketAddress to set
*/
public void setNameNodeAddress(InetSocketAddress nameNodeAddress) {
httpServer.setAttribute(
NAMENODE_ADDRESS_ATTRIBUTE_KEY, NetUtils.getConnectAddress(nameNodeAddress));
}
/**
* Sets fsimage for use by servlets.
*
* @param fsImage FSImage to set
*/
public void setFSImage(FSImage fsImage) {
httpServer.setAttribute(FSIMAGE_ATTRIBUTE_KEY, fsImage);
}
public void stop() throws Exception {
if (httpServer != null) {
httpServer.stop();
}
}
protected void tearDown() throws Exception {
server.stop();
}
protected void setUp() throws Exception {
server = new HttpServer("jmx", "0.0.0.0", 0, true);
server.start();
baseUrl = new URL("http://localhost:" + server.getPort() + "/");
}
