/** Start listening for edits via RPC. */
public void start() throws IOException {
Preconditions.checkState(!isStarted(), "JN already running");
validateAndCreateJournalDir(localDir);
DefaultMetricsSystem.initialize("JournalNode");
JvmMetrics.create(
"JournalNode",
conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY),
DefaultMetricsSystem.instance());
InetSocketAddress socAddr = JournalNodeRpcServer.getAddress(conf);
SecurityUtil.login(
conf,
DFSConfigKeys.DFS_JOURNALNODE_KEYTAB_FILE_KEY,
DFSConfigKeys.DFS_JOURNALNODE_USER_NAME_KEY,
socAddr.getHostName());
registerJNMXBean();
httpServer = new JournalNodeHttpServer(conf, this);
httpServer.start();
rpcServer = new JournalNodeRpcServer(conf, this);
rpcServer.start();
}
@Test
public void testDistributedLeaderStart() throws Exception {
RMContextImpl rmContext = new RMContextImpl();
rmContext.setDistributedEnabled(true);
rmContext.setHAEnabled(true);
GroupMembershipService groupMembershipService = new GroupMembershipService(null, rmContext);
groupMembershipService.init(conf);
NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf, rmContext);
try {
groupMembershipService.start();
rmContext.setRMGroupMembershipService(groupMembershipService);
rmContext.setStateStore(new NDBRMStateStore());
assertTrue("should be leader", rmContext.isLeader());
nmTokenSecretManager.start();
assertNotNull("did not roll master key", nmTokenSecretManager.getCurrentKey());
RMStateStore.RMState state = rmContext.getStateStore().loadState(rmContext);
assertNotNull(
"key not persisted to the database",
state.getSecretTokenMamagerKey(RMStateStore.KeyType.CURRENTNMTOKENMASTERKEY));
NMTokenSecretManagerInRM nmTokenSecretManager2 =
new NMTokenSecretManagerInRM(conf, rmContext);
nmTokenSecretManager2.recover(state);
assertEquals(nmTokenSecretManager.getCurrentKey(), nmTokenSecretManager2.getCurrentKey());
assertEquals(nmTokenSecretManager.getNextKey(), nmTokenSecretManager2.getNextKey());
} finally {
groupMembershipService.stop();
nmTokenSecretManager.stop();
DefaultMetricsSystem.shutdown();
}
}
@Override
public void start() {
// / Create the AMInfo for the current AppMaster
if (amInfos == null) {
amInfos = new LinkedList<AMInfo>();
}
AMInfo amInfo =
DragonBuilderUtils.newAMInfo(
appAttemptId, startTime, containerID, nmHost, nmPort, nmHttpPort);
amInfos.add(amInfo);
job = createJob(getConfig());
// metrics system init is really init & start.
// It's more test friendly to put it here.
DefaultMetricsSystem.initialize("DragonAppMaster");
// create a job event for job intialization
JobEvent initJobEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT);
// Send init to the job (this does NOT trigger job execution)
// This is a synchronous call, not an event through dispatcher. We want
// job-init to be done completely here.
jobEventDispatcher.handle(initJobEvent);
super.start();
// All components have started, start the job.
startJobs();
}
@After
public void tearDown() {
if (hostFile != null && hostFile.exists()) {
hostFile.delete();
}
ClusterMetrics.destroy();
if (rm != null) {
rm.stop();
}
MetricsSystem ms = DefaultMetricsSystem.instance();
if (ms.getSource("ClusterMetrics") != null) {
DefaultMetricsSystem.shutdown();
}
}
private static ObjectName getMBeanName(String serviceName, String nameName) {
ObjectName name = null;
String nameStr = "Hadoop:service=" + serviceName + ",name=" + nameName;
try {
name = DefaultMetricsSystem.newMBeanName(nameStr);
} catch (Exception e) {
LOG.warn("Error creating MBean object name: " + nameStr, e);
}
return name;
}
public static synchronized FSQueueMetrics forQueue(
String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) {
MetricsSystem ms = DefaultMetricsSystem.instance();
QueueMetrics metrics = queueMetrics.get(queueName);
if (metrics == null) {
metrics =
new FSQueueMetrics(ms, queueName, parent, enableUserMetrics, conf)
.tag(QUEUE_INFO, queueName);
// Register with the MetricsSystems
if (ms != null) {
metrics =
ms.register(
sourceName(queueName).toString(), "Metrics for queue: " + queueName, metrics);
}
queueMetrics.put(queueName, metrics);
}
return (FSQueueMetrics) metrics;
}
@SuppressWarnings("unchecked")
@Override
public void serviceStart() throws Exception {
// start all the components
startServices();
super.serviceStart();
this.state = DAGAppMasterState.IDLE;
// metrics system init is really init & start.
// It's more test friendly to put it here.
DefaultMetricsSystem.initialize("DAGAppMaster");
this.appsStartTime = clock.getTime();
AMStartedEvent startEvent =
new AMStartedEvent(appAttemptID, startTime, appsStartTime, appSubmitTime);
dispatcher.getEventHandler().handle(new DAGHistoryEvent(startEvent));
this.lastDAGCompletionTime = clock.getTime();
if (!isSession) {
startDAG();
} else {
LOG.info("In Session mode. Waiting for DAG over RPC");
this.dagSubmissionTimer = new Timer(true);
this.dagSubmissionTimer.scheduleAtFixedRate(
new TimerTask() {
@Override
public void run() {
checkAndHandleSessionTimeout();
}
},
sessionTimeoutInterval,
sessionTimeoutInterval / 10);
}
}
@Override
public void init() {
// Make sure that the metrics system doesn't throw an exception when
// registering a source with the same name
DefaultMetricsSystem.setMiniClusterMode(true);
}
@Test
public void testNonLeaderKeyReception()
throws InterruptedException, StorageInitializtionException, Exception {
// create a groupMembershipService that will be leader
RMContextImpl rmContext = new RMContextImpl();
rmContext.setDistributedEnabled(true);
rmContext.setHAEnabled(true);
GroupMembershipService groupMembershipService = new GroupMembershipService(null, rmContext);
groupMembershipService.init(conf);
NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf, rmContext);
// create a resrouce tracker
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.setBoolean(YarnConfiguration.DISTRIBUTED_RM, true);
MockRM mockRM = new MockRM(conf);
mockRM.init(conf);
try {
groupMembershipService.start();
rmContext.setRMGroupMembershipService(groupMembershipService);
rmContext.setStateStore(new NDBRMStateStore());
while (!rmContext.isLeader()) {
Thread.sleep(1000);
}
mockRM.start();
if (mockRM.getRMContext().isDistributedEnabled() && !mockRM.getRMContext().isLeader()) {
conf.set(
YarnConfiguration.EVENT_RT_CONFIG_PATH, "target/test-classes/RT_EventAPIConfig.ini");
NdbRtStreamingProcessor rtStreamingProcessor =
new NdbRtStreamingProcessor(mockRM.getRMContext());
RMStorageFactory.kickTheNdbEventStreamingAPI(false, conf);
new Thread(rtStreamingProcessor).start();
}
// this should be a resource tracker not a scheduler
assertFalse(mockRM.getRMContext().isLeader());
// simulate creation of a token on the sheduler
nmTokenSecretManager.start();
assertNotNull("did not roll master key", nmTokenSecretManager.getCurrentKey());
Thread.sleep(1000);
dummyUpdate();
Thread.sleep(1000);
RMStateStore.RMState state = rmContext.getStateStore().loadState(rmContext);
assertEquals(
"key not persisted to the database",
state.getSecretTokenMamagerKey(RMStateStore.KeyType.CURRENTNMTOKENMASTERKEY),
nmTokenSecretManager.getCurrentKey());
assertEquals(
nmTokenSecretManager.getCurrentKey(),
mockRM.getRMContext().getNMTokenSecretManager().getCurrentKey());
assertEquals(
nmTokenSecretManager.getNextKey(),
mockRM.getRMContext().getNMTokenSecretManager().getNextKey());
} finally {
groupMembershipService.stop();
mockRM.stop();
nmTokenSecretManager.stop();
DefaultMetricsSystem.shutdown();
}
}
@SuppressWarnings("UnusedDeclaration")
public PullServerAuxService() {
this(DefaultMetricsSystem.instance());
}
static {
DefaultMetricsSystem.setMiniClusterMode(true);
}
@Override
public void serviceStart() throws Exception {
super.serviceStart();
DefaultMetricsSystem.shutdown();
}
public static NameNode createNameNode(String argv[], Configuration conf) throws IOException {
if (conf == null) {
conf = new HdfsConfiguration();
}
StartupOption startOpt = parseArguments(argv);
if (startOpt == null) {
printUsage(System.err);
return null;
}
setStartupOption(conf, startOpt);
switch (startOpt) {
// HOP
case DROP_AND_CREATE_DB:
{ // delete everything other than inode and blocks table. this is tmp fix for safe mode
dropAndCreateDB(conf);
return null;
}
case FORMAT:
{
boolean aborted =
format(conf, startOpt.getForceFormat(), startOpt.getInteractiveFormat());
terminate(aborted ? 1 : 0);
return null; // avoid javac warning
}
case GENCLUSTERID:
{
System.err.println("Generating new cluster id:");
System.out.println(StorageInfo.newClusterID());
terminate(0);
return null;
}
case FINALIZE:
{
throw new UnsupportedOperationException("HOP: FINALIZE is not supported anymore");
}
case BOOTSTRAPSTANDBY:
{
throw new UnsupportedOperationException("HOP: BOOTSTRAPSTANDBY is not supported anymore");
}
case INITIALIZESHAREDEDITS:
{
throw new UnsupportedOperationException(
"HOP: INITIALIZESHAREDEDITS is not supported anymore");
}
case BACKUP:
case CHECKPOINT:
{
throw new UnsupportedOperationException(
"HOP: BACKUP/CHECKPOINT is not supported anymore");
}
case RECOVER:
{
new UnsupportedOperationException("Hops. Metadata recovery is not supported");
return null;
}
default:
{
DefaultMetricsSystem.initialize("NameNode");
return new NameNode(conf);
}
}
}
@After
public void tearDown() {
QueueMetrics.clearQueueMetrics();
DefaultMetricsSystem.shutdown();
}
