/** Start listening for edits via RPC. */
  public void start() throws IOException {
    Preconditions.checkState(!isStarted(), "JN already running");

    validateAndCreateJournalDir(localDir);

    DefaultMetricsSystem.initialize("JournalNode");
    JvmMetrics.create(
        "JournalNode",
        conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY),
        DefaultMetricsSystem.instance());

    InetSocketAddress socAddr = JournalNodeRpcServer.getAddress(conf);
    SecurityUtil.login(
        conf,
        DFSConfigKeys.DFS_JOURNALNODE_KEYTAB_FILE_KEY,
        DFSConfigKeys.DFS_JOURNALNODE_USER_NAME_KEY,
        socAddr.getHostName());

    registerJNMXBean();

    httpServer = new JournalNodeHttpServer(conf, this);
    httpServer.start();

    rpcServer = new JournalNodeRpcServer(conf, this);
    rpcServer.start();
  }
  @Test
  public void testDistributedLeaderStart() throws Exception {
    RMContextImpl rmContext = new RMContextImpl();
    rmContext.setDistributedEnabled(true);
    rmContext.setHAEnabled(true);
    GroupMembershipService groupMembershipService = new GroupMembershipService(null, rmContext);
    groupMembershipService.init(conf);
    NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf, rmContext);
    try {
      groupMembershipService.start();

      rmContext.setRMGroupMembershipService(groupMembershipService);
      rmContext.setStateStore(new NDBRMStateStore());

      assertTrue("should be leader", rmContext.isLeader());
      nmTokenSecretManager.start();
      assertNotNull("did not roll master key", nmTokenSecretManager.getCurrentKey());

      RMStateStore.RMState state = rmContext.getStateStore().loadState(rmContext);
      assertNotNull(
          "key not persisted to the database",
          state.getSecretTokenMamagerKey(RMStateStore.KeyType.CURRENTNMTOKENMASTERKEY));

      NMTokenSecretManagerInRM nmTokenSecretManager2 =
          new NMTokenSecretManagerInRM(conf, rmContext);
      nmTokenSecretManager2.recover(state);

      assertEquals(nmTokenSecretManager.getCurrentKey(), nmTokenSecretManager2.getCurrentKey());
      assertEquals(nmTokenSecretManager.getNextKey(), nmTokenSecretManager2.getNextKey());
    } finally {
      groupMembershipService.stop();
      nmTokenSecretManager.stop();
      DefaultMetricsSystem.shutdown();
    }
  }
Exemple #3
0
  @Override
  public void start() {
    // / Create the AMInfo for the current AppMaster
    if (amInfos == null) {
      amInfos = new LinkedList<AMInfo>();
    }
    AMInfo amInfo =
        DragonBuilderUtils.newAMInfo(
            appAttemptId, startTime, containerID, nmHost, nmPort, nmHttpPort);
    amInfos.add(amInfo);

    job = createJob(getConfig());

    // metrics system init is really init & start.
    // It's more test friendly to put it here.
    DefaultMetricsSystem.initialize("DragonAppMaster");

    // create a job event for job intialization
    JobEvent initJobEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT);
    // Send init to the job (this does NOT trigger job execution)
    // This is a synchronous call, not an event through dispatcher. We want
    // job-init to be done completely here.
    jobEventDispatcher.handle(initJobEvent);

    super.start();

    // All components have started, start the job.
    startJobs();
  }
  @After
  public void tearDown() {
    if (hostFile != null && hostFile.exists()) {
      hostFile.delete();
    }

    ClusterMetrics.destroy();
    if (rm != null) {
      rm.stop();
    }

    MetricsSystem ms = DefaultMetricsSystem.instance();
    if (ms.getSource("ClusterMetrics") != null) {
      DefaultMetricsSystem.shutdown();
    }
  }
Exemple #5
0
 private static ObjectName getMBeanName(String serviceName, String nameName) {
   ObjectName name = null;
   String nameStr = "Hadoop:service=" + serviceName + ",name=" + nameName;
   try {
     name = DefaultMetricsSystem.newMBeanName(nameStr);
   } catch (Exception e) {
     LOG.warn("Error creating MBean object name: " + nameStr, e);
   }
   return name;
 }
Exemple #6
0
  public static synchronized FSQueueMetrics forQueue(
      String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) {
    MetricsSystem ms = DefaultMetricsSystem.instance();
    QueueMetrics metrics = queueMetrics.get(queueName);
    if (metrics == null) {
      metrics =
          new FSQueueMetrics(ms, queueName, parent, enableUserMetrics, conf)
              .tag(QUEUE_INFO, queueName);

      // Register with the MetricsSystems
      if (ms != null) {
        metrics =
            ms.register(
                sourceName(queueName).toString(), "Metrics for queue: " + queueName, metrics);
      }
      queueMetrics.put(queueName, metrics);
    }

    return (FSQueueMetrics) metrics;
  }
  @SuppressWarnings("unchecked")
  @Override
  public void serviceStart() throws Exception {

    // start all the components
    startServices();
    super.serviceStart();

    this.state = DAGAppMasterState.IDLE;

    // metrics system init is really init & start.
    // It's more test friendly to put it here.
    DefaultMetricsSystem.initialize("DAGAppMaster");

    this.appsStartTime = clock.getTime();
    AMStartedEvent startEvent =
        new AMStartedEvent(appAttemptID, startTime, appsStartTime, appSubmitTime);
    dispatcher.getEventHandler().handle(new DAGHistoryEvent(startEvent));

    this.lastDAGCompletionTime = clock.getTime();

    if (!isSession) {
      startDAG();
    } else {
      LOG.info("In Session mode. Waiting for DAG over RPC");
      this.dagSubmissionTimer = new Timer(true);
      this.dagSubmissionTimer.scheduleAtFixedRate(
          new TimerTask() {
            @Override
            public void run() {
              checkAndHandleSessionTimeout();
            }
          },
          sessionTimeoutInterval,
          sessionTimeoutInterval / 10);
    }
  }
 @Override
 public void init() {
   // Make sure that the metrics system doesn't throw an exception when
   // registering a source with the same name
   DefaultMetricsSystem.setMiniClusterMode(true);
 }
  @Test
  public void testNonLeaderKeyReception()
      throws InterruptedException, StorageInitializtionException, Exception {
    // create a groupMembershipService that will be leader
    RMContextImpl rmContext = new RMContextImpl();
    rmContext.setDistributedEnabled(true);
    rmContext.setHAEnabled(true);
    GroupMembershipService groupMembershipService = new GroupMembershipService(null, rmContext);
    groupMembershipService.init(conf);
    NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf, rmContext);
    // create a resrouce tracker
    conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
    conf.setBoolean(YarnConfiguration.DISTRIBUTED_RM, true);
    MockRM mockRM = new MockRM(conf);
    mockRM.init(conf);

    try {
      groupMembershipService.start();
      rmContext.setRMGroupMembershipService(groupMembershipService);
      rmContext.setStateStore(new NDBRMStateStore());

      while (!rmContext.isLeader()) {
        Thread.sleep(1000);
      }

      mockRM.start();

      if (mockRM.getRMContext().isDistributedEnabled() && !mockRM.getRMContext().isLeader()) {
        conf.set(
            YarnConfiguration.EVENT_RT_CONFIG_PATH, "target/test-classes/RT_EventAPIConfig.ini");
        NdbRtStreamingProcessor rtStreamingProcessor =
            new NdbRtStreamingProcessor(mockRM.getRMContext());
        RMStorageFactory.kickTheNdbEventStreamingAPI(false, conf);
        new Thread(rtStreamingProcessor).start();
      }

      // this should be a resource tracker not a scheduler
      assertFalse(mockRM.getRMContext().isLeader());

      // simulate creation of a token on the sheduler
      nmTokenSecretManager.start();

      assertNotNull("did not roll master key", nmTokenSecretManager.getCurrentKey());
      Thread.sleep(1000);
      dummyUpdate();
      Thread.sleep(1000);
      RMStateStore.RMState state = rmContext.getStateStore().loadState(rmContext);
      assertEquals(
          "key not persisted to the database",
          state.getSecretTokenMamagerKey(RMStateStore.KeyType.CURRENTNMTOKENMASTERKEY),
          nmTokenSecretManager.getCurrentKey());

      assertEquals(
          nmTokenSecretManager.getCurrentKey(),
          mockRM.getRMContext().getNMTokenSecretManager().getCurrentKey());
      assertEquals(
          nmTokenSecretManager.getNextKey(),
          mockRM.getRMContext().getNMTokenSecretManager().getNextKey());
    } finally {
      groupMembershipService.stop();
      mockRM.stop();
      nmTokenSecretManager.stop();
      DefaultMetricsSystem.shutdown();
    }
  }
 @SuppressWarnings("UnusedDeclaration")
 public PullServerAuxService() {
   this(DefaultMetricsSystem.instance());
 }
 static {
   DefaultMetricsSystem.setMiniClusterMode(true);
 }
 @Override
 public void serviceStart() throws Exception {
   super.serviceStart();
   DefaultMetricsSystem.shutdown();
 }
Exemple #13
0
  public static NameNode createNameNode(String argv[], Configuration conf) throws IOException {
    if (conf == null) {
      conf = new HdfsConfiguration();
    }
    StartupOption startOpt = parseArguments(argv);
    if (startOpt == null) {
      printUsage(System.err);
      return null;
    }
    setStartupOption(conf, startOpt);

    switch (startOpt) {
        // HOP
      case DROP_AND_CREATE_DB:
        { // delete everything other than inode and blocks table. this is tmp fix for safe mode
          dropAndCreateDB(conf);
          return null;
        }
      case FORMAT:
        {
          boolean aborted =
              format(conf, startOpt.getForceFormat(), startOpt.getInteractiveFormat());
          terminate(aborted ? 1 : 0);
          return null; // avoid javac warning
        }
      case GENCLUSTERID:
        {
          System.err.println("Generating new cluster id:");
          System.out.println(StorageInfo.newClusterID());
          terminate(0);
          return null;
        }
      case FINALIZE:
        {
          throw new UnsupportedOperationException("HOP: FINALIZE is not supported anymore");
        }
      case BOOTSTRAPSTANDBY:
        {
          throw new UnsupportedOperationException("HOP: BOOTSTRAPSTANDBY is not supported anymore");
        }
      case INITIALIZESHAREDEDITS:
        {
          throw new UnsupportedOperationException(
              "HOP: INITIALIZESHAREDEDITS is not supported anymore");
        }
      case BACKUP:
      case CHECKPOINT:
        {
          throw new UnsupportedOperationException(
              "HOP: BACKUP/CHECKPOINT is not supported anymore");
        }
      case RECOVER:
        {
          new UnsupportedOperationException("Hops. Metadata recovery is not supported");
          return null;
        }
      default:
        {
          DefaultMetricsSystem.initialize("NameNode");
          return new NameNode(conf);
        }
    }
  }
 @After
 public void tearDown() {
   QueueMetrics.clearQueueMetrics();
   DefaultMetricsSystem.shutdown();
 }