/**
   * Basic idea of the test: 0. Setup token KEEP_ALIVE 1. create tokens. 2. register them for
   * renewal - to be cancelled on app complete 3. Complete app. 4. Verify token is alive within the
   * KEEP_ALIVE time 5. Verify token has been cancelled after the KEEP_ALIVE_TIME
   *
   * @throws IOException
   * @throws URISyntaxException
   */
  @Test(timeout = 60000)
  public void testDTKeepAlive1() throws Exception {
    Configuration lconf = new Configuration(conf);
    lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    // Keep tokens alive for 6 seconds.
    lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
    // Try removing tokens every second.
    lconf.setLong(YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS, 1000l);
    DelegationTokenRenewer localDtr = createNewDelegationTokenRenewer(lconf, counter);
    localDtr.init(lconf);
    RMContext mockContext = mock(RMContext.class);
    ClientRMService mockClientRMService = mock(ClientRMService.class);
    when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
    when(mockContext.getDelegationTokenRenewer()).thenReturn(localDtr);
    when(mockContext.getDispatcher()).thenReturn(dispatcher);
    InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
    when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
    localDtr.setRMContext(mockContext);
    localDtr.start();

    MyFS dfs = (MyFS) FileSystem.get(lconf);
    LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + lconf.hashCode());

    Credentials ts = new Credentials();
    // get the delegation tokens
    MyToken token1 = dfs.getDelegationToken(new Text("user1"));

    String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
    ts.addToken(new Text(nn1), token1);

    // register the tokens for renewal
    ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
    localDtr.addApplication(applicationId_0, ts, true, false);
    waitForEventsToGetProcessed(localDtr);
    if (!eventQueue.isEmpty()) {
      Event evt = eventQueue.take();
      if (evt instanceof RMAppEvent) {
        Assert.assertEquals(((RMAppEvent) evt).getType(), RMAppEventType.START);
      } else {
        fail("RMAppEvent.START was expected!!");
      }
    }

    localDtr.applicationFinished(applicationId_0);
    waitForEventsToGetProcessed(localDtr);

    // Token should still be around. Renewal should not fail.
    token1.renew(lconf);

    // Allow the keepalive time to run out
    Thread.sleep(10000l);

    // The token should have been cancelled at this point. Renewal will fail.
    try {
      token1.renew(lconf);
      fail("Renewal of cancelled token should have failed");
    } catch (InvalidToken ite) {
    }
  }
 private MyriadStateStore providesMyriadStateStore() {
   // TODO (sdaingade) Read the implementation class from yml
   // once multiple implementations are available.
   if (rmContext.getStateStore() instanceof MyriadStateStore) {
     return (MyriadStateStore) rmContext.getStateStore();
   }
   return null;
 }
  @Test(timeout = 20000)
  public void testConcurrentAddApplication()
      throws IOException, InterruptedException, BrokenBarrierException {
    final CyclicBarrier startBarrier = new CyclicBarrier(2);
    final CyclicBarrier endBarrier = new CyclicBarrier(2);

    // this token uses barriers to block during renew
    final Credentials creds1 = new Credentials();
    final Token<?> token1 = mock(Token.class);
    creds1.addToken(new Text("token"), token1);
    doReturn(true).when(token1).isManaged();
    doAnswer(
            new Answer<Long>() {
              public Long answer(InvocationOnMock invocation)
                  throws InterruptedException, BrokenBarrierException {
                startBarrier.await();
                endBarrier.await();
                return Long.MAX_VALUE;
              }
            })
        .when(token1)
        .renew(any(Configuration.class));

    // this dummy token fakes renewing
    final Credentials creds2 = new Credentials();
    final Token<?> token2 = mock(Token.class);
    creds2.addToken(new Text("token"), token2);
    doReturn(true).when(token2).isManaged();
    doReturn(Long.MAX_VALUE).when(token2).renew(any(Configuration.class));

    // fire up the renewer
    final DelegationTokenRenewer dtr = createNewDelegationTokenRenewer(conf, counter);
    dtr.init(conf);
    RMContext mockContext = mock(RMContext.class);
    ClientRMService mockClientRMService = mock(ClientRMService.class);
    when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
    InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
    when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
    dtr.setRMContext(mockContext);
    when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
    dtr.start();
    // submit a job that blocks during renewal
    Thread submitThread =
        new Thread() {
          @Override
          public void run() {
            dtr.addApplication(mock(ApplicationId.class), creds1, false, false);
          }
        };
    submitThread.start();

    // wait till 1st submit blocks, then submit another
    startBarrier.await();
    dtr.addApplication(mock(ApplicationId.class), creds2, false, false);
    // signal 1st to complete
    endBarrier.await();
    submitThread.join();
  }
  private void removeApplicationFromRenewal(ApplicationId applicationId) {
    rmContext.getSystemCredentialsForApps().remove(applicationId);
    Set<DelegationTokenToRenew> tokens = appTokens.get(applicationId);

    if (tokens != null && !tokens.isEmpty()) {
      synchronized (tokens) {
        Iterator<DelegationTokenToRenew> it = tokens.iterator();
        while (it.hasNext()) {
          DelegationTokenToRenew dttr = it.next();
          if (LOG.isDebugEnabled()) {
            LOG.debug(
                "Removing delegation token for appId="
                    + applicationId
                    + "; token="
                    + dttr.token.getService());
          }

          // cancel the timer
          if (dttr.timerTask != null) dttr.timerTask.cancel();

          // cancel the token
          cancelToken(dttr);

          it.remove();
        }
      }
    }
  }
  private void requestNewHdfsDelegationToken(
      ApplicationId applicationId, String user, boolean shouldCancelAtEnd)
      throws IOException, InterruptedException {
    // Get new hdfs tokens for this user
    Credentials credentials = new Credentials();
    Token<?>[] newTokens = obtainSystemTokensForUser(user, credentials);

    // Add new tokens to the toRenew list.
    LOG.info(
        "Received new tokens for " + applicationId + ". Received " + newTokens.length + " tokens.");
    if (newTokens.length > 0) {
      for (Token<?> token : newTokens) {
        if (token.isManaged()) {
          DelegationTokenToRenew tokenToRenew =
              new DelegationTokenToRenew(
                  applicationId, token, getConfig(), Time.now(), shouldCancelAtEnd, user);
          // renew the token to get the next expiration date.
          renewToken(tokenToRenew);
          setTimerForTokenRenewal(tokenToRenew);
          appTokens.get(applicationId).add(tokenToRenew);
          LOG.info("Received new token " + token);
        }
      }
    }
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    ByteBuffer byteBuffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    rmContext.getSystemCredentialsForApps().put(applicationId, byteBuffer);
  }
 @Before
 public void setUp() throws Exception {
   counter = new AtomicInteger(0);
   conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
   UserGroupInformation.setConfiguration(conf);
   eventQueue = new LinkedBlockingQueue<Event>();
   dispatcher = new AsyncDispatcher(eventQueue);
   Renewer.reset();
   delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
   delegationTokenRenewer.init(conf);
   RMContext mockContext = mock(RMContext.class);
   ClientRMService mockClientRMService = mock(ClientRMService.class);
   when(mockContext.getDelegationTokenRenewer()).thenReturn(delegationTokenRenewer);
   when(mockContext.getDispatcher()).thenReturn(dispatcher);
   when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
   InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
   when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
   delegationTokenRenewer.setRMContext(mockContext);
   delegationTokenRenewer.start();
 }
  @Before
  public void setup() throws Exception {
    Configuration conf = new Configuration();
    clock = new TestFairScheduler.MockClock();
    scheduler = mock(FairScheduler.class);
    when(scheduler.getConf()).thenReturn(new FairSchedulerConfiguration(conf));
    when(scheduler.getClock()).thenReturn(clock);
    AllocationConfiguration allocConf = new AllocationConfiguration(conf);
    when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);

    queueManager = new QueueManager(scheduler);
    queueManager.initialize(conf);
    queueMaxApps = allocConf.queueMaxApps;
    userMaxApps = allocConf.userMaxApps;
    maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler);
    appNum = 0;
    rmContext = mock(RMContext.class);
    when(rmContext.getEpoch()).thenReturn(0L);
  }
Example #8
0
 @Inject
 AppsList(RequestContext ctx, RMContext rmContext) {
   rc = ctx;
   apps = rmContext.getRMApps();
 }
 // enable RM to short-circuit token operations directly to itself
 private void setLocalSecretManagerAndServiceAddr() {
   RMDelegationTokenIdentifier.Renewer.setSecretManager(
       rmContext.getRMDelegationTokenSecretManager(),
       rmContext.getClientRMService().getBindAddress());
 }