/** * Basic idea of the test: 0. Setup token KEEP_ALIVE 1. create tokens. 2. register them for * renewal - to be cancelled on app complete 3. Complete app. 4. Verify token is alive within the * KEEP_ALIVE time 5. Verify token has been cancelled after the KEEP_ALIVE_TIME * * @throws IOException * @throws URISyntaxException */ @Test(timeout = 60000) public void testDTKeepAlive1() throws Exception { Configuration lconf = new Configuration(conf); lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); // Keep tokens alive for 6 seconds. lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l); // Try removing tokens every second. lconf.setLong(YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS, 1000l); DelegationTokenRenewer localDtr = createNewDelegationTokenRenewer(lconf, counter); localDtr.init(lconf); RMContext mockContext = mock(RMContext.class); ClientRMService mockClientRMService = mock(ClientRMService.class); when(mockContext.getClientRMService()).thenReturn(mockClientRMService); when(mockContext.getDelegationTokenRenewer()).thenReturn(localDtr); when(mockContext.getDispatcher()).thenReturn(dispatcher); InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234); when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); localDtr.setRMContext(mockContext); localDtr.start(); MyFS dfs = (MyFS) FileSystem.get(lconf); LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + lconf.hashCode()); Credentials ts = new Credentials(); // get the delegation tokens MyToken token1 = dfs.getDelegationToken(new Text("user1")); String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0"; ts.addToken(new Text(nn1), token1); // register the tokens for renewal ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0); localDtr.addApplication(applicationId_0, ts, true, false); waitForEventsToGetProcessed(localDtr); if (!eventQueue.isEmpty()) { Event evt = eventQueue.take(); if (evt instanceof RMAppEvent) { Assert.assertEquals(((RMAppEvent) evt).getType(), RMAppEventType.START); } else { fail("RMAppEvent.START was expected!!"); } } localDtr.applicationFinished(applicationId_0); waitForEventsToGetProcessed(localDtr); // Token should still be around. Renewal should not fail. token1.renew(lconf); // Allow the keepalive time to run out Thread.sleep(10000l); // The token should have been cancelled at this point. Renewal will fail. try { token1.renew(lconf); fail("Renewal of cancelled token should have failed"); } catch (InvalidToken ite) { } }
@Test(timeout = 20000) public void testConcurrentAddApplication() throws IOException, InterruptedException, BrokenBarrierException { final CyclicBarrier startBarrier = new CyclicBarrier(2); final CyclicBarrier endBarrier = new CyclicBarrier(2); // this token uses barriers to block during renew final Credentials creds1 = new Credentials(); final Token<?> token1 = mock(Token.class); creds1.addToken(new Text("token"), token1); doReturn(true).when(token1).isManaged(); doAnswer( new Answer<Long>() { public Long answer(InvocationOnMock invocation) throws InterruptedException, BrokenBarrierException { startBarrier.await(); endBarrier.await(); return Long.MAX_VALUE; } }) .when(token1) .renew(any(Configuration.class)); // this dummy token fakes renewing final Credentials creds2 = new Credentials(); final Token<?> token2 = mock(Token.class); creds2.addToken(new Text("token"), token2); doReturn(true).when(token2).isManaged(); doReturn(Long.MAX_VALUE).when(token2).renew(any(Configuration.class)); // fire up the renewer final DelegationTokenRenewer dtr = createNewDelegationTokenRenewer(conf, counter); dtr.init(conf); RMContext mockContext = mock(RMContext.class); ClientRMService mockClientRMService = mock(ClientRMService.class); when(mockContext.getClientRMService()).thenReturn(mockClientRMService); InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234); when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); dtr.setRMContext(mockContext); when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr); dtr.start(); // submit a job that blocks during renewal Thread submitThread = new Thread() { @Override public void run() { dtr.addApplication(mock(ApplicationId.class), creds1, false, false); } }; submitThread.start(); // wait till 1st submit blocks, then submit another startBarrier.await(); dtr.addApplication(mock(ApplicationId.class), creds2, false, false); // signal 1st to complete endBarrier.await(); submitThread.join(); }
@Before public void setUp() throws Exception { counter = new AtomicInteger(0); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); eventQueue = new LinkedBlockingQueue<Event>(); dispatcher = new AsyncDispatcher(eventQueue); Renewer.reset(); delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter); delegationTokenRenewer.init(conf); RMContext mockContext = mock(RMContext.class); ClientRMService mockClientRMService = mock(ClientRMService.class); when(mockContext.getDelegationTokenRenewer()).thenReturn(delegationTokenRenewer); when(mockContext.getDispatcher()).thenReturn(dispatcher); when(mockContext.getClientRMService()).thenReturn(mockClientRMService); InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234); when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); delegationTokenRenewer.setRMContext(mockContext); delegationTokenRenewer.start(); }
@Test public void testSetupShutdown() throws Exception { GetClusterNodesRequest request = Records.newRecord(GetClusterNodesRequest.class); ClientRMService clientRMService = yarnCluster.getResourceManager().getClientRMService(); GetClusterNodesResponse response = clientRMService.getClusterNodes(request); List<NodeReport> nodeReports = response.getNodeReports(); LOG.info("{}", nodeReports); for (NodeReport nr : nodeReports) { LOG.info("Node: {}", nr.getNodeId()); LOG.info("Total memory: {}", nr.getCapability()); LOG.info("Used memory: {}", nr.getUsed()); LOG.info("Number containers: {}", nr.getNumContainers()); } String appMasterJar = JarFinder.getJar(StreamingAppMaster.class); LOG.info("appmaster jar: " + appMasterJar); String testJar = JarFinder.getJar(StramMiniClusterTest.class); LOG.info("testJar: " + testJar); // create test application Properties dagProps = new Properties(); // input module (ensure shutdown works while windows are generated) dagProps.put( StreamingApplication.DT_PREFIX + "operator.numGen.classname", TestGeneratorInputOperator.class.getName()); dagProps.put(StreamingApplication.DT_PREFIX + "operator.numGen.maxTuples", "1"); // fake output adapter - to be ignored when determine shutdown // props.put(DAGContext.DT_PREFIX + "stream.output.classname", // HDFSOutputStream.class.getName()); // props.put(DAGContext.DT_PREFIX + "stream.output.inputNode", "module2"); // props.put(DAGContext.DT_PREFIX + "stream.output.filepath", // "miniclustertest-testSetupShutdown.out"); dagProps.put( StreamingApplication.DT_PREFIX + "operator.module1.classname", GenericTestOperator.class.getName()); dagProps.put( StreamingApplication.DT_PREFIX + "operator.module2.classname", GenericTestOperator.class.getName()); dagProps.put(StreamingApplication.DT_PREFIX + "stream.fromNumGen.source", "numGen.outport"); dagProps.put(StreamingApplication.DT_PREFIX + "stream.fromNumGen.sinks", "module1.inport1"); dagProps.put(StreamingApplication.DT_PREFIX + "stream.n1n2.source", "module1.outport1"); dagProps.put(StreamingApplication.DT_PREFIX + "stream.n1n2.sinks", "module2.inport1"); dagProps.setProperty( StreamingApplication.DT_PREFIX + LogicalPlan.MASTER_MEMORY_MB.getName(), "128"); dagProps.setProperty( StreamingApplication.DT_PREFIX + LogicalPlan.CONTAINER_JVM_OPTIONS.getName(), "-Dlog4j.properties=custom_log4j.properties"); dagProps.setProperty( StreamingApplication.DT_PREFIX + "operator.*." + OperatorContext.MEMORY_MB.getName(), "64"); dagProps.setProperty( StreamingApplication.DT_PREFIX + "operator.*." + OperatorContext.VCORES.getName(), "1"); dagProps.setProperty( StreamingApplication.DT_PREFIX + "operator.*.port.*." + Context.PortContext.BUFFER_MEMORY_MB.getName(), "32"); dagProps.setProperty(StreamingApplication.DT_PREFIX + LogicalPlan.DEBUG.getName(), "true"); // dagProps.setProperty(StreamingApplication.DT_PREFIX + // LogicalPlan.CONTAINERS_MAX_COUNT.getName(), "2"); LOG.info("dag properties: {}", dagProps); LOG.info("Initializing Client"); LogicalPlanConfiguration tb = new LogicalPlanConfiguration(conf); tb.addFromProperties(dagProps, null); LogicalPlan dag = createDAG(tb); Configuration yarnConf = new Configuration(yarnCluster.getConfig()); StramClient client = new StramClient(yarnConf, dag); try { client.start(); if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) { client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster } LOG.info("Running client"); client.startApplication(); boolean result = client.monitorApplication(); LOG.info("Client run completed. Result=" + result); Assert.assertTrue(result); } finally { client.stop(); } }