/** Test regular operation, including command line parameter parsing. */ @Test(timeout = 60000) // timeout after a minute. public void testDetachedMode() { LOG.info("Starting testDetachedMode()"); addTestAppender(FlinkYarnSessionCli.class, Level.INFO); Runner runner = startWithArgs( new String[] { "-j", flinkUberjar.getAbsolutePath(), "-t", flinkLibFolder.getAbsolutePath(), "-n", "1", "-jm", "768", "-tm", "1024", "--name", "MyCustomName", // test setting a custom name "--detached" }, "Flink JobManager is now running on", RunTypes.YARN_SESSION); checkForLogString("The Flink YARN client has been started in detached mode"); Assert.assertFalse("The runner should detach.", runner.isAlive()); LOG.info("Waiting until two containers are running"); // wait until two containers are running while (getRunningContainers() < 2) { sleep(500); } LOG.info("Two containers are running. Killing the application"); // kill application "externally". try { YarnClient yc = YarnClient.createYarnClient(); yc.init(yarnConfiguration); yc.start(); List<ApplicationReport> apps = yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING)); Assert.assertEquals(1, apps.size()); // Only one running ApplicationReport app = apps.get(0); Assert.assertEquals("MyCustomName", app.getName()); ApplicationId id = app.getApplicationId(); yc.killApplication(id); while (yc.getApplications(EnumSet.of(YarnApplicationState.KILLED)).size() == 0) { sleep(500); } } catch (Throwable t) { LOG.warn("Killing failed", t); Assert.fail(); } LOG.info("Finished testDetachedMode()"); }
/** * convert an AM report to a string for diagnostics * * @param report the report * @return the string value */ public static String reportToString(ApplicationReport report) { if (report == null) { return "Null application report"; } return "App " + report.getName() + "/" + report.getApplicationType() + "# " + report.getApplicationId() + " user " + report.getUser() + " is in state " + report.getYarnApplicationState() + "RPC: " + report.getHost() + ":" + report.getRpcPort(); }
@Test(timeout = 20000) public void testJobSubmissionFailure() throws Exception { when(resourceMgrDelegate.submitApplication(any(ApplicationSubmissionContext.class))) .thenReturn(appId); ApplicationReport report = mock(ApplicationReport.class); when(report.getApplicationId()).thenReturn(appId); when(report.getDiagnostics()).thenReturn(failString); when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.FAILED); when(resourceMgrDelegate.getApplicationReport(appId)).thenReturn(report); Credentials credentials = new Credentials(); File jobxml = new File(testWorkDir, "job.xml"); OutputStream out = new FileOutputStream(jobxml); conf.writeXml(out); out.close(); try { yarnRunner.submitJob(jobId, testWorkDir.getAbsolutePath().toString(), credentials); } catch (IOException io) { LOG.info("Logging exception:", io); assertTrue(io.getLocalizedMessage().contains(failString)); } }
@Before public void checkClusterEmpty() throws IOException, YarnException { if (yarnClient == null) { yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConfiguration); yarnClient.start(); } List<ApplicationReport> apps = yarnClient.getApplications(); for (ApplicationReport app : apps) { if (app.getYarnApplicationState() != YarnApplicationState.FINISHED && app.getYarnApplicationState() != YarnApplicationState.KILLED && app.getYarnApplicationState() != YarnApplicationState.FAILED) { Assert.fail( "There is at least one application on the cluster is not finished." + "App " + app.getApplicationId() + " is in state " + app.getYarnApplicationState()); } } }
/** * Verify the web service deployment and lifecycle functionality * * @throws Exception */ @Ignore // disabled due to web service init delay issue @Test public void testWebService() throws Exception { // single container topology of inline input and module Properties props = new Properties(); props.put( StreamingApplication.DT_PREFIX + "stream.input.classname", TestGeneratorInputOperator.class.getName()); props.put(StreamingApplication.DT_PREFIX + "stream.input.outputNode", "module1"); props.put( StreamingApplication.DT_PREFIX + "module.module1.classname", GenericTestOperator.class.getName()); LOG.info("Initializing Client"); LogicalPlanConfiguration tb = new LogicalPlanConfiguration(new Configuration(false)); tb.addFromProperties(props, null); StramClient client = new StramClient(new Configuration(yarnCluster.getConfig()), createDAG(tb)); if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) { client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster } try { client.start(); client.startApplication(); // attempt web service connection ApplicationReport appReport = client.getApplicationReport(); Thread.sleep(5000); // delay to give web service time to fully initialize Client wsClient = Client.create(); wsClient.setFollowRedirects(true); WebResource r = wsClient .resource("http://" + appReport.getTrackingUrl()) .path(StramWebServices.PATH) .path(StramWebServices.PATH_INFO); LOG.info("Requesting: " + r.getURI()); ClientResponse response = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); LOG.info("Got response: " + json.toString()); assertEquals("incorrect number of elements", 1, json.length()); assertEquals("appId", appReport.getApplicationId().toString(), json.get("id")); r = wsClient .resource("http://" + appReport.getTrackingUrl()) .path(StramWebServices.PATH) .path(StramWebServices.PATH_PHYSICAL_PLAN_OPERATORS); LOG.info("Requesting: " + r.getURI()); response = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); json = response.getEntity(JSONObject.class); LOG.info("Got response: " + json.toString()); } finally { // LOG.info("waiting..."); // synchronized (this) { // this.wait(); // } // boolean result = client.monitorApplication(); client.killApplication(); client.stop(); } }