/** Test regular operation, including command line parameter parsing. */
@Test(timeout = 60000) // timeout after a minute.
public void testDetachedMode() {
LOG.info("Starting testDetachedMode()");
addTestAppender(FlinkYarnSessionCli.class, Level.INFO);
Runner runner =
startWithArgs(
new String[] {
"-j",
flinkUberjar.getAbsolutePath(),
"-t",
flinkLibFolder.getAbsolutePath(),
"-n",
"1",
"-jm",
"768",
"-tm",
"1024",
"--name",
"MyCustomName", // test setting a custom name
"--detached"
},
"Flink JobManager is now running on",
RunTypes.YARN_SESSION);
checkForLogString("The Flink YARN client has been started in detached mode");
Assert.assertFalse("The runner should detach.", runner.isAlive());
LOG.info("Waiting until two containers are running");
// wait until two containers are running
while (getRunningContainers() < 2) {
sleep(500);
}
LOG.info("Two containers are running. Killing the application");
// kill application "externally".
try {
YarnClient yc = YarnClient.createYarnClient();
yc.init(yarnConfiguration);
yc.start();
List<ApplicationReport> apps = yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
Assert.assertEquals(1, apps.size()); // Only one running
ApplicationReport app = apps.get(0);
Assert.assertEquals("MyCustomName", app.getName());
ApplicationId id = app.getApplicationId();
yc.killApplication(id);
while (yc.getApplications(EnumSet.of(YarnApplicationState.KILLED)).size() == 0) {
sleep(500);
}
} catch (Throwable t) {
LOG.warn("Killing failed", t);
Assert.fail();
}
LOG.info("Finished testDetachedMode()");
}
/**
* convert an AM report to a string for diagnostics
*
* @param report the report
* @return the string value
*/
public static String reportToString(ApplicationReport report) {
if (report == null) {
return "Null application report";
}
return "App "
+ report.getName()
+ "/"
+ report.getApplicationType()
+ "# "
+ report.getApplicationId()
+ " user "
+ report.getUser()
+ " is in state "
+ report.getYarnApplicationState()
+ "RPC: "
+ report.getHost()
+ ":"
+ report.getRpcPort();
}
@Test(timeout = 20000)
public void testJobSubmissionFailure() throws Exception {
when(resourceMgrDelegate.submitApplication(any(ApplicationSubmissionContext.class)))
.thenReturn(appId);
ApplicationReport report = mock(ApplicationReport.class);
when(report.getApplicationId()).thenReturn(appId);
when(report.getDiagnostics()).thenReturn(failString);
when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.FAILED);
when(resourceMgrDelegate.getApplicationReport(appId)).thenReturn(report);
Credentials credentials = new Credentials();
File jobxml = new File(testWorkDir, "job.xml");
OutputStream out = new FileOutputStream(jobxml);
conf.writeXml(out);
out.close();
try {
yarnRunner.submitJob(jobId, testWorkDir.getAbsolutePath().toString(), credentials);
} catch (IOException io) {
LOG.info("Logging exception:", io);
assertTrue(io.getLocalizedMessage().contains(failString));
}
}
@Before
public void checkClusterEmpty() throws IOException, YarnException {
if (yarnClient == null) {
yarnClient = YarnClient.createYarnClient();
yarnClient.init(yarnConfiguration);
yarnClient.start();
}
List<ApplicationReport> apps = yarnClient.getApplications();
for (ApplicationReport app : apps) {
if (app.getYarnApplicationState() != YarnApplicationState.FINISHED
&& app.getYarnApplicationState() != YarnApplicationState.KILLED
&& app.getYarnApplicationState() != YarnApplicationState.FAILED) {
Assert.fail(
"There is at least one application on the cluster is not finished."
+ "App "
+ app.getApplicationId()
+ " is in state "
+ app.getYarnApplicationState());
}
}
}
/**
* Verify the web service deployment and lifecycle functionality
*
* @throws Exception
*/
@Ignore // disabled due to web service init delay issue
@Test
public void testWebService() throws Exception {
// single container topology of inline input and module
Properties props = new Properties();
props.put(
StreamingApplication.DT_PREFIX + "stream.input.classname",
TestGeneratorInputOperator.class.getName());
props.put(StreamingApplication.DT_PREFIX + "stream.input.outputNode", "module1");
props.put(
StreamingApplication.DT_PREFIX + "module.module1.classname",
GenericTestOperator.class.getName());
LOG.info("Initializing Client");
LogicalPlanConfiguration tb = new LogicalPlanConfiguration(new Configuration(false));
tb.addFromProperties(props, null);
StramClient client = new StramClient(new Configuration(yarnCluster.getConfig()), createDAG(tb));
if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster
}
try {
client.start();
client.startApplication();
// attempt web service connection
ApplicationReport appReport = client.getApplicationReport();
Thread.sleep(5000); // delay to give web service time to fully initialize
Client wsClient = Client.create();
wsClient.setFollowRedirects(true);
WebResource r =
wsClient
.resource("http://" + appReport.getTrackingUrl())
.path(StramWebServices.PATH)
.path(StramWebServices.PATH_INFO);
LOG.info("Requesting: " + r.getURI());
ClientResponse response = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
LOG.info("Got response: " + json.toString());
assertEquals("incorrect number of elements", 1, json.length());
assertEquals("appId", appReport.getApplicationId().toString(), json.get("id"));
r =
wsClient
.resource("http://" + appReport.getTrackingUrl())
.path(StramWebServices.PATH)
.path(StramWebServices.PATH_PHYSICAL_PLAN_OPERATORS);
LOG.info("Requesting: " + r.getURI());
response = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
LOG.info("Got response: " + json.toString());
} finally {
// LOG.info("waiting...");
// synchronized (this) {
// this.wait();
// }
// boolean result = client.monitorApplication();
client.killApplication();
client.stop();
}
}