@Test
public void testDynamicProperties() throws IOException {
Map<String, String> map = new HashMap<String, String>(System.getenv());
File tmpFolder = tmp.newFolder();
File fakeConf = new File(tmpFolder, "flink-conf.yaml");
fakeConf.createNewFile();
map.put(ConfigConstants.ENV_FLINK_CONF_DIR, tmpFolder.getAbsolutePath());
TestBaseUtils.setEnv(map);
FlinkYarnSessionCli cli = new FlinkYarnSessionCli("", "", false);
Options options = new Options();
cli.addGeneralOptions(options);
cli.addRunOptions(options);
CommandLineParser parser = new DefaultParser();
CommandLine cmd = null;
try {
cmd =
parser.parse(
options,
new String[] {"run", "-j", "fake.jar", "-n", "15", "-D", "akka.ask.timeout=5 min"});
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Parsing failed with " + e.getMessage());
}
AbstractYarnClusterDescriptor flinkYarnDescriptor = cli.createDescriptor(null, cmd);
Assert.assertNotNull(flinkYarnDescriptor);
Map<String, String> dynProperties =
FlinkYarnSessionCli.getDynamicProperties(flinkYarnDescriptor.getDynamicPropertiesEncoded());
Assert.assertEquals(1, dynProperties.size());
Assert.assertEquals("5 min", dynProperties.get("akka.ask.timeout"));
}
@Test
public void testCorrectSettingOfMaxSlots() throws Exception {
File confFile = tmp.newFile("flink-conf.yaml");
File jarFile = tmp.newFile("test.jar");
new CliFrontend(tmp.getRoot().getAbsolutePath());
String[] params = new String[] {"-yn", "2", "-ys", "3", jarFile.getAbsolutePath()};
RunOptions runOptions = CliFrontendParser.parseRunCommand(params);
FlinkYarnSessionCli yarnCLI = new TestCLI("y", "yarn");
AbstractYarnClusterDescriptor descriptor =
yarnCLI.createDescriptor("", runOptions.getCommandLine());
// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
Assert.assertEquals(3, descriptor.getTaskManagerSlots());
Assert.assertEquals(2, descriptor.getTaskManagerCount());
Configuration config = new Configuration();
CliFrontend.setJobManagerAddressInConfig(config, new InetSocketAddress("test", 9000));
ClusterClient client = new TestingYarnClusterClient(descriptor, config);
Assert.assertEquals(6, client.getMaxSlots());
}
@Test
public void testNotEnoughTaskSlots() throws Exception {
File confFile = tmp.newFile("flink-conf.yaml");
File jarFile = tmp.newFile("test.jar");
new CliFrontend(tmp.getRoot().getAbsolutePath());
String[] params = new String[] {"-yn", "2", "-ys", "3", "-p", "7", jarFile.getAbsolutePath()};
RunOptions runOptions = CliFrontendParser.parseRunCommand(params);
FlinkYarnSessionCli yarnCLI = new TestCLI("y", "yarn");
AbstractYarnClusterDescriptor descriptor =
yarnCLI.createDescriptor("", runOptions.getCommandLine());
// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
Assert.assertEquals(4, descriptor.getTaskManagerSlots());
Assert.assertEquals(2, descriptor.getTaskManagerCount());
}
@Override
public void run() {
try {
int returnValue;
switch (type) {
case YARN_SESSION:
yCli = new FlinkYarnSessionCli("", "", false);
returnValue = yCli.run(args);
break;
case CLI_FRONTEND:
TestingCLI cli;
try {
cli = new TestingCLI();
returnValue = cli.parseParameters(args);
} catch (Exception e) {
throw new RuntimeException(
"Failed to execute the following args with CliFrontend: " + Arrays.toString(args),
e);
}
final ClusterClient client = cli.getClusterClient();
try {
// check if the JobManager is still alive after running the job
final FiniteDuration finiteDuration = new FiniteDuration(10, TimeUnit.SECONDS);
ActorGateway jobManagerGateway = client.getJobManagerGateway();
Await.ready(
jobManagerGateway.ask(new Identify(true), finiteDuration), finiteDuration);
} catch (Exception e) {
throw new RuntimeException(
"It seems like the JobManager died although it should still be alive");
}
// verify we would have shut down anyways and then shutdown
Mockito.verify(cli.getSpiedClusterClient()).shutdown();
client.shutdown();
break;
default:
throw new RuntimeException("Unknown type " + type);
}
if (returnValue != this.expectedReturnValue) {
Assert.fail(
"The YARN session returned with unexpected value="
+ returnValue
+ " expected="
+ expectedReturnValue);
}
} catch (Throwable t) {
LOG.info("Runner stopped with exception", t);
// save error.
this.runnerError = t;
}
}
@Override
public void run() {
switch (type) {
case YARN_SESSION:
yCli = new FlinkYarnSessionCli("", "", false);
returnValue = yCli.run(args);
break;
case CLI_FRONTEND:
try {
CliFrontend cli = new CliFrontend();
returnValue = cli.parseParameters(args);
} catch (Exception e) {
throw new RuntimeException(e);
}
break;
default:
throw new RuntimeException("Unknown type " + type);
}
if (returnValue != 0) {
Assert.fail("The YARN session returned with non-null value=" + returnValue);
}
}
/** Stops the Yarn session */
public void sendStop() {
if (yCli != null) {
yCli.stop();
}
}