@Test public void testDynamicProperties() throws IOException { Map<String, String> map = new HashMap<String, String>(System.getenv()); File tmpFolder = tmp.newFolder(); File fakeConf = new File(tmpFolder, "flink-conf.yaml"); fakeConf.createNewFile(); map.put(ConfigConstants.ENV_FLINK_CONF_DIR, tmpFolder.getAbsolutePath()); TestBaseUtils.setEnv(map); FlinkYarnSessionCli cli = new FlinkYarnSessionCli("", "", false); Options options = new Options(); cli.addGeneralOptions(options); cli.addRunOptions(options); CommandLineParser parser = new DefaultParser(); CommandLine cmd = null; try { cmd = parser.parse( options, new String[] {"run", "-j", "fake.jar", "-n", "15", "-D", "akka.ask.timeout=5 min"}); } catch (Exception e) { e.printStackTrace(); Assert.fail("Parsing failed with " + e.getMessage()); } AbstractYarnClusterDescriptor flinkYarnDescriptor = cli.createDescriptor(null, cmd); Assert.assertNotNull(flinkYarnDescriptor); Map<String, String> dynProperties = FlinkYarnSessionCli.getDynamicProperties(flinkYarnDescriptor.getDynamicPropertiesEncoded()); Assert.assertEquals(1, dynProperties.size()); Assert.assertEquals("5 min", dynProperties.get("akka.ask.timeout")); }
@Test public void testCorrectSettingOfMaxSlots() throws Exception { File confFile = tmp.newFile("flink-conf.yaml"); File jarFile = tmp.newFile("test.jar"); new CliFrontend(tmp.getRoot().getAbsolutePath()); String[] params = new String[] {"-yn", "2", "-ys", "3", jarFile.getAbsolutePath()}; RunOptions runOptions = CliFrontendParser.parseRunCommand(params); FlinkYarnSessionCli yarnCLI = new TestCLI("y", "yarn"); AbstractYarnClusterDescriptor descriptor = yarnCLI.createDescriptor("", runOptions.getCommandLine()); // each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased. Assert.assertEquals(3, descriptor.getTaskManagerSlots()); Assert.assertEquals(2, descriptor.getTaskManagerCount()); Configuration config = new Configuration(); CliFrontend.setJobManagerAddressInConfig(config, new InetSocketAddress("test", 9000)); ClusterClient client = new TestingYarnClusterClient(descriptor, config); Assert.assertEquals(6, client.getMaxSlots()); }
@Test public void testNotEnoughTaskSlots() throws Exception { File confFile = tmp.newFile("flink-conf.yaml"); File jarFile = tmp.newFile("test.jar"); new CliFrontend(tmp.getRoot().getAbsolutePath()); String[] params = new String[] {"-yn", "2", "-ys", "3", "-p", "7", jarFile.getAbsolutePath()}; RunOptions runOptions = CliFrontendParser.parseRunCommand(params); FlinkYarnSessionCli yarnCLI = new TestCLI("y", "yarn"); AbstractYarnClusterDescriptor descriptor = yarnCLI.createDescriptor("", runOptions.getCommandLine()); // each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased. Assert.assertEquals(4, descriptor.getTaskManagerSlots()); Assert.assertEquals(2, descriptor.getTaskManagerCount()); }
@Override public void run() { try { int returnValue; switch (type) { case YARN_SESSION: yCli = new FlinkYarnSessionCli("", "", false); returnValue = yCli.run(args); break; case CLI_FRONTEND: TestingCLI cli; try { cli = new TestingCLI(); returnValue = cli.parseParameters(args); } catch (Exception e) { throw new RuntimeException( "Failed to execute the following args with CliFrontend: " + Arrays.toString(args), e); } final ClusterClient client = cli.getClusterClient(); try { // check if the JobManager is still alive after running the job final FiniteDuration finiteDuration = new FiniteDuration(10, TimeUnit.SECONDS); ActorGateway jobManagerGateway = client.getJobManagerGateway(); Await.ready( jobManagerGateway.ask(new Identify(true), finiteDuration), finiteDuration); } catch (Exception e) { throw new RuntimeException( "It seems like the JobManager died although it should still be alive"); } // verify we would have shut down anyways and then shutdown Mockito.verify(cli.getSpiedClusterClient()).shutdown(); client.shutdown(); break; default: throw new RuntimeException("Unknown type " + type); } if (returnValue != this.expectedReturnValue) { Assert.fail( "The YARN session returned with unexpected value=" + returnValue + " expected=" + expectedReturnValue); } } catch (Throwable t) { LOG.info("Runner stopped with exception", t); // save error. this.runnerError = t; } }
@Override public void run() { switch (type) { case YARN_SESSION: yCli = new FlinkYarnSessionCli("", "", false); returnValue = yCli.run(args); break; case CLI_FRONTEND: try { CliFrontend cli = new CliFrontend(); returnValue = cli.parseParameters(args); } catch (Exception e) { throw new RuntimeException(e); } break; default: throw new RuntimeException("Unknown type " + type); } if (returnValue != 0) { Assert.fail("The YARN session returned with non-null value=" + returnValue); } }
/** Stops the Yarn session */ public void sendStop() { if (yCli != null) { yCli.stop(); } }