/**
* Ensure that we will never have the following error.
*
* <p>The test works as follows: - Use the CliFrontend to invoke a jar file that loads a class
* which is only available in the jarfile itself (via a custom classloader) - Change the Usercode
* classloader of the PackagedProgram to a special classloader for this test - the classloader
* will accept the special class (and return a String.class)
*
* <p>org.apache.flink.client.program.ProgramInvocationException: The main method caused an error.
* at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:398) at
* org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:301)
* at org.apache.flink.client.program.Client.getOptimizedPlan(Client.java:140) at
* org.apache.flink.client.program.Client.getOptimizedPlanAsJson(Client.java:125) at
* org.apache.flink.client.CliFrontend.info(CliFrontend.java:439) at
* org.apache.flink.client.CliFrontend.parseParameters(CliFrontend.java:931) at
* org.apache.flink.client.CliFrontend.main(CliFrontend.java:951) Caused by: java.io.IOException:
* java.lang.RuntimeException: java.lang.ClassNotFoundException:
* org.apache.hadoop.hive.ql.io.RCFileInputFormat at
* org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:102) at
* org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:54) at
* tlabs.CDR_In_Report.createHCatInputFormat(CDR_In_Report.java:322) at
* tlabs.CDR_Out_Report.main(CDR_Out_Report.java:380) at
* sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
* sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at
* sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at
* java.lang.reflect.Method.invoke(Method.java:622) at
* org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:383)
*/
@Test
public void testPlanWithExternalClass() throws CompilerException, ProgramInvocationException {
final Boolean callme[] = {
false
}; // create a final object reference, to be able to change its val later
try {
String[] parameters = {
getTestJarPath(), "-c", TEST_JAR_CLASSLOADERTEST_CLASS, "some", "program"
};
CommandLine line =
new PosixParser()
.parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false);
CliFrontend frontend = new CliFrontend();
Object result = frontend.buildProgram(line);
assertTrue(result instanceof PackagedProgram);
PackagedProgram prog = spy((PackagedProgram) result);
ClassLoader testClassLoader =
new ClassLoader(prog.getUserCodeClassLoader()) {
@Override
public Class<?> loadClass(String name) throws ClassNotFoundException {
assertTrue(name.equals("org.apache.hadoop.hive.ql.io.RCFileInputFormat"));
callme[0] = true;
return String.class; // Intentionally return the wrong class.
}
};
when(prog.getUserCodeClassLoader()).thenReturn(testClassLoader);
Assert.assertArrayEquals(new String[] {"some", "program"}, prog.getArguments());
Assert.assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, prog.getMainClassName());
Configuration c = new Configuration();
c.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "devil");
Client cli = new Client(c, getClass().getClassLoader());
cli.getOptimizedPlanAsJson(prog, 666);
} catch (ProgramInvocationException pie) {
assertTrue("Classloader was not called", callme[0]);
// class not found exception is expected as some point
if (!(pie.getCause() instanceof ClassNotFoundException)) {
System.err.println(pie.getMessage());
pie.printStackTrace();
fail("Program caused an exception: " + pie.getMessage());
}
} catch (Exception e) {
assertTrue("Classloader was not called", callme[0]);
System.err.println(e.getMessage());
e.printStackTrace();
fail("Program caused an exception: " + e.getMessage());
}
}
@Test
public void testCorrectSettingOfMaxSlots() throws Exception {
File confFile = tmp.newFile("flink-conf.yaml");
File jarFile = tmp.newFile("test.jar");
new CliFrontend(tmp.getRoot().getAbsolutePath());
String[] params = new String[] {"-yn", "2", "-ys", "3", jarFile.getAbsolutePath()};
RunOptions runOptions = CliFrontendParser.parseRunCommand(params);
FlinkYarnSessionCli yarnCLI = new TestCLI("y", "yarn");
AbstractYarnClusterDescriptor descriptor =
yarnCLI.createDescriptor("", runOptions.getCommandLine());
// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
Assert.assertEquals(3, descriptor.getTaskManagerSlots());
Assert.assertEquals(2, descriptor.getTaskManagerCount());
Configuration config = new Configuration();
CliFrontend.setJobManagerAddressInConfig(config, new InetSocketAddress("test", 9000));
ClusterClient client = new TestingYarnClusterClient(descriptor, config);
Assert.assertEquals(6, client.getMaxSlots());
}
@Test
public void testFileNotJarFile() {
try {
String[] arguments = {"-j", getNonJarFilePath(), "-a", "plenty", "of", "arguments"};
CommandLine line =
new PosixParser()
.parse(CliFrontend.getProgramSpecificOptions(new Options()), arguments, false);
CliFrontend frontend = new CliFrontend();
Object result = frontend.buildProgram(line);
assertTrue(result == null);
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Program caused an exception: " + e.getMessage());
}
}
@Test
public void testNonExistingFileWithoutArguments() {
try {
String[] parameters = {"/some/none/existing/path"};
CommandLine line =
new PosixParser()
.parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false);
CliFrontend frontend = new CliFrontend();
Object result = frontend.buildProgram(line);
assertTrue(result == null);
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Program caused an exception: " + e.getMessage());
}
}
@Test
public void testVariantWithExplicitJarAndArgumentsOption() {
try {
String[] parameters = {"-j", getTestJarPath(), "-a", "some", "program", "arguments"};
CommandLine line =
new PosixParser()
.parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false);
CliFrontend frontend = new CliFrontend();
Object result = frontend.buildProgram(line);
assertTrue(result instanceof PackagedProgram);
PackagedProgram prog = (PackagedProgram) result;
Assert.assertArrayEquals(new String[] {"some", "program", "arguments"}, prog.getArguments());
Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName());
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
fail("Program caused an exception: " + e.getMessage());
}
}
@Override
public void run() {
switch (type) {
case YARN_SESSION:
yCli = new FlinkYarnSessionCli("", "", false);
returnValue = yCli.run(args);
break;
case CLI_FRONTEND:
try {
CliFrontend cli = new CliFrontend();
returnValue = cli.parseParameters(args);
} catch (Exception e) {
throw new RuntimeException(e);
}
break;
default:
throw new RuntimeException("Unknown type " + type);
}
if (returnValue != 0) {
Assert.fail("The YARN session returned with non-null value=" + returnValue);
}
}