@Test
public void testCliParser() throws Exception {
List<String> sparkSubmitArgs =
Arrays.asList(
parser.MASTER,
"local",
parser.DRIVER_MEMORY,
"42g",
parser.DRIVER_CLASS_PATH,
"/driverCp",
parser.DRIVER_JAVA_OPTIONS,
"extraJavaOpt",
parser.CONF,
"spark.randomOption=foo",
parser.CONF,
SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH + "=/driverLibPath");
Map<String, String> env = new HashMap<>();
List<String> cmd = buildCommand(sparkSubmitArgs, env);
assertTrue(
findInStringList(
env.get(CommandBuilderUtils.getLibPathEnvName()),
File.pathSeparator,
"/driverLibPath"));
assertTrue(findInStringList(findArgValue(cmd, "-cp"), File.pathSeparator, "/driverCp"));
assertTrue("Driver -Xmx should be configured.", cmd.contains("-Xmx42g"));
assertTrue(
"Command should contain user-defined conf.",
Collections.indexOfSubList(cmd, Arrays.asList(parser.CONF, "spark.randomOption=foo")) > 0);
}
private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) throws Exception {
String deployMode = isDriver ? "client" : "cluster";
SparkSubmitCommandBuilder launcher = newCommandBuilder(Collections.<String>emptyList());
launcher.childEnv.put(
CommandBuilderUtils.ENV_SPARK_HOME, System.getProperty("spark.test.home"));
launcher.master = "yarn";
launcher.deployMode = deployMode;
launcher.appResource = "/foo";
launcher.appName = "MyApp";
launcher.mainClass = "my.Class";
launcher.appArgs.add("foo");
launcher.appArgs.add("bar");
launcher.conf.put("spark.foo", "foo");
// either set the property through "--conf" or through default property file
if (!useDefaultPropertyFile) {
launcher.setPropertiesFile(dummyPropsFile.getAbsolutePath());
launcher.conf.put(SparkLauncher.DRIVER_MEMORY, "1g");
launcher.conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, "/driver");
launcher.conf.put(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Ddriver -XX:MaxPermSize=256m");
launcher.conf.put(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH, "/native");
} else {
launcher.childEnv.put(
"SPARK_CONF_DIR", System.getProperty("spark.test.home") + "/launcher/src/test/resources");
}
Map<String, String> env = new HashMap<>();
List<String> cmd = launcher.buildCommand(env);
// Checks below are different for driver and non-driver mode.
if (isDriver) {
assertTrue("Driver -Xmx should be configured.", cmd.contains("-Xmx1g"));
} else {
boolean found = false;
for (String arg : cmd) {
if (arg.startsWith("-Xmx")) {
found = true;
break;
}
}
assertFalse("Memory arguments should not be set.", found);
}
for (String arg : cmd) {
if (arg.startsWith("-XX:MaxPermSize=")) {
assertEquals("-XX:MaxPermSize=256m", arg);
}
}
String[] cp = findArgValue(cmd, "-cp").split(Pattern.quote(File.pathSeparator));
if (isDriver) {
assertTrue("Driver classpath should contain provided entry.", contains("/driver", cp));
} else {
assertFalse("Driver classpath should not be in command.", contains("/driver", cp));
}
String libPath = env.get(CommandBuilderUtils.getLibPathEnvName());
if (isDriver) {
assertNotNull("Native library path should be set.", libPath);
assertTrue(
"Native library path should contain provided entry.",
contains("/native", libPath.split(Pattern.quote(File.pathSeparator))));
} else {
assertNull("Native library should not be set.", libPath);
}
// Checks below are the same for both driver and non-driver mode.
if (!useDefaultPropertyFile) {
assertEquals(dummyPropsFile.getAbsolutePath(), findArgValue(cmd, parser.PROPERTIES_FILE));
}
assertEquals("yarn", findArgValue(cmd, parser.MASTER));
assertEquals(deployMode, findArgValue(cmd, parser.DEPLOY_MODE));
assertEquals("my.Class", findArgValue(cmd, parser.CLASS));
assertEquals("MyApp", findArgValue(cmd, parser.NAME));
boolean appArgsOk = false;
for (int i = 0; i < cmd.size(); i++) {
if (cmd.get(i).equals("/foo")) {
assertEquals("foo", cmd.get(i + 1));
assertEquals("bar", cmd.get(i + 2));
assertEquals(cmd.size(), i + 3);
appArgsOk = true;
break;
}
}
assertTrue("App resource and args should be added to command.", appArgsOk);
Map<String, String> conf = parseConf(cmd, parser);
assertEquals("foo", conf.get("spark.foo"));
}
