@Test
public void testSparkSql() {
repl.interpret("case class Person(name:String, age:Int)\n", context);
repl.interpret(
"val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n",
context);
assertEquals(Code.SUCCESS, repl.interpret("people.take(3)", context).code());
if (getSparkVersionNumber()
<= 11) { // spark 1.2 or later does not allow create multiple SparkContext in the same jvm
// by default.
// create new interpreter
SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties());
repl2.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl2);
repl2.open();
repl2.interpret("case class Man(name:String, age:Int)", context);
repl2.interpret(
"val man = sc.parallelize(Seq(Man(\"moon\", 33), Man(\"jobs\", 51), Man(\"gates\", 51), Man(\"park\", 34)))",
context);
assertEquals(Code.SUCCESS, repl2.interpret("man.take(3)", context).code());
repl2.close();
}
}
@Before
public void setUp() throws Exception {
tmpDir =
new File(
System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
System.setProperty("zeppelin.dep.localrepo", tmpDir.getAbsolutePath() + "/local-repo");
tmpDir.mkdirs();
if (repl == null) {
intpGroup = new InterpreterGroup();
intpGroup.put("note", new LinkedList<Interpreter>());
repl = new SparkInterpreter(getSparkTestProperties());
repl.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl);
repl.open();
}
context =
new InterpreterContext(
"note",
"id",
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("id"),
new LinkedList<InterpreterContextRunner>(),
new InterpreterOutput(
new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {}
}));
}
@Test
public void shareSingleSparkContext() throws InterruptedException {
// create another SparkInterpreter
SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties());
repl2.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl2);
repl2.open();
assertEquals(
Code.SUCCESS, repl.interpret("print(sc.parallelize(1 to 10).count())", context).code());
assertEquals(
Code.SUCCESS, repl2.interpret("print(sc.parallelize(1 to 10).count())", context).code());
repl2.close();
}
@Test
public void testEnableImplicitImport() {
if (getSparkVersionNumber() >= 13) {
// Set option of importing implicits to "true", and initialize new Spark repl
Properties p = getSparkTestProperties();
p.setProperty("zeppelin.spark.importImplicit", "true");
SparkInterpreter repl2 = new SparkInterpreter(p);
repl2.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl2);
repl2.open();
String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")";
assertEquals(Code.SUCCESS, repl2.interpret(ddl, context).code());
repl2.close();
}
}
@Test
public void testDisableImplicitImport() {
if (getSparkVersionNumber() >= 13) {
// Set option of importing implicits to "false", and initialize new Spark repl
// this test should return error status when creating DataFrame from sequence
Properties p = getSparkTestProperties();
p.setProperty("zeppelin.spark.importImplicit", "false");
SparkInterpreter repl2 = new SparkInterpreter(p);
repl2.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl2);
repl2.open();
String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")";
assertEquals(Code.ERROR, repl2.interpret(ddl, context).code());
repl2.close();
}
}