@Before
public void setUp() throws Exception {
tmpDir =
new File(
System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
System.setProperty("zeppelin.dep.localrepo", tmpDir.getAbsolutePath() + "/local-repo");
tmpDir.mkdirs();
if (repl == null) {
Properties p = new Properties();
repl = new ScaldingInterpreter(p);
repl.open();
}
InterpreterGroup intpGroup = new InterpreterGroup();
context =
new InterpreterContext(
"note",
"id",
"title",
"text",
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LinkedList<InterpreterContextRunner>());
}
// Create new Setting and return Setting ID
protected String createTempSetting(String tempName) throws IOException {
InterpreterGroup interpreterGroup =
ZeppelinServer.notebook
.getInterpreterFactory()
.add(tempName, "newGroup", new InterpreterOption(false), new Properties());
return interpreterGroup.getId();
}
private void snapshotAngularObjectRegistry() {
angularObjects = new HashMap<String, List<AngularObject>>();
List<InterpreterSetting> settings = replLoader.getInterpreterSettings();
if (settings == null || settings.size() == 0) {
return;
}
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup();
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
angularObjects.put(intpGroup.getId(), registry.getAllWithGlobal(id));
}
}
@ZeppelinApi
public Interpreter getInterpreterInTheSameSessionByClassName(String className) {
synchronized (interpreterGroup) {
for (List<Interpreter> interpreters : interpreterGroup.values()) {
boolean belongsToSameNoteGroup = false;
Interpreter interpreterFound = null;
for (Interpreter intp : interpreters) {
if (intp.getClassName().equals(className)) {
interpreterFound = intp;
}
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
if (this == p) {
belongsToSameNoteGroup = true;
}
}
if (belongsToSameNoteGroup) {
return interpreterFound;
}
}
}
return null;
}
@Test
public void testSparkSql() {
repl.interpret("case class Person(name:String, age:Int)\n", context);
repl.interpret(
"val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n",
context);
assertEquals(Code.SUCCESS, repl.interpret("people.take(3)", context).code());
if (getSparkVersionNumber()
<= 11) { // spark 1.2 or later does not allow create multiple SparkContext in the same jvm
// by default.
// create new interpreter
SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties());
repl2.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl2);
repl2.open();
repl2.interpret("case class Man(name:String, age:Int)", context);
repl2.interpret(
"val man = sc.parallelize(Seq(Man(\"moon\", 33), Man(\"jobs\", 51), Man(\"gates\", 51), Man(\"park\", 34)))",
context);
assertEquals(Code.SUCCESS, repl2.interpret("man.take(3)", context).code());
repl2.close();
}
}
@Before
public void setUp() throws Exception {
tmpDir =
new File(
System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
System.setProperty("zeppelin.dep.localrepo", tmpDir.getAbsolutePath() + "/local-repo");
tmpDir.mkdirs();
if (repl == null) {
intpGroup = new InterpreterGroup();
intpGroup.put("note", new LinkedList<Interpreter>());
repl = new SparkInterpreter(getSparkTestProperties());
repl.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl);
repl.open();
}
context =
new InterpreterContext(
"note",
"id",
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("id"),
new LinkedList<InterpreterContextRunner>(),
new InterpreterOutput(
new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {}
}));
}
private void removeAllAngularObjectInParagraph(String user, String paragraphId) {
angularObjects = new HashMap<>();
List<InterpreterSetting> settings = factory.getInterpreterSettings(getId());
if (settings == null || settings.size() == 0) {
return;
}
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
// remove paragraph scope object
((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, paragraphId);
// remove app scope object
List<ApplicationState> appStates = getParagraph(paragraphId).getAllApplicationStates();
if (appStates != null) {
for (ApplicationState app : appStates) {
((RemoteAngularObjectRegistry) registry)
.removeAllAndNotifyRemoteProcess(id, app.getId());
}
}
} else {
registry.removeAll(id, paragraphId);
// remove app scope object
List<ApplicationState> appStates = getParagraph(paragraphId).getAllApplicationStates();
if (appStates != null) {
for (ApplicationState app : appStates) {
registry.removeAll(id, app.getId());
}
}
}
}
}
@Test
public void shareSingleSparkContext() throws InterruptedException {
// create another SparkInterpreter
SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties());
repl2.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl2);
repl2.open();
assertEquals(
Code.SUCCESS, repl.interpret("print(sc.parallelize(1 to 10).count())", context).code());
assertEquals(
Code.SUCCESS, repl2.interpret("print(sc.parallelize(1 to 10).count())", context).code());
repl2.close();
}
@Test
public void testEnableImplicitImport() {
if (getSparkVersionNumber() >= 13) {
// Set option of importing implicits to "true", and initialize new Spark repl
Properties p = getSparkTestProperties();
p.setProperty("zeppelin.spark.importImplicit", "true");
SparkInterpreter repl2 = new SparkInterpreter(p);
repl2.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl2);
repl2.open();
String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")";
assertEquals(Code.SUCCESS, repl2.interpret(ddl, context).code());
repl2.close();
}
}
@Test
public void testDisableImplicitImport() {
if (getSparkVersionNumber() >= 13) {
// Set option of importing implicits to "false", and initialize new Spark repl
// this test should return error status when creating DataFrame from sequence
Properties p = getSparkTestProperties();
p.setProperty("zeppelin.spark.importImplicit", "false");
SparkInterpreter repl2 = new SparkInterpreter(p);
repl2.setInterpreterGroup(intpGroup);
intpGroup.get("note").add(repl2);
repl2.open();
String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")";
assertEquals(Code.ERROR, repl2.interpret(ddl, context).code());
repl2.close();
}
}
private Note loadNoteFromRepo(String id) {
Note note = null;
try {
note = notebookRepo.get(id);
} catch (IOException e) {
logger.error("Failed to load " + id, e);
}
if (note == null) {
return null;
}
// set NoteInterpreterLoader
NoteInterpreterLoader noteInterpreterLoader = new NoteInterpreterLoader(replFactory);
note.setReplLoader(noteInterpreterLoader);
noteInterpreterLoader.setNoteId(note.id());
// set JobListenerFactory
note.setJobListenerFactory(jobListenerFactory);
// set notebookRepo
note.setNotebookRepo(notebookRepo);
Map<String, SnapshotAngularObject> angularObjectSnapshot =
new HashMap<String, SnapshotAngularObject>();
// restore angular object --------------
Date lastUpdatedDate = new Date(0);
for (Paragraph p : note.getParagraphs()) {
p.setNote(note);
if (p.getDateFinished() != null && lastUpdatedDate.before(p.getDateFinished())) {
lastUpdatedDate = p.getDateFinished();
}
}
Map<String, List<AngularObject>> savedObjects = note.getAngularObjects();
if (savedObjects != null) {
for (String intpGroupName : savedObjects.keySet()) {
List<AngularObject> objectList = savedObjects.get(intpGroupName);
for (AngularObject savedObject : objectList) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(savedObject.getName());
if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) {
angularObjectSnapshot.put(
savedObject.getName(),
new SnapshotAngularObject(intpGroupName, savedObject, lastUpdatedDate));
}
}
}
}
synchronized (notes) {
notes.put(note.id(), note);
refreshCron(note.id());
}
for (String name : angularObjectSnapshot.keySet()) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
List<InterpreterSetting> settings = replFactory.get();
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup();
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
String noteId = snapshot.getAngularObject().getNoteId();
// at this point, remote interpreter process is not created.
// so does not make sense add it to the remote.
//
// therefore instead of addAndNotifyRemoteProcess(), need to use add()
// that results add angularObject only in ZeppelinServer side not remoteProcessSide
registry.add(name, snapshot.getAngularObject().get(), noteId);
}
}
}
return note;
}
@SuppressWarnings("rawtypes")
private Note loadNoteFromRepo(String id, AuthenticationInfo subject) {
Note note = null;
try {
note = notebookRepo.get(id, subject);
} catch (IOException e) {
logger.error("Failed to load " + id, e);
}
if (note == null) {
return null;
}
// Manually inject ALL dependencies, as DI constructor was NOT used
note.setIndex(this.notebookIndex);
note.setCredentials(this.credentials);
note.setInterpreterFactory(replFactory);
note.setJobListenerFactory(jobListenerFactory);
note.setNotebookRepo(notebookRepo);
Map<String, SnapshotAngularObject> angularObjectSnapshot = new HashMap<>();
// restore angular object --------------
Date lastUpdatedDate = new Date(0);
for (Paragraph p : note.getParagraphs()) {
p.setNote(note);
if (p.getDateFinished() != null && lastUpdatedDate.before(p.getDateFinished())) {
lastUpdatedDate = p.getDateFinished();
}
}
Map<String, List<AngularObject>> savedObjects = note.getAngularObjects();
if (savedObjects != null) {
for (String intpGroupName : savedObjects.keySet()) {
List<AngularObject> objectList = savedObjects.get(intpGroupName);
for (AngularObject object : objectList) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(object.getName());
if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) {
angularObjectSnapshot.put(
object.getName(),
new SnapshotAngularObject(intpGroupName, object, lastUpdatedDate));
}
}
}
}
note.setNoteEventListener(this);
synchronized (notes) {
notes.put(note.getId(), note);
refreshCron(note.getId());
}
for (String name : angularObjectSnapshot.keySet()) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
List<InterpreterSetting> settings = replFactory.get();
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(note.getId());
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
String noteId = snapshot.getAngularObject().getNoteId();
String paragraphId = snapshot.getAngularObject().getParagraphId();
// at this point, remote interpreter process is not created.
// so does not make sense add it to the remote.
//
// therefore instead of addAndNotifyRemoteProcess(), need to use add()
// that results add angularObject only in ZeppelinServer side not remoteProcessSide
registry.add(name, snapshot.getAngularObject().get(), noteId, paragraphId);
}
}
}
return note;
}