/**
* Clone existing note.
*
* @param sourceNoteID - the note ID to clone
* @param newNoteName - the name of the new note
* @return noteId
* @throws IOException, CloneNotSupportedException, IllegalArgumentException
*/
public Note cloneNote(String sourceNoteID, String newNoteName, AuthenticationInfo subject)
throws IOException, CloneNotSupportedException, IllegalArgumentException {
Note sourceNote = getNote(sourceNoteID);
if (sourceNote == null) {
throw new IllegalArgumentException(sourceNoteID + "not found");
}
Note newNote = createNote(subject);
if (newNoteName != null) {
newNote.setName(newNoteName);
} else {
newNote.setName("Note " + newNote.getId());
}
// Copy the interpreter bindings
List<String> boundInterpreterSettingsIds = getBindedInterpreterSettingsIds(sourceNote.getId());
bindInterpretersToNote(newNote.getId(), boundInterpreterSettingsIds);
List<Paragraph> paragraphs = sourceNote.getParagraphs();
for (Paragraph p : paragraphs) {
newNote.addCloneParagraph(p);
}
notebookIndex.addIndexDoc(newNote);
newNote.persist(subject);
return newNote;
}
/**
* Create new note REST API
*
* @param message - JSON with new note name
* @return JSON with new note ID
* @throws IOException
*/
@POST
@Path("/")
@ZeppelinApi
public Response createNote(String message) throws IOException {
LOG.info("Create new notebook by JSON {}", message);
NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note note = notebook.createNote(subject);
List<NewParagraphRequest> initialParagraphs = request.getParagraphs();
if (initialParagraphs != null) {
for (NewParagraphRequest paragraphRequest : initialParagraphs) {
Paragraph p = note.addParagraph();
p.setTitle(paragraphRequest.getTitle());
p.setText(paragraphRequest.getText());
}
}
note.addParagraph(); // add one paragraph to the last
String noteName = request.getName();
if (noteName.isEmpty()) {
noteName = "Note " + note.getId();
}
note.setName(noteName);
note.persist(subject);
notebookServer.broadcastNote(note);
notebookServer.broadcastNoteList(subject);
return new JsonResponse<>(Status.CREATED, "", note.getId()).build();
}
public List<Map<String, Object>> getJobListBymNotebookId(String notebookID) {
final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";
long lastRunningUnixTime = 0;
boolean isNotebookRunning = false;
Note jobNote = getNote(notebookID);
List<Map<String, Object>> notesInfo = new LinkedList<>();
if (jobNote == null) {
return notesInfo;
}
Map<String, Object> info = new HashMap<>();
info.put("notebookId", jobNote.getId());
String notebookName = jobNote.getName();
if (notebookName != null && !notebookName.equals("")) {
info.put("notebookName", jobNote.getName());
} else {
info.put("notebookName", "Note " + jobNote.getId());
}
// set notebook type ( cron or normal )
if (jobNote.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD)
&& !jobNote.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) {
info.put("notebookType", "cron");
} else {
info.put("notebookType", "normal");
}
// set paragraphs
List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
for (Paragraph paragraph : jobNote.getParagraphs()) {
// check paragraph's status.
if (paragraph.getStatus().isRunning()) {
isNotebookRunning = true;
}
// get data for the job manager.
Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph);
lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph);
paragraphsInfo.add(paragraphItem);
}
// set interpreter bind type
String interpreterGroupName = null;
if (replFactory.getInterpreterSettings(jobNote.getId()) != null
&& replFactory.getInterpreterSettings(jobNote.getId()).size() >= 1) {
interpreterGroupName = replFactory.getInterpreterSettings(jobNote.getId()).get(0).getName();
}
// notebook json object root information.
info.put("interpreter", interpreterGroupName);
info.put("isRunningJob", isNotebookRunning);
info.put("unixTimeLastRun", lastRunningUnixTime);
info.put("paragraphs", paragraphsInfo);
notesInfo.add(info);
return notesInfo;
};
@Test
public void testRestartInterpreterPerNote() throws IOException, InterruptedException {
// create new note
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.addParagraph();
Paragraph p = note.getLastParagraph();
Map config = p.getConfig();
config.put("enabled", true);
// run markdown paragraph.
p.setConfig(config);
p.setText("%md markdown");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
assertEquals("<p>markdown</p>\n", p.getResult().message());
// get md interpreter
InterpreterSetting mdIntpSetting = null;
for (InterpreterSetting setting :
ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
if (setting.getName().equals("md")) {
mdIntpSetting = setting;
break;
}
}
String jsonRequest = "{\"noteId\":\"" + note.getId() + "\"}";
// Restart isolated mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.ISOLATED);
PutMethod put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("isolated interpreter restart:", put, isAllowed());
put.releaseConnection();
// Restart scoped mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.SCOPED);
put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("scoped interpreter restart:", put, isAllowed());
put.releaseConnection();
// Restart shared mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.SHARED);
put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("shared interpreter restart:", put, isAllowed());
put.releaseConnection();
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
String noteId = context.getJobDetail().getJobDataMap().getString("noteId");
Note note = notebook.getNote(noteId);
note.runAll();
while (!note.isTerminated()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
logger.error(e.toString(), e);
}
}
boolean releaseResource = false;
try {
Map<String, Object> config = note.getConfig();
if (config != null && config.containsKey("releaseresource")) {
releaseResource = (boolean) note.getConfig().get("releaseresource");
}
} catch (ClassCastException e) {
logger.error(e.getMessage(), e);
}
if (releaseResource) {
for (InterpreterSetting setting :
notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
notebook.getInterpreterFactory().restart(setting.getId());
}
}
}
/**
* Create new note.
*
* @throws IOException
*/
public Note createNote(List<String> interpreterIds, AuthenticationInfo subject)
throws IOException {
Note note =
new Note(notebookRepo, replFactory, jobListenerFactory, notebookIndex, credentials, this);
synchronized (notes) {
notes.put(note.getId(), note);
}
if (interpreterIds != null) {
bindInterpretersToNote(note.getId(), interpreterIds);
}
notebookIndex.addIndexDoc(note);
note.persist(subject);
fireNoteCreateEvent(note);
return note;
}
public List<InterpreterSetting> getBindedInterpreterSettings(String id) {
Note note = getNote(id);
if (note != null) {
return replFactory.getInterpreterSettings(note.getId());
} else {
return new LinkedList<>();
}
}
private boolean isValidInterpreter(String replName) {
try {
return factory.getInterpreter(user, note.getId(), replName) != null;
} catch (InterpreterException e) {
// ignore this exception, it would be recaught when running paragraph.
return false;
}
}
/**
* import new note REST API
*
* @param req - notebook Json
* @return JSON with new note ID
* @throws IOException
*/
@POST
@Path("import")
@ZeppelinApi
public Response importNotebook(String req) throws IOException {
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note newNote = notebook.importNote(req, null, subject);
return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build();
}
List<String> getBindedInterpreterSettingsIds(String id) {
Note note = getNote(id);
if (note != null) {
return getInterpreterFactory().getInterpreters(note.getId());
} else {
return new LinkedList<>();
}
}
@Test
public void testInterpreterRestart() throws IOException, InterruptedException {
// create new note
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.addParagraph();
Paragraph p = note.getLastParagraph();
Map config = p.getConfig();
config.put("enabled", true);
// run markdown paragraph
p.setConfig(config);
p.setText("%md markdown");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
assertEquals("<p>markdown</p>\n", p.getResult().message());
// restart interpreter
for (InterpreterSetting setting :
ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
if (setting.getName().equals("md")) {
// Call Restart Interpreter REST API
PutMethod put = httpPut("/interpreter/setting/restart/" + setting.getId(), "");
assertThat("test interpreter restart:", put, isAllowed());
put.releaseConnection();
break;
}
}
// run markdown paragraph, again
p = note.addParagraph();
p.setConfig(config);
p.setText("%md markdown restarted");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
assertEquals("<p>markdown restarted</p>\n", p.getResult().message());
// cleanup
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testInterpreterAutoBinding() throws IOException {
// create note
Note note = ZeppelinServer.notebook.createNote(anonymous);
// check interpreter is binded
GetMethod get = httpGet("/notebook/interpreter/bind/" + note.getId());
assertThat(get, isAllowed());
get.addRequestHeader("Origin", "http://localhost");
Map<String, Object> resp =
gson.fromJson(
get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {}.getType());
List<Map<String, String>> body = (List<Map<String, String>>) resp.get("body");
assertTrue(0 < body.size());
get.releaseConnection();
// cleanup
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
private InterpreterSetting getInterpreterSettingById(String id) {
InterpreterSetting setting = null;
for (InterpreterSetting i : factory.getInterpreterSettings(note.getId())) {
if (id.startsWith(i.getId())) {
setting = i;
break;
}
}
return setting;
}
public List<Map<String, Object>> getJobListByParagraphId(String paragraphID) {
String gotNoteId = null;
List<Note> notes = getAllNotes();
for (Note note : notes) {
Paragraph p = note.getParagraph(paragraphID);
if (p != null) {
gotNoteId = note.getId();
}
}
return getJobListBymNotebookId(gotNoteId);
}
/**
* Clone note REST API
*
* @param
* @return JSON with status.CREATED
* @throws IOException, CloneNotSupportedException, IllegalArgumentException
*/
@POST
@Path("{notebookId}")
@ZeppelinApi
public Response cloneNote(@PathParam("notebookId") String notebookId, String message)
throws IOException, CloneNotSupportedException, IllegalArgumentException {
LOG.info("clone notebook by JSON {}", message);
NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class);
String newNoteName = request.getName();
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note newNote = notebook.cloneNote(notebookId, newNoteName, subject);
notebookServer.broadcastNote(newNote);
notebookServer.broadcastNoteList(subject);
return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build();
}
public void initializeJobListenerForParagraph(Paragraph paragraph) {
final Note paragraphNote = paragraph.getNote();
if (paragraphNote.getId().equals(this.getId())) {
throw new IllegalArgumentException(
format(
"The paragraph %s from note %s " + "does not belong to note %s",
paragraph.getId(), paragraphNote.getId(), this.getId()));
}
boolean foundParagraph = false;
for (Paragraph ownParagraph : paragraphs) {
if (paragraph.getId().equals(ownParagraph.getId())) {
paragraph.setListener(this.jobListenerFactory.getParagraphJobListener(this));
foundParagraph = true;
}
}
if (!foundParagraph) {
throw new IllegalArgumentException(
format(
"Cannot find paragraph %s " + "from note %s",
paragraph.getId(), paragraphNote.getId()));
}
}
public List<InterpreterCompletion> getInterpreterCompletion() {
List<InterpreterCompletion> completion = new LinkedList();
for (InterpreterSetting intp : factory.getInterpreterSettings(note.getId())) {
List<InterpreterInfo> intInfo = intp.getInterpreterInfos();
if (intInfo.size() > 1) {
for (InterpreterInfo info : intInfo) {
String name = intp.getName() + "." + info.getName();
completion.add(new InterpreterCompletion(name, name));
}
} else {
completion.add(new InterpreterCompletion(intp.getName(), intp.getName()));
}
}
return completion;
}
public void bindInterpretersToNote(String id, List<String> interpreterSettingIds)
throws IOException {
Note note = getNote(id);
if (note != null) {
List<InterpreterSetting> currentBindings = replFactory.getInterpreterSettings(id);
for (InterpreterSetting setting : currentBindings) {
if (!interpreterSettingIds.contains(setting.getId())) {
fireUnbindInterpreter(note, setting);
}
}
replFactory.setInterpreters(note.getId(), interpreterSettingIds);
// comment out while note.getNoteReplLoader().setInterpreters(...) do the same
// replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds);
}
}
@Override
public void save(Note note, AuthenticationInfo subject) throws IOException {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.setPrettyPrinting();
Gson gson = gsonBuilder.create();
String json = gson.toJson(note);
String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json";
File file = File.createTempFile("note", "json");
try {
Writer writer = new OutputStreamWriter(new FileOutputStream(file));
writer.write(json);
writer.close();
s3client.putObject(new PutObjectRequest(bucketName, key, file));
} catch (AmazonClientException ace) {
throw new IOException("Unable to store note in S3: " + ace, ace);
} finally {
FileUtils.deleteQuietly(file);
}
}
private InterpreterContext getInterpreterContext(InterpreterOutput output) {
AngularObjectRegistry registry = null;
ResourcePool resourcePool = null;
if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<>();
for (Paragraph p : note.getParagraphs()) {
runners.add(new ParagraphRunner(note, note.getId(), p.getId()));
}
final Paragraph self = this;
Credentials credentials = note.getCredentials();
if (authenticationInfo != null) {
UserCredentials userCredentials =
credentials.getUserCredentials(authenticationInfo.getUser());
authenticationInfo.setUserCredentials(userCredentials);
}
InterpreterContext interpreterContext =
new InterpreterContext(
note.getId(),
getId(),
getRequiredReplName(),
this.getTitle(),
this.getText(),
this.getAuthenticationInfo(),
this.getConfig(),
this.settings,
registry,
resourcePool,
runners,
output);
return interpreterContext;
}
public Interpreter getRepl(String name) {
return factory.getInterpreter(user, note.getId(), name);
}
private boolean noteHasInterpreters() {
return !factory.getInterpreterSettings(note.getId()).isEmpty();
}
public InterpreterContextRunner getInterpreterContextRunner() {
return new ParagraphRunner(note, note.getId(), getId());
}
public List<Map<String, Object>> getJobListByUnixTime(
boolean needsReload, long lastUpdateServerUnixTime, AuthenticationInfo subject) {
final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";
if (needsReload) {
try {
reloadAllNotes(subject);
} catch (IOException e) {
logger.error("Fail to reload notes from repository");
}
}
List<Note> notes = getAllNotes();
List<Map<String, Object>> notesInfo = new LinkedList<>();
for (Note note : notes) {
boolean isNotebookRunning = false;
boolean isUpdateNotebook = false;
long lastRunningUnixTime = 0;
Map<String, Object> info = new HashMap<>();
// set notebook ID
info.put("notebookId", note.getId());
// set notebook Name
String notebookName = note.getName();
if (notebookName != null && !notebookName.equals("")) {
info.put("notebookName", note.getName());
} else {
info.put("notebookName", "Note " + note.getId());
}
// set notebook type ( cron or normal )
if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD)
&& !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) {
info.put("notebookType", "cron");
} else {
info.put("notebookType", "normal");
}
// set paragraphs
List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
for (Paragraph paragraph : note.getParagraphs()) {
// check paragraph's status.
if (paragraph.getStatus().isRunning()) {
isNotebookRunning = true;
isUpdateNotebook = true;
}
// get data for the job manager.
Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph);
lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph);
// is update notebook for last server update time.
if (lastRunningUnixTime > lastUpdateServerUnixTime) {
isUpdateNotebook = true;
}
paragraphsInfo.add(paragraphItem);
}
// set interpreter bind type
String interpreterGroupName = null;
if (replFactory.getInterpreterSettings(note.getId()) != null
&& replFactory.getInterpreterSettings(note.getId()).size() >= 1) {
interpreterGroupName = replFactory.getInterpreterSettings(note.getId()).get(0).getName();
}
// not update and not running -> pass
if (!isUpdateNotebook && !isNotebookRunning) {
continue;
}
// notebook json object root information.
info.put("interpreter", interpreterGroupName);
info.put("isRunningJob", isNotebookRunning);
info.put("unixTimeLastRun", lastRunningUnixTime);
info.put("paragraphs", paragraphsInfo);
notesInfo.add(info);
}
return notesInfo;
}
@SuppressWarnings("rawtypes")
private Note loadNoteFromRepo(String id, AuthenticationInfo subject) {
Note note = null;
try {
note = notebookRepo.get(id, subject);
} catch (IOException e) {
logger.error("Failed to load " + id, e);
}
if (note == null) {
return null;
}
// Manually inject ALL dependencies, as DI constructor was NOT used
note.setIndex(this.notebookIndex);
note.setCredentials(this.credentials);
note.setInterpreterFactory(replFactory);
note.setJobListenerFactory(jobListenerFactory);
note.setNotebookRepo(notebookRepo);
Map<String, SnapshotAngularObject> angularObjectSnapshot = new HashMap<>();
// restore angular object --------------
Date lastUpdatedDate = new Date(0);
for (Paragraph p : note.getParagraphs()) {
p.setNote(note);
if (p.getDateFinished() != null && lastUpdatedDate.before(p.getDateFinished())) {
lastUpdatedDate = p.getDateFinished();
}
}
Map<String, List<AngularObject>> savedObjects = note.getAngularObjects();
if (savedObjects != null) {
for (String intpGroupName : savedObjects.keySet()) {
List<AngularObject> objectList = savedObjects.get(intpGroupName);
for (AngularObject object : objectList) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(object.getName());
if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) {
angularObjectSnapshot.put(
object.getName(),
new SnapshotAngularObject(intpGroupName, object, lastUpdatedDate));
}
}
}
}
note.setNoteEventListener(this);
synchronized (notes) {
notes.put(note.getId(), note);
refreshCron(note.getId());
}
for (String name : angularObjectSnapshot.keySet()) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
List<InterpreterSetting> settings = replFactory.get();
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(note.getId());
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
String noteId = snapshot.getAngularObject().getNoteId();
String paragraphId = snapshot.getAngularObject().getParagraphId();
// at this point, remote interpreter process is not created.
// so does not make sense add it to the remote.
//
// therefore instead of addAndNotifyRemoteProcess(), need to use add()
// that results add angularObject only in ZeppelinServer side not remoteProcessSide
registry.add(name, snapshot.getAngularObject().get(), noteId, paragraphId);
}
}
}
return note;
}