private Note getNote(String key) throws IOException {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.setPrettyPrinting();
Gson gson =
gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create();
S3Object s3object;
try {
s3object = s3client.getObject(new GetObjectRequest(bucketName, key));
} catch (AmazonClientException ace) {
throw new IOException("Unable to retrieve object from S3: " + ace, ace);
}
Note note;
try (InputStream ins = s3object.getObjectContent()) {
String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
note = gson.fromJson(json, Note.class);
}
for (Paragraph p : note.getParagraphs()) {
if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) {
p.setStatus(Status.ABORT);
}
}
return note;
}
private Map<String, String> populateParagraphInfo(Paragraph p) {
Map<String, String> info = new HashMap<>();
info.put("id", p.getId());
info.put("status", p.getStatus().toString());
if (p.getDateStarted() != null) {
info.put("started", p.getDateStarted().toString());
}
if (p.getDateFinished() != null) {
info.put("finished", p.getDateFinished().toString());
}
if (p.getStatus().isRunning()) {
info.put("progress", String.valueOf(p.progress()));
} else {
info.put("progress", String.valueOf(100));
}
return info;
}
public List<Map<String, Object>> getJobListBymNotebookId(String notebookID) {
final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";
long lastRunningUnixTime = 0;
boolean isNotebookRunning = false;
Note jobNote = getNote(notebookID);
List<Map<String, Object>> notesInfo = new LinkedList<>();
if (jobNote == null) {
return notesInfo;
}
Map<String, Object> info = new HashMap<>();
info.put("notebookId", jobNote.getId());
String notebookName = jobNote.getName();
if (notebookName != null && !notebookName.equals("")) {
info.put("notebookName", jobNote.getName());
} else {
info.put("notebookName", "Note " + jobNote.getId());
}
// set notebook type ( cron or normal )
if (jobNote.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD)
&& !jobNote.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) {
info.put("notebookType", "cron");
} else {
info.put("notebookType", "normal");
}
// set paragraphs
List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
for (Paragraph paragraph : jobNote.getParagraphs()) {
// check paragraph's status.
if (paragraph.getStatus().isRunning()) {
isNotebookRunning = true;
}
// get data for the job manager.
Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph);
lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph);
paragraphsInfo.add(paragraphItem);
}
// set interpreter bind type
String interpreterGroupName = null;
if (replFactory.getInterpreterSettings(jobNote.getId()) != null
&& replFactory.getInterpreterSettings(jobNote.getId()).size() >= 1) {
interpreterGroupName = replFactory.getInterpreterSettings(jobNote.getId()).get(0).getName();
}
// notebook json object root information.
info.put("interpreter", interpreterGroupName);
info.put("isRunningJob", isNotebookRunning);
info.put("unixTimeLastRun", lastRunningUnixTime);
info.put("paragraphs", paragraphsInfo);
notesInfo.add(info);
return notesInfo;
};
@Test
public void testInterpreterRestart() throws IOException, InterruptedException {
// create new note
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.addParagraph();
Paragraph p = note.getLastParagraph();
Map config = p.getConfig();
config.put("enabled", true);
// run markdown paragraph
p.setConfig(config);
p.setText("%md markdown");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
assertEquals("<p>markdown</p>\n", p.getResult().message());
// restart interpreter
for (InterpreterSetting setting :
ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
if (setting.getName().equals("md")) {
// Call Restart Interpreter REST API
PutMethod put = httpPut("/interpreter/setting/restart/" + setting.getId(), "");
assertThat("test interpreter restart:", put, isAllowed());
put.releaseConnection();
break;
}
}
// run markdown paragraph, again
p = note.addParagraph();
p.setConfig(config);
p.setText("%md markdown restarted");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
assertEquals("<p>markdown restarted</p>\n", p.getResult().message());
// cleanup
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testRestartInterpreterPerNote() throws IOException, InterruptedException {
// create new note
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.addParagraph();
Paragraph p = note.getLastParagraph();
Map config = p.getConfig();
config.put("enabled", true);
// run markdown paragraph.
p.setConfig(config);
p.setText("%md markdown");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
assertEquals("<p>markdown</p>\n", p.getResult().message());
// get md interpreter
InterpreterSetting mdIntpSetting = null;
for (InterpreterSetting setting :
ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
if (setting.getName().equals("md")) {
mdIntpSetting = setting;
break;
}
}
String jsonRequest = "{\"noteId\":\"" + note.getId() + "\"}";
// Restart isolated mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.ISOLATED);
PutMethod put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("isolated interpreter restart:", put, isAllowed());
put.releaseConnection();
// Restart scoped mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.SCOPED);
put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("scoped interpreter restart:", put, isAllowed());
put.releaseConnection();
// Restart shared mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.SHARED);
put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("shared interpreter restart:", put, isAllowed());
put.releaseConnection();
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
private Map<String, Object> getParagraphForJobManagerItem(Paragraph paragraph) {
Map<String, Object> paragraphItem = new HashMap<>();
// set paragraph id
paragraphItem.put("id", paragraph.getId());
// set paragraph name
String paragraphName = paragraph.getTitle();
if (paragraphName != null) {
paragraphItem.put("name", paragraphName);
} else {
paragraphItem.put("name", paragraph.getId());
}
// set status for paragraph.
paragraphItem.put("status", paragraph.getStatus().toString());
return paragraphItem;
}
public List<Map<String, Object>> getJobListByUnixTime(
boolean needsReload, long lastUpdateServerUnixTime, AuthenticationInfo subject) {
final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";
if (needsReload) {
try {
reloadAllNotes(subject);
} catch (IOException e) {
logger.error("Fail to reload notes from repository");
}
}
List<Note> notes = getAllNotes();
List<Map<String, Object>> notesInfo = new LinkedList<>();
for (Note note : notes) {
boolean isNotebookRunning = false;
boolean isUpdateNotebook = false;
long lastRunningUnixTime = 0;
Map<String, Object> info = new HashMap<>();
// set notebook ID
info.put("notebookId", note.getId());
// set notebook Name
String notebookName = note.getName();
if (notebookName != null && !notebookName.equals("")) {
info.put("notebookName", note.getName());
} else {
info.put("notebookName", "Note " + note.getId());
}
// set notebook type ( cron or normal )
if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD)
&& !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) {
info.put("notebookType", "cron");
} else {
info.put("notebookType", "normal");
}
// set paragraphs
List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
for (Paragraph paragraph : note.getParagraphs()) {
// check paragraph's status.
if (paragraph.getStatus().isRunning()) {
isNotebookRunning = true;
isUpdateNotebook = true;
}
// get data for the job manager.
Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph);
lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph);
// is update notebook for last server update time.
if (lastRunningUnixTime > lastUpdateServerUnixTime) {
isUpdateNotebook = true;
}
paragraphsInfo.add(paragraphItem);
}
// set interpreter bind type
String interpreterGroupName = null;
if (replFactory.getInterpreterSettings(note.getId()) != null
&& replFactory.getInterpreterSettings(note.getId()).size() >= 1) {
interpreterGroupName = replFactory.getInterpreterSettings(note.getId()).get(0).getName();
}
// not update and not running -> pass
if (!isUpdateNotebook && !isNotebookRunning) {
continue;
}
// notebook json object root information.
info.put("interpreter", interpreterGroupName);
info.put("isRunningJob", isNotebookRunning);
info.put("unixTimeLastRun", lastRunningUnixTime);
info.put("paragraphs", paragraphsInfo);
notesInfo.add(info);
}
return notesInfo;
}
