/** Clear all paragraph output of note */
public void clearAllParagraphOutput() {
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
p.setReturn(null, null);
}
}
}
/**
* Create new note REST API
*
* @param message - JSON with new note name
* @return JSON with new note ID
* @throws IOException
*/
@POST
@Path("/")
@ZeppelinApi
public Response createNote(String message) throws IOException {
LOG.info("Create new notebook by JSON {}", message);
NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note note = notebook.createNote(subject);
List<NewParagraphRequest> initialParagraphs = request.getParagraphs();
if (initialParagraphs != null) {
for (NewParagraphRequest paragraphRequest : initialParagraphs) {
Paragraph p = note.addParagraph();
p.setTitle(paragraphRequest.getTitle());
p.setText(paragraphRequest.getText());
}
}
note.addParagraph(); // add one paragraph to the last
String noteName = request.getName();
if (noteName.isEmpty()) {
noteName = "Note " + note.getId();
}
note.setName(noteName);
note.persist(subject);
notebookServer.broadcastNote(note);
notebookServer.broadcastNoteList(subject);
return new JsonResponse<>(Status.CREATED, "", note.getId()).build();
}
/**
* Insert paragraph REST API
*
* @param message - JSON containing paragraph's information
* @return JSON with status.OK
* @throws IOException
*/
@POST
@Path("{notebookId}/paragraph")
@ZeppelinApi
public Response insertParagraph(@PathParam("notebookId") String notebookId, String message)
throws IOException {
LOG.info("insert paragraph {} {}", notebookId, message);
Note note = notebook.getNote(notebookId);
if (note == null) {
return new JsonResponse(Status.NOT_FOUND, "note not found.").build();
}
NewParagraphRequest request = gson.fromJson(message, NewParagraphRequest.class);
Paragraph p;
Double indexDouble = request.getIndex();
if (indexDouble == null) {
p = note.addParagraph();
} else {
p = note.insertParagraph(indexDouble.intValue());
}
p.setTitle(request.getTitle());
p.setText(request.getText());
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.persist(subject);
notebookServer.broadcastNote(note);
return new JsonResponse(Status.CREATED, "", p.getId()).build();
}
/**
* Run paragraph job REST API
*
* @param message - JSON with params if user wants to update dynamic form's value null, empty
* string, empty json if user doesn't want to update
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@POST
@Path("job/{notebookId}/{paragraphId}")
@ZeppelinApi
public Response runParagraph(
@PathParam("notebookId") String notebookId,
@PathParam("paragraphId") String paragraphId,
String message)
throws IOException, IllegalArgumentException {
LOG.info("run paragraph job {} {} {}", notebookId, paragraphId, message);
Note note = notebook.getNote(notebookId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
Paragraph paragraph = note.getParagraph(paragraphId);
if (paragraph == null) {
return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build();
}
// handle params if presented
if (!StringUtils.isEmpty(message)) {
RunParagraphWithParametersRequest request =
gson.fromJson(message, RunParagraphWithParametersRequest.class);
Map<String, Object> paramsForUpdating = request.getParams();
if (paramsForUpdating != null) {
paragraph.settings.getParams().putAll(paramsForUpdating);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.persist(subject);
}
}
note.run(paragraph.getId());
return new JsonResponse<>(Status.OK).build();
}
private Note getNote(String key) throws IOException {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.setPrettyPrinting();
Gson gson =
gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create();
S3Object s3object;
try {
s3object = s3client.getObject(new GetObjectRequest(bucketName, key));
} catch (AmazonClientException ace) {
throw new IOException("Unable to retrieve object from S3: " + ace, ace);
}
Note note;
try (InputStream ins = s3object.getObjectContent()) {
String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
note = gson.fromJson(json, Note.class);
}
for (Paragraph p : note.getParagraphs()) {
if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) {
p.setStatus(Status.ABORT);
}
}
return note;
}
private InterpreterContext getInterpreterContext() {
AngularObjectRegistry registry = null;
if (!getNoteReplLoader().getInterpreterSettings().isEmpty()) {
InterpreterSetting intpGroup = getNoteReplLoader().getInterpreterSettings().get(0);
registry = intpGroup.getInterpreterGroup().getAngularObjectRegistry();
}
List<InterpreterContextRunner> runners = new LinkedList<InterpreterContextRunner>();
for (Paragraph p : note.getParagraphs()) {
runners.add(new ParagraphRunner(note, note.id(), p.getId()));
}
InterpreterContext interpreterContext =
new InterpreterContext(
note.id(),
getId(),
this.getTitle(),
this.getText(),
this.getConfig(),
this.settings,
registry,
runners);
return interpreterContext;
}
void setInterpreterFactory(InterpreterFactory factory) {
this.factory = factory;
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
p.setInterpreterFactory(factory);
}
}
}
private InterpreterContext getInterpreterContext() {
AngularObjectRegistry registry = null;
ResourcePool resourcePool = null;
if (!getNoteReplLoader().getInterpreterSettings().isEmpty()) {
InterpreterSetting intpGroup = getNoteReplLoader().getInterpreterSettings().get(0);
registry = intpGroup.getInterpreterGroup().getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup().getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<InterpreterContextRunner>();
for (Paragraph p : note.getParagraphs()) {
runners.add(new ParagraphRunner(note, note.id(), p.getId()));
}
final Paragraph self = this;
InterpreterContext interpreterContext =
new InterpreterContext(
note.id(),
getId(),
this.getTitle(),
this.getText(),
this.getConfig(),
this.settings,
registry,
resourcePool,
runners,
new InterpreterOutput(
new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
updateParagraphResult(out);
((ParagraphJobListener) getListener())
.onOutputAppend(self, out, new String(line));
}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
updateParagraphResult(out);
((ParagraphJobListener) getListener())
.onOutputUpdate(self, out, new String(output));
}
private void updateParagraphResult(InterpreterOutput out) {
// update paragraph result
Throwable t = null;
String message = null;
try {
message = new String(out.toByteArray());
} catch (IOException e) {
logger().error(e.getMessage(), e);
t = e;
}
setReturn(new InterpreterResult(Code.SUCCESS, out.getType(), message), t);
}
}));
return interpreterContext;
}
public List<Map<String, Object>> getJobListBymNotebookId(String notebookID) {
final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";
long lastRunningUnixTime = 0;
boolean isNotebookRunning = false;
Note jobNote = getNote(notebookID);
List<Map<String, Object>> notesInfo = new LinkedList<>();
if (jobNote == null) {
return notesInfo;
}
Map<String, Object> info = new HashMap<>();
info.put("notebookId", jobNote.getId());
String notebookName = jobNote.getName();
if (notebookName != null && !notebookName.equals("")) {
info.put("notebookName", jobNote.getName());
} else {
info.put("notebookName", "Note " + jobNote.getId());
}
// set notebook type ( cron or normal )
if (jobNote.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD)
&& !jobNote.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) {
info.put("notebookType", "cron");
} else {
info.put("notebookType", "normal");
}
// set paragraphs
List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
for (Paragraph paragraph : jobNote.getParagraphs()) {
// check paragraph's status.
if (paragraph.getStatus().isRunning()) {
isNotebookRunning = true;
}
// get data for the job manager.
Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph);
lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph);
paragraphsInfo.add(paragraphItem);
}
// set interpreter bind type
String interpreterGroupName = null;
if (replFactory.getInterpreterSettings(jobNote.getId()) != null
&& replFactory.getInterpreterSettings(jobNote.getId()).size() >= 1) {
interpreterGroupName = replFactory.getInterpreterSettings(jobNote.getId()).get(0).getName();
}
// notebook json object root information.
info.put("interpreter", interpreterGroupName);
info.put("isRunningJob", isNotebookRunning);
info.put("unixTimeLastRun", lastRunningUnixTime);
info.put("paragraphs", paragraphsInfo);
notesInfo.add(info);
return notesInfo;
};
public void removeNote(String id, AuthenticationInfo subject) {
Note note;
synchronized (notes) {
note = notes.remove(id);
}
replFactory.removeNoteInterpreterSettingBinding(id);
notebookIndex.deleteIndexDocs(note);
notebookAuthorization.removeNote(id);
// remove from all interpreter instance's angular object registry
for (InterpreterSetting settings : replFactory.get()) {
AngularObjectRegistry registry = settings.getInterpreterGroup(id).getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
// remove paragraph scope object
for (Paragraph p : note.getParagraphs()) {
((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, p.getId());
// remove app scope object
List<ApplicationState> appStates = p.getAllApplicationStates();
if (appStates != null) {
for (ApplicationState app : appStates) {
((RemoteAngularObjectRegistry) registry)
.removeAllAndNotifyRemoteProcess(id, app.getId());
}
}
}
// remove notebook scope object
((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, null);
} else {
// remove paragraph scope object
for (Paragraph p : note.getParagraphs()) {
registry.removeAll(id, p.getId());
// remove app scope object
List<ApplicationState> appStates = p.getAllApplicationStates();
if (appStates != null) {
for (ApplicationState app : appStates) {
registry.removeAll(id, app.getId());
}
}
}
// remove notebook scope object
registry.removeAll(id, null);
}
}
ResourcePoolUtils.removeResourcesBelongsToNote(id);
fireNoteRemoveEvent(note);
try {
note.unpersist(subject);
} catch (IOException e) {
logger.error(e.toString(), e);
}
}
public Map<String, String> generateSingleParagraphInfo(String paragraphId) {
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
if (p.getId().equals(paragraphId)) {
return populateParagraphInfo(p);
}
}
return new HashMap<>();
}
}
public Paragraph getParagraph(String paragraphId) {
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
if (p.getId().equals(paragraphId)) {
return p;
}
}
}
return null;
}
/**
* Run a single paragraph.
*
* @param paragraphId
*/
public void run(String paragraphId) {
Paragraph p = getParagraph(paragraphId);
p.setNoteReplLoader(replLoader);
p.setListener(jobListenerFactory.getParagraphJobListener(this));
Interpreter intp = replLoader.get(p.getRequiredReplName());
if (intp == null) {
throw new InterpreterException("Interpreter " + p.getRequiredReplName() + " not found");
}
intp.getScheduler().submit(p);
}
/**
* Run all paragraphs sequentially.
*
* @param jobListener
*/
public void runAll() {
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
p.setNoteReplLoader(replLoader);
p.setListener(jobListenerFactory.getParagraphJobListener(this));
Interpreter intp = replLoader.get(p.getRequiredReplName());
intp.getScheduler().submit(p);
}
}
}
/** Check whether all paragraphs belongs to this note has terminated */
boolean isTerminated() {
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
if (!p.isTerminated()) {
return false;
}
}
}
return true;
}
/**
* Clear paragraph output by id.
*
* @param paragraphId ID of paragraph
* @return Paragraph
*/
public Paragraph clearParagraphOutput(String paragraphId) {
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
if (p.getId().equals(paragraphId)) {
p.setReturn(null, null);
return p;
}
}
}
return null;
}
/**
* Remove paragraph by id.
*
* @param paragraphId
* @return
*/
public Paragraph removeParagraph(String paragraphId) {
synchronized (paragraphs) {
for (int i = 0; i < paragraphs.size(); i++) {
Paragraph p = paragraphs.get(i);
if (p.getId().equals(paragraphId)) {
paragraphs.remove(i);
return p;
}
}
}
return null;
}
/**
* Insert paragraph in given index.
*
* @param index index of paragraphs
*/
public Paragraph insertParagraph(int index, AuthenticationInfo authenticationInfo) {
Paragraph p = new Paragraph(this, this, factory);
p.setAuthenticationInfo(authenticationInfo);
setParagraphMagic(p, index);
synchronized (paragraphs) {
paragraphs.add(index, p);
}
if (noteEventListener != null) {
noteEventListener.onParagraphCreate(p);
}
return p;
}
/**
* Clear paragraph output by id.
*
* @param paragraphId
* @return
*/
public Paragraph clearParagraphOutput(String paragraphId) {
synchronized (paragraphs) {
for (int i = 0; i < paragraphs.size(); i++) {
Paragraph p = paragraphs.get(i);
if (p.getId().equals(paragraphId)) {
p.setReturn(null, null);
return p;
}
}
}
return null;
}
/** Run all paragraphs sequentially. */
public synchronized void runAll() {
String cronExecutingUser = (String) getConfig().get("cronExecutingUser");
if (null == cronExecutingUser) {
cronExecutingUser = "******";
}
for (Paragraph p : getParagraphs()) {
if (!p.isEnabled()) {
continue;
}
AuthenticationInfo authenticationInfo = new AuthenticationInfo();
authenticationInfo.setUser(cronExecutingUser);
p.setAuthenticationInfo(authenticationInfo);
run(p.getId());
}
}
/**
* Return new note for specific user. this inserts and replaces user paragraph which doesn't
* exists in original paragraph
*
* @param user specific user
* @return new Note for the user
*/
public Note getUserNote(String user) {
Note newNote = new Note();
newNote.id = getId();
newNote.config = getConfig();
newNote.angularObjects = getAngularObjects();
Paragraph newParagraph;
for (Paragraph p : paragraphs) {
newParagraph = p.getUserParagraph(user);
if (null == newParagraph) {
newParagraph = p.cloneParagraphForUser(user);
}
newNote.paragraphs.add(newParagraph);
}
return newNote;
}
/**
* Stop(delete) notebook jobs REST API
*
* @param
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@DELETE
@Path("job/{notebookId}")
@ZeppelinApi
public Response stopNoteJobs(@PathParam("notebookId") String notebookId)
throws IOException, IllegalArgumentException {
LOG.info("stop notebook jobs {} ", notebookId);
Note note = notebook.getNote(notebookId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
for (Paragraph p : note.getParagraphs()) {
if (!p.isTerminated()) {
p.abort();
}
}
return new JsonResponse<>(Status.OK).build();
}
private Map<String, Object> getParagraphForJobManagerItem(Paragraph paragraph) {
Map<String, Object> paragraphItem = new HashMap<>();
// set paragraph id
paragraphItem.put("id", paragraph.getId());
// set paragraph name
String paragraphName = paragraph.getTitle();
if (paragraphName != null) {
paragraphItem.put("name", paragraphName);
} else {
paragraphItem.put("name", paragraph.getId());
}
// set status for paragraph.
paragraphItem.put("status", paragraph.getStatus().toString());
return paragraphItem;
}
/**
* Stop(delete) paragraph job REST API
*
* @param
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@DELETE
@Path("job/{notebookId}/{paragraphId}")
@ZeppelinApi
public Response stopParagraph(
@PathParam("notebookId") String notebookId, @PathParam("paragraphId") String paragraphId)
throws IOException, IllegalArgumentException {
LOG.info("stop paragraph job {} ", notebookId);
Note note = notebook.getNote(notebookId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
Paragraph p = note.getParagraph(paragraphId);
if (p == null) {
return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build();
}
p.abort();
return new JsonResponse<>(Status.OK).build();
}
/**
* Stop(delete) paragraph job REST API
*
* @param notebookId ID of Notebook
* @param paragraphId ID of Paragraph
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@DELETE
@Path("job/{notebookId}/{paragraphId}")
@ZeppelinApi
public Response stopParagraph(
@PathParam("notebookId") String notebookId, @PathParam("paragraphId") String paragraphId)
throws IOException, IllegalArgumentException {
/** TODO(jl): Fixed notebookId to noteId https://issues.apache.org/jira/browse/ZEPPELIN-1163 */
LOG.info("stop paragraph job {} ", notebookId);
Note note = notebook.getNote(notebookId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
Paragraph p = note.getParagraph(paragraphId);
if (p == null) {
return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build();
}
p.abort();
return new JsonResponse<>(Status.OK).build();
}
/**
* Remove paragraph by id.
*
* @param paragraphId ID of paragraph
* @return a paragraph that was deleted, or <code>null</code> otherwise
*/
public Paragraph removeParagraph(String user, String paragraphId) {
removeAllAngularObjectInParagraph(user, paragraphId);
ResourcePoolUtils.removeResourcesBelongsToParagraph(getId(), paragraphId);
synchronized (paragraphs) {
Iterator<Paragraph> i = paragraphs.iterator();
while (i.hasNext()) {
Paragraph p = i.next();
if (p.getId().equals(paragraphId)) {
index.deleteIndexDoc(this, p);
i.remove();
if (noteEventListener != null) {
noteEventListener.onParagraphRemove(p);
}
return p;
}
}
}
return null;
}
@Test
public void testRestartInterpreterPerNote() throws IOException, InterruptedException {
// create new note
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.addParagraph();
Paragraph p = note.getLastParagraph();
Map config = p.getConfig();
config.put("enabled", true);
// run markdown paragraph.
p.setConfig(config);
p.setText("%md markdown");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
assertEquals("<p>markdown</p>\n", p.getResult().message());
// get md interpreter
InterpreterSetting mdIntpSetting = null;
for (InterpreterSetting setting :
ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
if (setting.getName().equals("md")) {
mdIntpSetting = setting;
break;
}
}
String jsonRequest = "{\"noteId\":\"" + note.getId() + "\"}";
// Restart isolated mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.ISOLATED);
PutMethod put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("isolated interpreter restart:", put, isAllowed());
put.releaseConnection();
// Restart scoped mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.SCOPED);
put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("scoped interpreter restart:", put, isAllowed());
put.releaseConnection();
// Restart shared mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.SHARED);
put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("shared interpreter restart:", put, isAllowed());
put.releaseConnection();
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
private InterpreterContext getInterpreterContext(InterpreterOutput output) {
AngularObjectRegistry registry = null;
ResourcePool resourcePool = null;
if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<>();
for (Paragraph p : note.getParagraphs()) {
runners.add(new ParagraphRunner(note, note.getId(), p.getId()));
}
final Paragraph self = this;
Credentials credentials = note.getCredentials();
if (authenticationInfo != null) {
UserCredentials userCredentials =
credentials.getUserCredentials(authenticationInfo.getUser());
authenticationInfo.setUserCredentials(userCredentials);
}
InterpreterContext interpreterContext =
new InterpreterContext(
note.getId(),
getId(),
getRequiredReplName(),
this.getTitle(),
this.getText(),
this.getAuthenticationInfo(),
this.getConfig(),
this.settings,
registry,
resourcePool,
runners,
output);
return interpreterContext;
}
private long getUnixTimeLastRunParagraph(Paragraph paragraph) {
Date lastRunningDate;
long lastRunningUnixTime;
Date paragaraphDate = paragraph.getDateStarted();
// diff started time <-> finishied time
if (paragaraphDate == null) {
paragaraphDate = paragraph.getDateFinished();
} else {
if (paragraph.getDateFinished() != null
&& paragraph.getDateFinished().after(paragaraphDate)) {
paragaraphDate = paragraph.getDateFinished();
}
}
// finished time and started time is not exists.
if (paragaraphDate == null) {
paragaraphDate = paragraph.getDateCreated();
}
// set last update unixtime(ms).
lastRunningDate = paragaraphDate;
lastRunningUnixTime = lastRunningDate.getTime();
return lastRunningUnixTime;
}
private void setParagraphMagic(Paragraph p, int index) {
if (paragraphs.size() > 0) {
String magic;
if (index == 0) {
magic = paragraphs.get(0).getMagic();
} else {
magic = paragraphs.get(index - 1).getMagic();
}
if (StringUtils.isNotEmpty(magic)) {
p.setText(magic + "\n");
}
}
}