/**
* Clone paragraph and add it to note.
*
* @param srcParagraph source paragraph
*/
void addCloneParagraph(Paragraph srcParagraph) {
// Keep paragraph original ID
final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory);
Map<String, Object> config = new HashMap<>(srcParagraph.getConfig());
Map<String, Object> param = new HashMap<>(srcParagraph.settings.getParams());
Map<String, Input> form = new HashMap<>(srcParagraph.settings.getForms());
newParagraph.setConfig(config);
newParagraph.settings.setParams(param);
newParagraph.settings.setForms(form);
newParagraph.setText(srcParagraph.getText());
newParagraph.setTitle(srcParagraph.getTitle());
try {
Gson gson = new Gson();
String resultJson = gson.toJson(srcParagraph.getReturn());
InterpreterResult result = gson.fromJson(resultJson, InterpreterResult.class);
newParagraph.setReturn(result, null);
} catch (Exception e) {
// 'result' part of Note consists of exception, instead of actual interpreter results
logger.warn(
"Paragraph " + srcParagraph.getId() + " has a result with exception. " + e.getMessage());
}
synchronized (paragraphs) {
paragraphs.add(newParagraph);
}
if (noteEventListener != null) {
noteEventListener.onParagraphCreate(newParagraph);
}
}
@Test
public void testSyncUpdateMain() throws IOException {
/* create note */
Note note = notebookSync.createNote();
Paragraph p1 = note.addParagraph();
Map config = p1.getConfig();
config.put("enabled", true);
p1.setConfig(config);
p1.setText("hello world");
/* new paragraph exists in note instance */
assertEquals(1, note.getParagraphs().size());
/* new paragraph not yet saved into storages */
assertEquals(
0, notebookRepoSync.get(0, notebookRepoSync.list(0).get(0).getId()).getParagraphs().size());
assertEquals(
0, notebookRepoSync.get(1, notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());
/* save to storage under index 0 (first storage) */
notebookRepoSync.save(0, note);
/* check paragraph saved to first storage */
assertEquals(
1, notebookRepoSync.get(0, notebookRepoSync.list(0).get(0).getId()).getParagraphs().size());
/* check paragraph isn't saved to second storage */
assertEquals(
0, notebookRepoSync.get(1, notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());
/* apply sync */
notebookRepoSync.sync();
/* check whether added to second storage */
assertEquals(
1, notebookRepoSync.get(1, notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());
/* check whether same paragraph id */
assertEquals(
p1.getId(),
notebookRepoSync
.get(0, notebookRepoSync.list(0).get(0).getId())
.getLastParagraph()
.getId());
assertEquals(
p1.getId(),
notebookRepoSync
.get(1, notebookRepoSync.list(1).get(0).getId())
.getLastParagraph()
.getId());
}
/**
* Run a single paragraph.
*
* @param paragraphId ID of paragraph
*/
public void run(String paragraphId) {
Paragraph p = getParagraph(paragraphId);
if (p.isBlankParagraph()) {
logger.info("skip to run blank paragraph. {}", p.getId());
return;
}
p.setListener(jobListenerFactory.getParagraphJobListener(this));
String requiredReplName = p.getRequiredReplName();
Interpreter intp = factory.getInterpreter(p.getUser(), getId(), requiredReplName);
if (intp == null) {
String intpExceptionMsg =
p.getJobName() + "'s Interpreter " + requiredReplName + " not found";
InterpreterException intpException = new InterpreterException(intpExceptionMsg);
InterpreterResult intpResult =
new InterpreterResult(InterpreterResult.Code.ERROR, intpException.getMessage());
p.setReturn(intpResult, intpException);
p.setStatus(Job.Status.ERROR);
throw intpException;
}
if (p.getConfig().get("enabled") == null || (Boolean) p.getConfig().get("enabled")) {
p.setAuthenticationInfo(p.getAuthenticationInfo());
intp.getScheduler().submit(p);
}
}
private InterpreterContext getInterpreterContext() {
AngularObjectRegistry registry = null;
if (!getNoteReplLoader().getInterpreterSettings().isEmpty()) {
InterpreterSetting intpGroup = getNoteReplLoader().getInterpreterSettings().get(0);
registry = intpGroup.getInterpreterGroup().getAngularObjectRegistry();
}
List<InterpreterContextRunner> runners = new LinkedList<InterpreterContextRunner>();
for (Paragraph p : note.getParagraphs()) {
runners.add(new ParagraphRunner(note, note.id(), p.getId()));
}
InterpreterContext interpreterContext =
new InterpreterContext(
note.id(),
getId(),
this.getTitle(),
this.getText(),
this.getConfig(),
this.settings,
registry,
runners);
return interpreterContext;
}
/**
* Run paragraph job REST API
*
* @param message - JSON with params if user wants to update dynamic form's value null, empty
* string, empty json if user doesn't want to update
* @return JSON with status.OK
* @throws IOException, IllegalArgumentException
*/
@POST
@Path("job/{notebookId}/{paragraphId}")
@ZeppelinApi
public Response runParagraph(
@PathParam("notebookId") String notebookId,
@PathParam("paragraphId") String paragraphId,
String message)
throws IOException, IllegalArgumentException {
LOG.info("run paragraph job {} {} {}", notebookId, paragraphId, message);
Note note = notebook.getNote(notebookId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
Paragraph paragraph = note.getParagraph(paragraphId);
if (paragraph == null) {
return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build();
}
// handle params if presented
if (!StringUtils.isEmpty(message)) {
RunParagraphWithParametersRequest request =
gson.fromJson(message, RunParagraphWithParametersRequest.class);
Map<String, Object> paramsForUpdating = request.getParams();
if (paramsForUpdating != null) {
paragraph.settings.getParams().putAll(paramsForUpdating);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.persist(subject);
}
}
note.run(paragraph.getId());
return new JsonResponse<>(Status.OK).build();
}
/**
* Insert paragraph REST API
*
* @param message - JSON containing paragraph's information
* @return JSON with status.OK
* @throws IOException
*/
@POST
@Path("{notebookId}/paragraph")
@ZeppelinApi
public Response insertParagraph(@PathParam("notebookId") String notebookId, String message)
throws IOException {
LOG.info("insert paragraph {} {}", notebookId, message);
Note note = notebook.getNote(notebookId);
if (note == null) {
return new JsonResponse(Status.NOT_FOUND, "note not found.").build();
}
NewParagraphRequest request = gson.fromJson(message, NewParagraphRequest.class);
Paragraph p;
Double indexDouble = request.getIndex();
if (indexDouble == null) {
p = note.addParagraph();
} else {
p = note.insertParagraph(indexDouble.intValue());
}
p.setTitle(request.getTitle());
p.setText(request.getText());
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.persist(subject);
notebookServer.broadcastNote(note);
return new JsonResponse(Status.CREATED, "", p.getId()).build();
}
private InterpreterContext getInterpreterContext() {
AngularObjectRegistry registry = null;
ResourcePool resourcePool = null;
if (!getNoteReplLoader().getInterpreterSettings().isEmpty()) {
InterpreterSetting intpGroup = getNoteReplLoader().getInterpreterSettings().get(0);
registry = intpGroup.getInterpreterGroup().getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup().getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<InterpreterContextRunner>();
for (Paragraph p : note.getParagraphs()) {
runners.add(new ParagraphRunner(note, note.id(), p.getId()));
}
final Paragraph self = this;
InterpreterContext interpreterContext =
new InterpreterContext(
note.id(),
getId(),
this.getTitle(),
this.getText(),
this.getConfig(),
this.settings,
registry,
resourcePool,
runners,
new InterpreterOutput(
new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
updateParagraphResult(out);
((ParagraphJobListener) getListener())
.onOutputAppend(self, out, new String(line));
}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
updateParagraphResult(out);
((ParagraphJobListener) getListener())
.onOutputUpdate(self, out, new String(output));
}
private void updateParagraphResult(InterpreterOutput out) {
// update paragraph result
Throwable t = null;
String message = null;
try {
message = new String(out.toByteArray());
} catch (IOException e) {
logger().error(e.getMessage(), e);
t = e;
}
setReturn(new InterpreterResult(Code.SUCCESS, out.getType(), message), t);
}
}));
return interpreterContext;
}
public void removeNote(String id, AuthenticationInfo subject) {
Note note;
synchronized (notes) {
note = notes.remove(id);
}
replFactory.removeNoteInterpreterSettingBinding(id);
notebookIndex.deleteIndexDocs(note);
notebookAuthorization.removeNote(id);
// remove from all interpreter instance's angular object registry
for (InterpreterSetting settings : replFactory.get()) {
AngularObjectRegistry registry = settings.getInterpreterGroup(id).getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
// remove paragraph scope object
for (Paragraph p : note.getParagraphs()) {
((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, p.getId());
// remove app scope object
List<ApplicationState> appStates = p.getAllApplicationStates();
if (appStates != null) {
for (ApplicationState app : appStates) {
((RemoteAngularObjectRegistry) registry)
.removeAllAndNotifyRemoteProcess(id, app.getId());
}
}
}
// remove notebook scope object
((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, null);
} else {
// remove paragraph scope object
for (Paragraph p : note.getParagraphs()) {
registry.removeAll(id, p.getId());
// remove app scope object
List<ApplicationState> appStates = p.getAllApplicationStates();
if (appStates != null) {
for (ApplicationState app : appStates) {
registry.removeAll(id, app.getId());
}
}
}
// remove notebook scope object
registry.removeAll(id, null);
}
}
ResourcePoolUtils.removeResourcesBelongsToNote(id);
fireNoteRemoveEvent(note);
try {
note.unpersist(subject);
} catch (IOException e) {
logger.error(e.toString(), e);
}
}
@Test
public void testInterpreterRestart() throws IOException, InterruptedException {
// create new note
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.addParagraph();
Paragraph p = note.getLastParagraph();
Map config = p.getConfig();
config.put("enabled", true);
// run markdown paragraph
p.setConfig(config);
p.setText("%md markdown");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
assertEquals("<p>markdown</p>\n", p.getResult().message());
// restart interpreter
for (InterpreterSetting setting :
ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
if (setting.getName().equals("md")) {
// Call Restart Interpreter REST API
PutMethod put = httpPut("/interpreter/setting/restart/" + setting.getId(), "");
assertThat("test interpreter restart:", put, isAllowed());
put.releaseConnection();
break;
}
}
// run markdown paragraph, again
p = note.addParagraph();
p.setConfig(config);
p.setText("%md markdown restarted");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
assertEquals("<p>markdown restarted</p>\n", p.getResult().message());
// cleanup
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
public Paragraph getParagraph(String paragraphId) {
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
if (p.getId().equals(paragraphId)) {
return p;
}
}
}
return null;
}
public Map<String, String> generateSingleParagraphInfo(String paragraphId) {
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
if (p.getId().equals(paragraphId)) {
return populateParagraphInfo(p);
}
}
return new HashMap<>();
}
}
private Map<String, Object> getParagraphForJobManagerItem(Paragraph paragraph) {
Map<String, Object> paragraphItem = new HashMap<>();
// set paragraph id
paragraphItem.put("id", paragraph.getId());
// set paragraph name
String paragraphName = paragraph.getTitle();
if (paragraphName != null) {
paragraphItem.put("name", paragraphName);
} else {
paragraphItem.put("name", paragraph.getId());
}
// set status for paragraph.
paragraphItem.put("status", paragraph.getStatus().toString());
return paragraphItem;
}
/**
* Clear paragraph output by id.
*
* @param paragraphId ID of paragraph
* @return Paragraph
*/
public Paragraph clearParagraphOutput(String paragraphId) {
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
if (p.getId().equals(paragraphId)) {
p.setReturn(null, null);
return p;
}
}
}
return null;
}
/**
* Clear paragraph output by id.
*
* @param paragraphId
* @return
*/
public Paragraph clearParagraphOutput(String paragraphId) {
synchronized (paragraphs) {
for (int i = 0; i < paragraphs.size(); i++) {
Paragraph p = paragraphs.get(i);
if (p.getId().equals(paragraphId)) {
p.setReturn(null, null);
return p;
}
}
}
return null;
}
/**
* Remove paragraph by id.
*
* @param paragraphId
* @return
*/
public Paragraph removeParagraph(String paragraphId) {
synchronized (paragraphs) {
for (int i = 0; i < paragraphs.size(); i++) {
Paragraph p = paragraphs.get(i);
if (p.getId().equals(paragraphId)) {
paragraphs.remove(i);
return p;
}
}
}
return null;
}
@Test
public void testRestartInterpreterPerNote() throws IOException, InterruptedException {
// create new note
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.addParagraph();
Paragraph p = note.getLastParagraph();
Map config = p.getConfig();
config.put("enabled", true);
// run markdown paragraph.
p.setConfig(config);
p.setText("%md markdown");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
assertEquals("<p>markdown</p>\n", p.getResult().message());
// get md interpreter
InterpreterSetting mdIntpSetting = null;
for (InterpreterSetting setting :
ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
if (setting.getName().equals("md")) {
mdIntpSetting = setting;
break;
}
}
String jsonRequest = "{\"noteId\":\"" + note.getId() + "\"}";
// Restart isolated mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.ISOLATED);
PutMethod put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("isolated interpreter restart:", put, isAllowed());
put.releaseConnection();
// Restart scoped mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.SCOPED);
put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("scoped interpreter restart:", put, isAllowed());
put.releaseConnection();
// Restart shared mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.SHARED);
put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("shared interpreter restart:", put, isAllowed());
put.releaseConnection();
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
/** Run all paragraphs sequentially. */
public synchronized void runAll() {
String cronExecutingUser = (String) getConfig().get("cronExecutingUser");
if (null == cronExecutingUser) {
cronExecutingUser = "******";
}
for (Paragraph p : getParagraphs()) {
if (!p.isEnabled()) {
continue;
}
AuthenticationInfo authenticationInfo = new AuthenticationInfo();
authenticationInfo.setUser(cronExecutingUser);
p.setAuthenticationInfo(authenticationInfo);
run(p.getId());
}
}
public void initializeJobListenerForParagraph(Paragraph paragraph) {
final Note paragraphNote = paragraph.getNote();
if (paragraphNote.getId().equals(this.getId())) {
throw new IllegalArgumentException(
format(
"The paragraph %s from note %s " + "does not belong to note %s",
paragraph.getId(), paragraphNote.getId(), this.getId()));
}
boolean foundParagraph = false;
for (Paragraph ownParagraph : paragraphs) {
if (paragraph.getId().equals(ownParagraph.getId())) {
paragraph.setListener(this.jobListenerFactory.getParagraphJobListener(this));
foundParagraph = true;
}
}
if (!foundParagraph) {
throw new IllegalArgumentException(
format(
"Cannot find paragraph %s " + "from note %s",
paragraph.getId(), paragraphNote.getId()));
}
}
private Map<String, String> populateParagraphInfo(Paragraph p) {
Map<String, String> info = new HashMap<>();
info.put("id", p.getId());
info.put("status", p.getStatus().toString());
if (p.getDateStarted() != null) {
info.put("started", p.getDateStarted().toString());
}
if (p.getDateFinished() != null) {
info.put("finished", p.getDateFinished().toString());
}
if (p.getStatus().isRunning()) {
info.put("progress", String.valueOf(p.progress()));
} else {
info.put("progress", String.valueOf(100));
}
return info;
}
/**
* Remove paragraph by id.
*
* @param paragraphId ID of paragraph
* @return a paragraph that was deleted, or <code>null</code> otherwise
*/
public Paragraph removeParagraph(String user, String paragraphId) {
removeAllAngularObjectInParagraph(user, paragraphId);
ResourcePoolUtils.removeResourcesBelongsToParagraph(getId(), paragraphId);
synchronized (paragraphs) {
Iterator<Paragraph> i = paragraphs.iterator();
while (i.hasNext()) {
Paragraph p = i.next();
if (p.getId().equals(paragraphId)) {
index.deleteIndexDoc(this, p);
i.remove();
if (noteEventListener != null) {
noteEventListener.onParagraphRemove(p);
}
return p;
}
}
}
return null;
}
private InterpreterContext getInterpreterContext(InterpreterOutput output) {
AngularObjectRegistry registry = null;
ResourcePool resourcePool = null;
if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<>();
for (Paragraph p : note.getParagraphs()) {
runners.add(new ParagraphRunner(note, note.getId(), p.getId()));
}
final Paragraph self = this;
Credentials credentials = note.getCredentials();
if (authenticationInfo != null) {
UserCredentials userCredentials =
credentials.getUserCredentials(authenticationInfo.getUser());
authenticationInfo.setUserCredentials(userCredentials);
}
InterpreterContext interpreterContext =
new InterpreterContext(
note.getId(),
getId(),
getRequiredReplName(),
this.getTitle(),
this.getText(),
this.getAuthenticationInfo(),
this.getConfig(),
this.settings,
registry,
resourcePool,
runners,
output);
return interpreterContext;
}