/** * Clone existing note. * * @param sourceNoteID - the note ID to clone * @param newNoteName - the name of the new note * @return noteId * @throws IOException, CloneNotSupportedException, IllegalArgumentException */ public Note cloneNote(String sourceNoteID, String newNoteName, AuthenticationInfo subject) throws IOException, CloneNotSupportedException, IllegalArgumentException { Note sourceNote = getNote(sourceNoteID); if (sourceNote == null) { throw new IllegalArgumentException(sourceNoteID + "not found"); } Note newNote = createNote(subject); if (newNoteName != null) { newNote.setName(newNoteName); } else { newNote.setName("Note " + newNote.getId()); } // Copy the interpreter bindings List<String> boundInterpreterSettingsIds = getBindedInterpreterSettingsIds(sourceNote.getId()); bindInterpretersToNote(newNote.getId(), boundInterpreterSettingsIds); List<Paragraph> paragraphs = sourceNote.getParagraphs(); for (Paragraph p : paragraphs) { newNote.addCloneParagraph(p); } notebookIndex.addIndexDoc(newNote); newNote.persist(subject); return newNote; }
/** * Create new note REST API * * @param message - JSON with new note name * @return JSON with new note ID * @throws IOException */ @POST @Path("/") @ZeppelinApi public Response createNote(String message) throws IOException { LOG.info("Create new notebook by JSON {}", message); NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class); AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); Note note = notebook.createNote(subject); List<NewParagraphRequest> initialParagraphs = request.getParagraphs(); if (initialParagraphs != null) { for (NewParagraphRequest paragraphRequest : initialParagraphs) { Paragraph p = note.addParagraph(); p.setTitle(paragraphRequest.getTitle()); p.setText(paragraphRequest.getText()); } } note.addParagraph(); // add one paragraph to the last String noteName = request.getName(); if (noteName.isEmpty()) { noteName = "Note " + note.getId(); } note.setName(noteName); note.persist(subject); notebookServer.broadcastNote(note); notebookServer.broadcastNoteList(subject); return new JsonResponse<>(Status.CREATED, "", note.getId()).build(); }
public List<Map<String, Object>> getJobListBymNotebookId(String notebookID) { final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; long lastRunningUnixTime = 0; boolean isNotebookRunning = false; Note jobNote = getNote(notebookID); List<Map<String, Object>> notesInfo = new LinkedList<>(); if (jobNote == null) { return notesInfo; } Map<String, Object> info = new HashMap<>(); info.put("notebookId", jobNote.getId()); String notebookName = jobNote.getName(); if (notebookName != null && !notebookName.equals("")) { info.put("notebookName", jobNote.getName()); } else { info.put("notebookName", "Note " + jobNote.getId()); } // set notebook type ( cron or normal ) if (jobNote.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) && !jobNote.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { info.put("notebookType", "cron"); } else { info.put("notebookType", "normal"); } // set paragraphs List<Map<String, Object>> paragraphsInfo = new LinkedList<>(); for (Paragraph paragraph : jobNote.getParagraphs()) { // check paragraph's status. if (paragraph.getStatus().isRunning()) { isNotebookRunning = true; } // get data for the job manager. Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph); lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph); paragraphsInfo.add(paragraphItem); } // set interpreter bind type String interpreterGroupName = null; if (replFactory.getInterpreterSettings(jobNote.getId()) != null && replFactory.getInterpreterSettings(jobNote.getId()).size() >= 1) { interpreterGroupName = replFactory.getInterpreterSettings(jobNote.getId()).get(0).getName(); } // notebook json object root information. info.put("interpreter", interpreterGroupName); info.put("isRunningJob", isNotebookRunning); info.put("unixTimeLastRun", lastRunningUnixTime); info.put("paragraphs", paragraphsInfo); notesInfo.add(info); return notesInfo; };
@Test public void testRestartInterpreterPerNote() throws IOException, InterruptedException { // create new note Note note = ZeppelinServer.notebook.createNote(anonymous); note.addParagraph(); Paragraph p = note.getLastParagraph(); Map config = p.getConfig(); config.put("enabled", true); // run markdown paragraph. p.setConfig(config); p.setText("%md markdown"); p.setAuthenticationInfo(anonymous); note.run(p.getId()); while (p.getStatus() != Status.FINISHED) { Thread.sleep(100); } assertEquals("<p>markdown</p>\n", p.getResult().message()); // get md interpreter InterpreterSetting mdIntpSetting = null; for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) { if (setting.getName().equals("md")) { mdIntpSetting = setting; break; } } String jsonRequest = "{\"noteId\":\"" + note.getId() + "\"}"; // Restart isolated mode of Interpreter for note. mdIntpSetting.getOption().setPerNote(InterpreterOption.ISOLATED); PutMethod put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest); assertThat("isolated interpreter restart:", put, isAllowed()); put.releaseConnection(); // Restart scoped mode of Interpreter for note. mdIntpSetting.getOption().setPerNote(InterpreterOption.SCOPED); put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest); assertThat("scoped interpreter restart:", put, isAllowed()); put.releaseConnection(); // Restart shared mode of Interpreter for note. mdIntpSetting.getOption().setPerNote(InterpreterOption.SHARED); put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest); assertThat("shared interpreter restart:", put, isAllowed()); put.releaseConnection(); ZeppelinServer.notebook.removeNote(note.getId(), anonymous); }
@Override public void execute(JobExecutionContext context) throws JobExecutionException { String noteId = context.getJobDetail().getJobDataMap().getString("noteId"); Note note = notebook.getNote(noteId); note.runAll(); while (!note.isTerminated()) { try { Thread.sleep(1000); } catch (InterruptedException e) { logger.error(e.toString(), e); } } boolean releaseResource = false; try { Map<String, Object> config = note.getConfig(); if (config != null && config.containsKey("releaseresource")) { releaseResource = (boolean) note.getConfig().get("releaseresource"); } } catch (ClassCastException e) { logger.error(e.getMessage(), e); } if (releaseResource) { for (InterpreterSetting setting : notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) { notebook.getInterpreterFactory().restart(setting.getId()); } } }
/** * Create new note. * * @throws IOException */ public Note createNote(List<String> interpreterIds, AuthenticationInfo subject) throws IOException { Note note = new Note(notebookRepo, replFactory, jobListenerFactory, notebookIndex, credentials, this); synchronized (notes) { notes.put(note.getId(), note); } if (interpreterIds != null) { bindInterpretersToNote(note.getId(), interpreterIds); } notebookIndex.addIndexDoc(note); note.persist(subject); fireNoteCreateEvent(note); return note; }
public List<InterpreterSetting> getBindedInterpreterSettings(String id) { Note note = getNote(id); if (note != null) { return replFactory.getInterpreterSettings(note.getId()); } else { return new LinkedList<>(); } }
private boolean isValidInterpreter(String replName) { try { return factory.getInterpreter(user, note.getId(), replName) != null; } catch (InterpreterException e) { // ignore this exception, it would be recaught when running paragraph. return false; } }
/** * import new note REST API * * @param req - notebook Json * @return JSON with new note ID * @throws IOException */ @POST @Path("import") @ZeppelinApi public Response importNotebook(String req) throws IOException { AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); Note newNote = notebook.importNote(req, null, subject); return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build(); }
List<String> getBindedInterpreterSettingsIds(String id) { Note note = getNote(id); if (note != null) { return getInterpreterFactory().getInterpreters(note.getId()); } else { return new LinkedList<>(); } }
@Test public void testInterpreterRestart() throws IOException, InterruptedException { // create new note Note note = ZeppelinServer.notebook.createNote(anonymous); note.addParagraph(); Paragraph p = note.getLastParagraph(); Map config = p.getConfig(); config.put("enabled", true); // run markdown paragraph p.setConfig(config); p.setText("%md markdown"); p.setAuthenticationInfo(anonymous); note.run(p.getId()); while (p.getStatus() != Status.FINISHED) { Thread.sleep(100); } assertEquals("<p>markdown</p>\n", p.getResult().message()); // restart interpreter for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) { if (setting.getName().equals("md")) { // Call Restart Interpreter REST API PutMethod put = httpPut("/interpreter/setting/restart/" + setting.getId(), ""); assertThat("test interpreter restart:", put, isAllowed()); put.releaseConnection(); break; } } // run markdown paragraph, again p = note.addParagraph(); p.setConfig(config); p.setText("%md markdown restarted"); p.setAuthenticationInfo(anonymous); note.run(p.getId()); while (p.getStatus() != Status.FINISHED) { Thread.sleep(100); } assertEquals("<p>markdown restarted</p>\n", p.getResult().message()); // cleanup ZeppelinServer.notebook.removeNote(note.getId(), anonymous); }
@Test public void testInterpreterAutoBinding() throws IOException { // create note Note note = ZeppelinServer.notebook.createNote(anonymous); // check interpreter is binded GetMethod get = httpGet("/notebook/interpreter/bind/" + note.getId()); assertThat(get, isAllowed()); get.addRequestHeader("Origin", "http://localhost"); Map<String, Object> resp = gson.fromJson( get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {}.getType()); List<Map<String, String>> body = (List<Map<String, String>>) resp.get("body"); assertTrue(0 < body.size()); get.releaseConnection(); // cleanup ZeppelinServer.notebook.removeNote(note.getId(), anonymous); }
private InterpreterSetting getInterpreterSettingById(String id) { InterpreterSetting setting = null; for (InterpreterSetting i : factory.getInterpreterSettings(note.getId())) { if (id.startsWith(i.getId())) { setting = i; break; } } return setting; }
public List<Map<String, Object>> getJobListByParagraphId(String paragraphID) { String gotNoteId = null; List<Note> notes = getAllNotes(); for (Note note : notes) { Paragraph p = note.getParagraph(paragraphID); if (p != null) { gotNoteId = note.getId(); } } return getJobListBymNotebookId(gotNoteId); }
/** * Clone note REST API * * @param * @return JSON with status.CREATED * @throws IOException, CloneNotSupportedException, IllegalArgumentException */ @POST @Path("{notebookId}") @ZeppelinApi public Response cloneNote(@PathParam("notebookId") String notebookId, String message) throws IOException, CloneNotSupportedException, IllegalArgumentException { LOG.info("clone notebook by JSON {}", message); NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class); String newNoteName = request.getName(); AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); Note newNote = notebook.cloneNote(notebookId, newNoteName, subject); notebookServer.broadcastNote(newNote); notebookServer.broadcastNoteList(subject); return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build(); }
public void initializeJobListenerForParagraph(Paragraph paragraph) { final Note paragraphNote = paragraph.getNote(); if (paragraphNote.getId().equals(this.getId())) { throw new IllegalArgumentException( format( "The paragraph %s from note %s " + "does not belong to note %s", paragraph.getId(), paragraphNote.getId(), this.getId())); } boolean foundParagraph = false; for (Paragraph ownParagraph : paragraphs) { if (paragraph.getId().equals(ownParagraph.getId())) { paragraph.setListener(this.jobListenerFactory.getParagraphJobListener(this)); foundParagraph = true; } } if (!foundParagraph) { throw new IllegalArgumentException( format( "Cannot find paragraph %s " + "from note %s", paragraph.getId(), paragraphNote.getId())); } }
public List<InterpreterCompletion> getInterpreterCompletion() { List<InterpreterCompletion> completion = new LinkedList(); for (InterpreterSetting intp : factory.getInterpreterSettings(note.getId())) { List<InterpreterInfo> intInfo = intp.getInterpreterInfos(); if (intInfo.size() > 1) { for (InterpreterInfo info : intInfo) { String name = intp.getName() + "." + info.getName(); completion.add(new InterpreterCompletion(name, name)); } } else { completion.add(new InterpreterCompletion(intp.getName(), intp.getName())); } } return completion; }
public void bindInterpretersToNote(String id, List<String> interpreterSettingIds) throws IOException { Note note = getNote(id); if (note != null) { List<InterpreterSetting> currentBindings = replFactory.getInterpreterSettings(id); for (InterpreterSetting setting : currentBindings) { if (!interpreterSettingIds.contains(setting.getId())) { fireUnbindInterpreter(note, setting); } } replFactory.setInterpreters(note.getId(), interpreterSettingIds); // comment out while note.getNoteReplLoader().setInterpreters(...) do the same // replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds); } }
@Override public void save(Note note, AuthenticationInfo subject) throws IOException { GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.setPrettyPrinting(); Gson gson = gsonBuilder.create(); String json = gson.toJson(note); String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json"; File file = File.createTempFile("note", "json"); try { Writer writer = new OutputStreamWriter(new FileOutputStream(file)); writer.write(json); writer.close(); s3client.putObject(new PutObjectRequest(bucketName, key, file)); } catch (AmazonClientException ace) { throw new IOException("Unable to store note in S3: " + ace, ace); } finally { FileUtils.deleteQuietly(file); } }
private InterpreterContext getInterpreterContext(InterpreterOutput output) { AngularObjectRegistry registry = null; ResourcePool resourcePool = null; if (!factory.getInterpreterSettings(note.getId()).isEmpty()) { InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0); registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry(); resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool(); } List<InterpreterContextRunner> runners = new LinkedList<>(); for (Paragraph p : note.getParagraphs()) { runners.add(new ParagraphRunner(note, note.getId(), p.getId())); } final Paragraph self = this; Credentials credentials = note.getCredentials(); if (authenticationInfo != null) { UserCredentials userCredentials = credentials.getUserCredentials(authenticationInfo.getUser()); authenticationInfo.setUserCredentials(userCredentials); } InterpreterContext interpreterContext = new InterpreterContext( note.getId(), getId(), getRequiredReplName(), this.getTitle(), this.getText(), this.getAuthenticationInfo(), this.getConfig(), this.settings, registry, resourcePool, runners, output); return interpreterContext; }
public Interpreter getRepl(String name) { return factory.getInterpreter(user, note.getId(), name); }
private boolean noteHasInterpreters() { return !factory.getInterpreterSettings(note.getId()).isEmpty(); }
public InterpreterContextRunner getInterpreterContextRunner() { return new ParagraphRunner(note, note.getId(), getId()); }
public List<Map<String, Object>> getJobListByUnixTime( boolean needsReload, long lastUpdateServerUnixTime, AuthenticationInfo subject) { final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron"; if (needsReload) { try { reloadAllNotes(subject); } catch (IOException e) { logger.error("Fail to reload notes from repository"); } } List<Note> notes = getAllNotes(); List<Map<String, Object>> notesInfo = new LinkedList<>(); for (Note note : notes) { boolean isNotebookRunning = false; boolean isUpdateNotebook = false; long lastRunningUnixTime = 0; Map<String, Object> info = new HashMap<>(); // set notebook ID info.put("notebookId", note.getId()); // set notebook Name String notebookName = note.getName(); if (notebookName != null && !notebookName.equals("")) { info.put("notebookName", note.getName()); } else { info.put("notebookName", "Note " + note.getId()); } // set notebook type ( cron or normal ) if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { info.put("notebookType", "cron"); } else { info.put("notebookType", "normal"); } // set paragraphs List<Map<String, Object>> paragraphsInfo = new LinkedList<>(); for (Paragraph paragraph : note.getParagraphs()) { // check paragraph's status. if (paragraph.getStatus().isRunning()) { isNotebookRunning = true; isUpdateNotebook = true; } // get data for the job manager. Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph); lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph); // is update notebook for last server update time. if (lastRunningUnixTime > lastUpdateServerUnixTime) { isUpdateNotebook = true; } paragraphsInfo.add(paragraphItem); } // set interpreter bind type String interpreterGroupName = null; if (replFactory.getInterpreterSettings(note.getId()) != null && replFactory.getInterpreterSettings(note.getId()).size() >= 1) { interpreterGroupName = replFactory.getInterpreterSettings(note.getId()).get(0).getName(); } // not update and not running -> pass if (!isUpdateNotebook && !isNotebookRunning) { continue; } // notebook json object root information. info.put("interpreter", interpreterGroupName); info.put("isRunningJob", isNotebookRunning); info.put("unixTimeLastRun", lastRunningUnixTime); info.put("paragraphs", paragraphsInfo); notesInfo.add(info); } return notesInfo; }
@SuppressWarnings("rawtypes") private Note loadNoteFromRepo(String id, AuthenticationInfo subject) { Note note = null; try { note = notebookRepo.get(id, subject); } catch (IOException e) { logger.error("Failed to load " + id, e); } if (note == null) { return null; } // Manually inject ALL dependencies, as DI constructor was NOT used note.setIndex(this.notebookIndex); note.setCredentials(this.credentials); note.setInterpreterFactory(replFactory); note.setJobListenerFactory(jobListenerFactory); note.setNotebookRepo(notebookRepo); Map<String, SnapshotAngularObject> angularObjectSnapshot = new HashMap<>(); // restore angular object -------------- Date lastUpdatedDate = new Date(0); for (Paragraph p : note.getParagraphs()) { p.setNote(note); if (p.getDateFinished() != null && lastUpdatedDate.before(p.getDateFinished())) { lastUpdatedDate = p.getDateFinished(); } } Map<String, List<AngularObject>> savedObjects = note.getAngularObjects(); if (savedObjects != null) { for (String intpGroupName : savedObjects.keySet()) { List<AngularObject> objectList = savedObjects.get(intpGroupName); for (AngularObject object : objectList) { SnapshotAngularObject snapshot = angularObjectSnapshot.get(object.getName()); if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) { angularObjectSnapshot.put( object.getName(), new SnapshotAngularObject(intpGroupName, object, lastUpdatedDate)); } } } } note.setNoteEventListener(this); synchronized (notes) { notes.put(note.getId(), note); refreshCron(note.getId()); } for (String name : angularObjectSnapshot.keySet()) { SnapshotAngularObject snapshot = angularObjectSnapshot.get(name); List<InterpreterSetting> settings = replFactory.get(); for (InterpreterSetting setting : settings) { InterpreterGroup intpGroup = setting.getInterpreterGroup(note.getId()); if (intpGroup.getId().equals(snapshot.getIntpGroupId())) { AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry(); String noteId = snapshot.getAngularObject().getNoteId(); String paragraphId = snapshot.getAngularObject().getParagraphId(); // at this point, remote interpreter process is not created. // so does not make sense add it to the remote. // // therefore instead of addAndNotifyRemoteProcess(), need to use add() // that results add angularObject only in ZeppelinServer side not remoteProcessSide registry.add(name, snapshot.getAngularObject().get(), noteId, paragraphId); } } } return note; }