/** set note authorization information */ @PUT @Path("{noteId}/permissions") @ZeppelinApi public Response putNotePermissions(@PathParam("noteId") String noteId, String req) throws IOException { HashMap<String, HashSet> permMap = gson.fromJson(req, new TypeToken<HashMap<String, HashSet>>() {}.getType()); Note note = notebook.getNote(noteId); String principal = SecurityUtils.getPrincipal(); HashSet<String> roles = SecurityUtils.getRoles(); LOG.info( "Set permissions {} {} {} {} {}", noteId, principal, permMap.get("owners"), permMap.get("readers"), permMap.get("writers")); HashSet<String> userAndRoles = new HashSet<String>(); userAndRoles.add(principal); userAndRoles.addAll(roles); if (!notebookAuthorization.isOwner(noteId, userAndRoles)) { return new JsonResponse<>( Status.FORBIDDEN, ownerPermissionError(userAndRoles, notebookAuthorization.getOwners(noteId))) .build(); } HashSet readers = permMap.get("readers"); HashSet owners = permMap.get("owners"); HashSet writers = permMap.get("writers"); // Set readers, if writers and owners is empty -> set to user requesting the change if (readers != null && !readers.isEmpty()) { if (writers.isEmpty()) { writers = Sets.newHashSet(SecurityUtils.getPrincipal()); } if (owners.isEmpty()) { owners = Sets.newHashSet(SecurityUtils.getPrincipal()); } } // Set writers, if owners is empty -> set to user requesting the change if (writers != null && !writers.isEmpty()) { if (owners.isEmpty()) { owners = Sets.newHashSet(SecurityUtils.getPrincipal()); } } notebookAuthorization.setReaders(noteId, readers); notebookAuthorization.setWriters(noteId, writers); notebookAuthorization.setOwners(noteId, owners); LOG.debug( "After set permissions {} {} {}", notebookAuthorization.getOwners(noteId), notebookAuthorization.getReaders(noteId), notebookAuthorization.getWriters(noteId)); AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); note.persist(subject); notebookServer.broadcastNote(note); return new JsonResponse<>(Status.OK).build(); }
/** * Insert paragraph REST API * * @param message - JSON containing paragraph's information * @return JSON with status.OK * @throws IOException */ @POST @Path("{notebookId}/paragraph") @ZeppelinApi public Response insertParagraph(@PathParam("notebookId") String notebookId, String message) throws IOException { LOG.info("insert paragraph {} {}", notebookId, message); Note note = notebook.getNote(notebookId); if (note == null) { return new JsonResponse(Status.NOT_FOUND, "note not found.").build(); } NewParagraphRequest request = gson.fromJson(message, NewParagraphRequest.class); Paragraph p; Double indexDouble = request.getIndex(); if (indexDouble == null) { p = note.addParagraph(); } else { p = note.insertParagraph(indexDouble.intValue()); } p.setTitle(request.getTitle()); p.setText(request.getText()); AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); note.persist(subject); notebookServer.broadcastNote(note); return new JsonResponse(Status.CREATED, "", p.getId()).build(); }
private Note getNote(String key) throws IOException { GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.setPrettyPrinting(); Gson gson = gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create(); S3Object s3object; try { s3object = s3client.getObject(new GetObjectRequest(bucketName, key)); } catch (AmazonClientException ace) { throw new IOException("Unable to retrieve object from S3: " + ace, ace); } Note note; try (InputStream ins = s3object.getObjectContent()) { String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING)); note = gson.fromJson(json, Note.class); } for (Paragraph p : note.getParagraphs()) { if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) { p.setStatus(Status.ABORT); } } return note; }
/** * Clone existing note. * * @param sourceNoteID - the note ID to clone * @param newNoteName - the name of the new note * @return noteId * @throws IOException, CloneNotSupportedException, IllegalArgumentException */ public Note cloneNote(String sourceNoteID, String newNoteName, AuthenticationInfo subject) throws IOException, CloneNotSupportedException, IllegalArgumentException { Note sourceNote = getNote(sourceNoteID); if (sourceNote == null) { throw new IllegalArgumentException(sourceNoteID + "not found"); } Note newNote = createNote(subject); if (newNoteName != null) { newNote.setName(newNoteName); } else { newNote.setName("Note " + newNote.getId()); } // Copy the interpreter bindings List<String> boundInterpreterSettingsIds = getBindedInterpreterSettingsIds(sourceNote.getId()); bindInterpretersToNote(newNote.getId(), boundInterpreterSettingsIds); List<Paragraph> paragraphs = sourceNote.getParagraphs(); for (Paragraph p : paragraphs) { newNote.addCloneParagraph(p); } notebookIndex.addIndexDoc(newNote); newNote.persist(subject); return newNote; }
/** * Run paragraph job REST API * * @param message - JSON with params if user wants to update dynamic form's value null, empty * string, empty json if user doesn't want to update * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @POST @Path("job/{notebookId}/{paragraphId}") @ZeppelinApi public Response runParagraph( @PathParam("notebookId") String notebookId, @PathParam("paragraphId") String paragraphId, String message) throws IOException, IllegalArgumentException { LOG.info("run paragraph job {} {} {}", notebookId, paragraphId, message); Note note = notebook.getNote(notebookId); if (note == null) { return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } Paragraph paragraph = note.getParagraph(paragraphId); if (paragraph == null) { return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build(); } // handle params if presented if (!StringUtils.isEmpty(message)) { RunParagraphWithParametersRequest request = gson.fromJson(message, RunParagraphWithParametersRequest.class); Map<String, Object> paramsForUpdating = request.getParams(); if (paramsForUpdating != null) { paragraph.settings.getParams().putAll(paramsForUpdating); AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); note.persist(subject); } } note.run(paragraph.getId()); return new JsonResponse<>(Status.OK).build(); }
/** * Register cron job REST API * * @param message - JSON with cron expressions. * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @POST @Path("cron/{notebookId}") @ZeppelinApi public Response registerCronJob(@PathParam("notebookId") String notebookId, String message) throws IOException, IllegalArgumentException { LOG.info("Register cron job note={} request cron msg={}", notebookId, message); CronRequest request = gson.fromJson(message, CronRequest.class); Note note = notebook.getNote(notebookId); if (note == null) { return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } if (!CronExpression.isValidExpression(request.getCronString())) { return new JsonResponse<>(Status.BAD_REQUEST, "wrong cron expressions.").build(); } Map<String, Object> config = note.getConfig(); config.put("cron", request.getCronString()); note.setConfig(config); notebook.refreshCron(note.id()); return new JsonResponse<>(Status.OK).build(); }
/** * Delete paragraph REST API * * @param * @return JSON with status.OK * @throws IOException */ @DELETE @Path("{notebookId}/paragraph/{paragraphId}") @ZeppelinApi public Response deleteParagraph( @PathParam("notebookId") String notebookId, @PathParam("paragraphId") String paragraphId) throws IOException { LOG.info("delete paragraph {} {}", notebookId, paragraphId); Note note = notebook.getNote(notebookId); if (note == null) { return new JsonResponse(Status.NOT_FOUND, "note not found.").build(); } Paragraph p = note.getParagraph(paragraphId); if (p == null) { return new JsonResponse(Status.NOT_FOUND, "paragraph not found.").build(); } AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); note.removeParagraph(paragraphId); note.persist(subject); notebookServer.broadcastNote(note); return new JsonResponse(Status.OK, "").build(); }
/** * Move paragraph REST API * * @param newIndex - new index to move * @return JSON with status.OK * @throws IOException */ @POST @Path("{notebookId}/paragraph/{paragraphId}/move/{newIndex}") @ZeppelinApi public Response moveParagraph( @PathParam("notebookId") String notebookId, @PathParam("paragraphId") String paragraphId, @PathParam("newIndex") String newIndex) throws IOException { LOG.info("move paragraph {} {} {}", notebookId, paragraphId, newIndex); Note note = notebook.getNote(notebookId); if (note == null) { return new JsonResponse(Status.NOT_FOUND, "note not found.").build(); } Paragraph p = note.getParagraph(paragraphId); if (p == null) { return new JsonResponse(Status.NOT_FOUND, "paragraph not found.").build(); } try { note.moveParagraph(paragraphId, Integer.parseInt(newIndex), true); AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); note.persist(subject); notebookServer.broadcastNote(note); return new JsonResponse(Status.OK, "").build(); } catch (IndexOutOfBoundsException e) { LOG.error("Exception in NotebookRestApi while moveParagraph ", e); return new JsonResponse(Status.BAD_REQUEST, "paragraph's new index is out of bound").build(); } }
@Override public void execute(JobExecutionContext context) throws JobExecutionException { String noteId = context.getJobDetail().getJobDataMap().getString("noteId"); Note note = notebook.getNote(noteId); note.runAll(); }
private InterpreterContext getInterpreterContext() { AngularObjectRegistry registry = null; if (!getNoteReplLoader().getInterpreterSettings().isEmpty()) { InterpreterSetting intpGroup = getNoteReplLoader().getInterpreterSettings().get(0); registry = intpGroup.getInterpreterGroup().getAngularObjectRegistry(); } List<InterpreterContextRunner> runners = new LinkedList<InterpreterContextRunner>(); for (Paragraph p : note.getParagraphs()) { runners.add(new ParagraphRunner(note, note.id(), p.getId())); } InterpreterContext interpreterContext = new InterpreterContext( note.id(), getId(), this.getTitle(), this.getText(), this.getConfig(), this.settings, registry, runners); return interpreterContext; }
List<String> getBindedInterpreterSettingsIds(String id) { Note note = getNote(id); if (note != null) { return getInterpreterFactory().getInterpreters(note.getId()); } else { return new LinkedList<>(); } }
public void bindInterpretersToNote(String id, List<String> interpreterSettingIds) throws IOException { Note note = getNote(id); if (note != null) { note.getNoteReplLoader().setInterpreters(interpreterSettingIds); replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds); } }
public List<InterpreterSetting> getBindedInterpreterSettings(String id) { Note note = getNote(id); if (note != null) { return note.getNoteReplLoader().getInterpreterSettings(); } else { return new LinkedList<InterpreterSetting>(); } }
/** * import new note REST API * * @param req - notebook Json * @return JSON with new note ID * @throws IOException */ @POST @Path("import") @ZeppelinApi public Response importNotebook(String req) throws IOException { AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); Note newNote = notebook.importNote(req, null, subject); return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build(); }
public List<InterpreterSetting> getBindedInterpreterSettings(String id) { Note note = getNote(id); if (note != null) { return replFactory.getInterpreterSettings(note.getId()); } else { return new LinkedList<>(); } }
private InterpreterContext getInterpreterContext() { AngularObjectRegistry registry = null; ResourcePool resourcePool = null; if (!getNoteReplLoader().getInterpreterSettings().isEmpty()) { InterpreterSetting intpGroup = getNoteReplLoader().getInterpreterSettings().get(0); registry = intpGroup.getInterpreterGroup().getAngularObjectRegistry(); resourcePool = intpGroup.getInterpreterGroup().getResourcePool(); } List<InterpreterContextRunner> runners = new LinkedList<InterpreterContextRunner>(); for (Paragraph p : note.getParagraphs()) { runners.add(new ParagraphRunner(note, note.id(), p.getId())); } final Paragraph self = this; InterpreterContext interpreterContext = new InterpreterContext( note.id(), getId(), this.getTitle(), this.getText(), this.getConfig(), this.settings, registry, resourcePool, runners, new InterpreterOutput( new InterpreterOutputListener() { @Override public void onAppend(InterpreterOutput out, byte[] line) { updateParagraphResult(out); ((ParagraphJobListener) getListener()) .onOutputAppend(self, out, new String(line)); } @Override public void onUpdate(InterpreterOutput out, byte[] output) { updateParagraphResult(out); ((ParagraphJobListener) getListener()) .onOutputUpdate(self, out, new String(output)); } private void updateParagraphResult(InterpreterOutput out) { // update paragraph result Throwable t = null; String message = null; try { message = new String(out.toByteArray()); } catch (IOException e) { logger().error(e.getMessage(), e); t = e; } setReturn(new InterpreterResult(Code.SUCCESS, out.getType(), message), t); } })); return interpreterContext; }
public void removeNote(String id, AuthenticationInfo subject) { Note note; synchronized (notes) { note = notes.remove(id); } replFactory.removeNoteInterpreterSettingBinding(id); notebookIndex.deleteIndexDocs(note); notebookAuthorization.removeNote(id); // remove from all interpreter instance's angular object registry for (InterpreterSetting settings : replFactory.get()) { AngularObjectRegistry registry = settings.getInterpreterGroup(id).getAngularObjectRegistry(); if (registry instanceof RemoteAngularObjectRegistry) { // remove paragraph scope object for (Paragraph p : note.getParagraphs()) { ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, p.getId()); // remove app scope object List<ApplicationState> appStates = p.getAllApplicationStates(); if (appStates != null) { for (ApplicationState app : appStates) { ((RemoteAngularObjectRegistry) registry) .removeAllAndNotifyRemoteProcess(id, app.getId()); } } } // remove notebook scope object ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, null); } else { // remove paragraph scope object for (Paragraph p : note.getParagraphs()) { registry.removeAll(id, p.getId()); // remove app scope object List<ApplicationState> appStates = p.getAllApplicationStates(); if (appStates != null) { for (ApplicationState app : appStates) { registry.removeAll(id, app.getId()); } } } // remove notebook scope object registry.removeAll(id, null); } } ResourcePoolUtils.removeResourcesBelongsToNote(id); fireNoteRemoveEvent(note); try { note.unpersist(subject); } catch (IOException e) { logger.error(e.toString(), e); } }
public List<Map<String, Object>> getJobListByParagraphId(String paragraphID) { String gotNoteId = null; List<Note> notes = getAllNotes(); for (Note note : notes) { Paragraph p = note.getParagraph(paragraphID); if (p != null) { gotNoteId = note.getId(); } } return getJobListBymNotebookId(gotNoteId); }
/** * Get notebook job status REST API * * @param * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @GET @Path("job/{notebookId}") @ZeppelinApi public Response getNoteJobStatus(@PathParam("notebookId") String notebookId) throws IOException, IllegalArgumentException { LOG.info("get notebook job status."); Note note = notebook.getNote(notebookId); if (note == null) { return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } return new JsonResponse<>(Status.OK, null, note.generateParagraphsInfo()).build(); }
/** * Run notebook jobs REST API * * @param * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @POST @Path("job/{notebookId}") @ZeppelinApi public Response runNoteJobs(@PathParam("notebookId") String notebookId) throws IOException, IllegalArgumentException { LOG.info("run notebook jobs {} ", notebookId); Note note = notebook.getNote(notebookId); if (note == null) { return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } note.runAll(); return new JsonResponse<>(Status.OK).build(); }
/** * import JSON as a new note. * * @param sourceJson - the note JSON to import * @param noteName - the name of the new note * @return notebook ID * @throws IOException */ public Note importNote(String sourceJson, String noteName, AuthenticationInfo subject) throws IOException { GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.setPrettyPrinting(); Gson gson = gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create(); JsonReader reader = new JsonReader(new StringReader(sourceJson)); reader.setLenient(true); Note newNote; try { Note oldNote = gson.fromJson(reader, Note.class); newNote = createNote(subject); if (noteName != null) newNote.setName(noteName); else newNote.setName(oldNote.getName()); List<Paragraph> paragraphs = oldNote.getParagraphs(); for (Paragraph p : paragraphs) { newNote.addCloneParagraph(p); } newNote.persist(subject); } catch (IOException e) { logger.error(e.toString(), e); throw e; } return newNote; }
/** * Clone note REST API * * @param * @return JSON with status.CREATED * @throws IOException, CloneNotSupportedException, IllegalArgumentException */ @POST @Path("{notebookId}") @ZeppelinApi public Response cloneNote(@PathParam("notebookId") String notebookId, String message) throws IOException, CloneNotSupportedException, IllegalArgumentException { LOG.info("clone notebook by JSON {}", message); NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class); String newNoteName = request.getName(); AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); Note newNote = notebook.cloneNote(notebookId, newNoteName, subject); notebookServer.broadcastNote(newNote); notebookServer.broadcastNoteList(subject); return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build(); }
/** * Create new note. * * @return * @throws IOException */ public Note createNote(List<String> interpreterIds) throws IOException { NoteInterpreterLoader intpLoader = new NoteInterpreterLoader(replFactory); Note note = new Note(notebookRepo, intpLoader, jobListenerFactory); intpLoader.setNoteId(note.id()); synchronized (notes) { notes.put(note.id(), note); } if (interpreterIds != null) { bindInterpretersToNote(note.id(), interpreterIds); } note.persist(); return note; }
/** * Get cron job REST API * * @param * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @GET @Path("cron/{notebookId}") @ZeppelinApi public Response getCronJob(@PathParam("notebookId") String notebookId) throws IOException, IllegalArgumentException { LOG.info("Get cron job note {}", notebookId); Note note = notebook.getNote(notebookId); if (note == null) { return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } return new JsonResponse<>(Status.OK, note.getConfig().get("cron")).build(); }
public void refreshCron(String id) { removeCron(id); synchronized (notes) { Note note = notes.get(id); if (note == null) { return; } Map<String, Object> config = note.getConfig(); if (config == null) { return; } String cronExpr = (String) note.getConfig().get("cron"); if (cronExpr == null || cronExpr.trim().length() == 0) { return; } JobDetail newJob = JobBuilder.newJob(CronJob.class) .withIdentity(id, "note") .usingJobData("noteId", id) .build(); Map<String, Object> info = note.getInfo(); info.put("cron", null); CronTrigger trigger = null; try { trigger = TriggerBuilder.newTrigger() .withIdentity("trigger_" + id, "note") .withSchedule(CronScheduleBuilder.cronSchedule(cronExpr)) .forJob(id, "note") .build(); } catch (Exception e) { logger.error("Error", e); info.put("cron", e.getMessage()); } try { if (trigger != null) { quartzSched.scheduleJob(newJob, trigger); } } catch (SchedulerException e) { logger.error("Error", e); info.put("cron", "Scheduler Exception"); } } }
@Test public void testSyncUpdateMain() throws IOException { /* create note */ Note note = notebookSync.createNote(); Paragraph p1 = note.addParagraph(); Map config = p1.getConfig(); config.put("enabled", true); p1.setConfig(config); p1.setText("hello world"); /* new paragraph exists in note instance */ assertEquals(1, note.getParagraphs().size()); /* new paragraph not yet saved into storages */ assertEquals( 0, notebookRepoSync.get(0, notebookRepoSync.list(0).get(0).getId()).getParagraphs().size()); assertEquals( 0, notebookRepoSync.get(1, notebookRepoSync.list(1).get(0).getId()).getParagraphs().size()); /* save to storage under index 0 (first storage) */ notebookRepoSync.save(0, note); /* check paragraph saved to first storage */ assertEquals( 1, notebookRepoSync.get(0, notebookRepoSync.list(0).get(0).getId()).getParagraphs().size()); /* check paragraph isn't saved to second storage */ assertEquals( 0, notebookRepoSync.get(1, notebookRepoSync.list(1).get(0).getId()).getParagraphs().size()); /* apply sync */ notebookRepoSync.sync(); /* check whether added to second storage */ assertEquals( 1, notebookRepoSync.get(1, notebookRepoSync.list(1).get(0).getId()).getParagraphs().size()); /* check whether same paragraph id */ assertEquals( p1.getId(), notebookRepoSync .get(0, notebookRepoSync.list(0).get(0).getId()) .getLastParagraph() .getId()); assertEquals( p1.getId(), notebookRepoSync .get(1, notebookRepoSync.list(1).get(0).getId()) .getLastParagraph() .getId()); }
/** * Create new note. * * @throws IOException */ public Note createNote(List<String> interpreterIds, AuthenticationInfo subject) throws IOException { Note note = new Note(notebookRepo, replFactory, jobListenerFactory, notebookIndex, credentials, this); synchronized (notes) { notes.put(note.getId(), note); } if (interpreterIds != null) { bindInterpretersToNote(note.getId(), interpreterIds); } notebookIndex.addIndexDoc(note); note.persist(subject); fireNoteCreateEvent(note); return note; }
public void bindInterpretersToNote(String id, List<String> interpreterSettingIds) throws IOException { Note note = getNote(id); if (note != null) { List<InterpreterSetting> currentBindings = replFactory.getInterpreterSettings(id); for (InterpreterSetting setting : currentBindings) { if (!interpreterSettingIds.contains(setting.getId())) { fireUnbindInterpreter(note, setting); } } replFactory.setInterpreters(note.getId(), interpreterSettingIds); // comment out while note.getNoteReplLoader().setInterpreters(...) do the same // replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds); } }
private boolean isValidInterpreter(String replName) { try { return factory.getInterpreter(user, note.getId(), replName) != null; } catch (InterpreterException e) { // ignore this exception, it would be recaught when running paragraph. return false; } }
/** * Return new note for specific user. this inserts and replaces user paragraph which doesn't * exists in original paragraph * * @param user specific user * @return new Note for the user */ public Note getUserNote(String user) { Note newNote = new Note(); newNote.id = getId(); newNote.config = getConfig(); newNote.angularObjects = getAngularObjects(); Paragraph newParagraph; for (Paragraph p : paragraphs) { newParagraph = p.getUserParagraph(user); if (null == newParagraph) { newParagraph = p.cloneParagraphForUser(user); } newNote.paragraphs.add(newParagraph); } return newNote; }