Exemplo n.º 1
0
  /**
   * Clone existing note.
   *
   * @param sourceNoteID - the note ID to clone
   * @param newNoteName - the name of the new note
   * @return noteId
   * @throws IOException, CloneNotSupportedException, IllegalArgumentException
   */
  public Note cloneNote(String sourceNoteID, String newNoteName, AuthenticationInfo subject)
      throws IOException, CloneNotSupportedException, IllegalArgumentException {

    Note sourceNote = getNote(sourceNoteID);
    if (sourceNote == null) {
      throw new IllegalArgumentException(sourceNoteID + "not found");
    }
    Note newNote = createNote(subject);
    if (newNoteName != null) {
      newNote.setName(newNoteName);
    } else {
      newNote.setName("Note " + newNote.getId());
    }
    // Copy the interpreter bindings
    List<String> boundInterpreterSettingsIds = getBindedInterpreterSettingsIds(sourceNote.getId());
    bindInterpretersToNote(newNote.getId(), boundInterpreterSettingsIds);

    List<Paragraph> paragraphs = sourceNote.getParagraphs();
    for (Paragraph p : paragraphs) {
      newNote.addCloneParagraph(p);
    }

    notebookIndex.addIndexDoc(newNote);
    newNote.persist(subject);
    return newNote;
  }
  /**
   * Create new note REST API
   *
   * @param message - JSON with new note name
   * @return JSON with new note ID
   * @throws IOException
   */
  @POST
  @Path("/")
  @ZeppelinApi
  public Response createNote(String message) throws IOException {
    LOG.info("Create new notebook by JSON {}", message);
    NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class);
    AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
    Note note = notebook.createNote(subject);
    List<NewParagraphRequest> initialParagraphs = request.getParagraphs();
    if (initialParagraphs != null) {
      for (NewParagraphRequest paragraphRequest : initialParagraphs) {
        Paragraph p = note.addParagraph();
        p.setTitle(paragraphRequest.getTitle());
        p.setText(paragraphRequest.getText());
      }
    }
    note.addParagraph(); // add one paragraph to the last
    String noteName = request.getName();
    if (noteName.isEmpty()) {
      noteName = "Note " + note.getId();
    }

    note.setName(noteName);
    note.persist(subject);
    notebookServer.broadcastNote(note);
    notebookServer.broadcastNoteList(subject);
    return new JsonResponse<>(Status.CREATED, "", note.getId()).build();
  }
Exemplo n.º 3
0
  public List<Map<String, Object>> getJobListBymNotebookId(String notebookID) {
    final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";
    long lastRunningUnixTime = 0;
    boolean isNotebookRunning = false;
    Note jobNote = getNote(notebookID);
    List<Map<String, Object>> notesInfo = new LinkedList<>();
    if (jobNote == null) {
      return notesInfo;
    }

    Map<String, Object> info = new HashMap<>();

    info.put("notebookId", jobNote.getId());
    String notebookName = jobNote.getName();
    if (notebookName != null && !notebookName.equals("")) {
      info.put("notebookName", jobNote.getName());
    } else {
      info.put("notebookName", "Note " + jobNote.getId());
    }
    // set notebook type ( cron or normal )
    if (jobNote.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD)
        && !jobNote.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) {
      info.put("notebookType", "cron");
    } else {
      info.put("notebookType", "normal");
    }

    // set paragraphs
    List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
    for (Paragraph paragraph : jobNote.getParagraphs()) {
      // check paragraph's status.
      if (paragraph.getStatus().isRunning()) {
        isNotebookRunning = true;
      }

      // get data for the job manager.
      Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph);
      lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph);

      paragraphsInfo.add(paragraphItem);
    }

    // set interpreter bind type
    String interpreterGroupName = null;
    if (replFactory.getInterpreterSettings(jobNote.getId()) != null
        && replFactory.getInterpreterSettings(jobNote.getId()).size() >= 1) {
      interpreterGroupName = replFactory.getInterpreterSettings(jobNote.getId()).get(0).getName();
    }

    // notebook json object root information.
    info.put("interpreter", interpreterGroupName);
    info.put("isRunningJob", isNotebookRunning);
    info.put("unixTimeLastRun", lastRunningUnixTime);
    info.put("paragraphs", paragraphsInfo);
    notesInfo.add(info);

    return notesInfo;
  };
  @Test
  public void testRestartInterpreterPerNote() throws IOException, InterruptedException {
    // create new note
    Note note = ZeppelinServer.notebook.createNote(anonymous);
    note.addParagraph();
    Paragraph p = note.getLastParagraph();
    Map config = p.getConfig();
    config.put("enabled", true);

    // run markdown paragraph.
    p.setConfig(config);
    p.setText("%md markdown");
    p.setAuthenticationInfo(anonymous);
    note.run(p.getId());
    while (p.getStatus() != Status.FINISHED) {
      Thread.sleep(100);
    }
    assertEquals("<p>markdown</p>\n", p.getResult().message());

    // get md interpreter
    InterpreterSetting mdIntpSetting = null;
    for (InterpreterSetting setting :
        ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
      if (setting.getName().equals("md")) {
        mdIntpSetting = setting;
        break;
      }
    }

    String jsonRequest = "{\"noteId\":\"" + note.getId() + "\"}";

    // Restart isolated mode of Interpreter for note.
    mdIntpSetting.getOption().setPerNote(InterpreterOption.ISOLATED);
    PutMethod put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
    assertThat("isolated interpreter restart:", put, isAllowed());
    put.releaseConnection();

    // Restart scoped mode of Interpreter for note.
    mdIntpSetting.getOption().setPerNote(InterpreterOption.SCOPED);
    put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
    assertThat("scoped interpreter restart:", put, isAllowed());
    put.releaseConnection();

    // Restart shared mode of Interpreter for note.
    mdIntpSetting.getOption().setPerNote(InterpreterOption.SHARED);
    put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
    assertThat("shared interpreter restart:", put, isAllowed());
    put.releaseConnection();

    ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
  }
Exemplo n.º 5
0
    @Override
    public void execute(JobExecutionContext context) throws JobExecutionException {

      String noteId = context.getJobDetail().getJobDataMap().getString("noteId");
      Note note = notebook.getNote(noteId);
      note.runAll();

      while (!note.isTerminated()) {
        try {
          Thread.sleep(1000);
        } catch (InterruptedException e) {
          logger.error(e.toString(), e);
        }
      }

      boolean releaseResource = false;
      try {
        Map<String, Object> config = note.getConfig();
        if (config != null && config.containsKey("releaseresource")) {
          releaseResource = (boolean) note.getConfig().get("releaseresource");
        }
      } catch (ClassCastException e) {
        logger.error(e.getMessage(), e);
      }
      if (releaseResource) {
        for (InterpreterSetting setting :
            notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
          notebook.getInterpreterFactory().restart(setting.getId());
        }
      }
    }
Exemplo n.º 6
0
  /**
   * Create new note.
   *
   * @throws IOException
   */
  public Note createNote(List<String> interpreterIds, AuthenticationInfo subject)
      throws IOException {
    Note note =
        new Note(notebookRepo, replFactory, jobListenerFactory, notebookIndex, credentials, this);
    synchronized (notes) {
      notes.put(note.getId(), note);
    }
    if (interpreterIds != null) {
      bindInterpretersToNote(note.getId(), interpreterIds);
    }

    notebookIndex.addIndexDoc(note);
    note.persist(subject);
    fireNoteCreateEvent(note);
    return note;
  }
Exemplo n.º 7
0
 public List<InterpreterSetting> getBindedInterpreterSettings(String id) {
   Note note = getNote(id);
   if (note != null) {
     return replFactory.getInterpreterSettings(note.getId());
   } else {
     return new LinkedList<>();
   }
 }
Exemplo n.º 8
0
 private boolean isValidInterpreter(String replName) {
   try {
     return factory.getInterpreter(user, note.getId(), replName) != null;
   } catch (InterpreterException e) {
     // ignore this exception, it would be recaught when running paragraph.
     return false;
   }
 }
 /**
  * import new note REST API
  *
  * @param req - notebook Json
  * @return JSON with new note ID
  * @throws IOException
  */
 @POST
 @Path("import")
 @ZeppelinApi
 public Response importNotebook(String req) throws IOException {
   AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
   Note newNote = notebook.importNote(req, null, subject);
   return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build();
 }
Exemplo n.º 10
0
 List<String> getBindedInterpreterSettingsIds(String id) {
   Note note = getNote(id);
   if (note != null) {
     return getInterpreterFactory().getInterpreters(note.getId());
   } else {
     return new LinkedList<>();
   }
 }
Exemplo n.º 11
0
  @Test
  public void testInterpreterRestart() throws IOException, InterruptedException {
    // create new note
    Note note = ZeppelinServer.notebook.createNote(anonymous);
    note.addParagraph();
    Paragraph p = note.getLastParagraph();
    Map config = p.getConfig();
    config.put("enabled", true);

    // run markdown paragraph
    p.setConfig(config);
    p.setText("%md markdown");
    p.setAuthenticationInfo(anonymous);
    note.run(p.getId());
    while (p.getStatus() != Status.FINISHED) {
      Thread.sleep(100);
    }
    assertEquals("<p>markdown</p>\n", p.getResult().message());

    // restart interpreter
    for (InterpreterSetting setting :
        ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
      if (setting.getName().equals("md")) {
        // Call Restart Interpreter REST API
        PutMethod put = httpPut("/interpreter/setting/restart/" + setting.getId(), "");
        assertThat("test interpreter restart:", put, isAllowed());
        put.releaseConnection();
        break;
      }
    }

    // run markdown paragraph, again
    p = note.addParagraph();
    p.setConfig(config);
    p.setText("%md markdown restarted");
    p.setAuthenticationInfo(anonymous);
    note.run(p.getId());
    while (p.getStatus() != Status.FINISHED) {
      Thread.sleep(100);
    }
    assertEquals("<p>markdown restarted</p>\n", p.getResult().message());
    // cleanup
    ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
  }
Exemplo n.º 12
0
  @Test
  public void testInterpreterAutoBinding() throws IOException {
    // create note
    Note note = ZeppelinServer.notebook.createNote(anonymous);

    // check interpreter is binded
    GetMethod get = httpGet("/notebook/interpreter/bind/" + note.getId());
    assertThat(get, isAllowed());
    get.addRequestHeader("Origin", "http://localhost");
    Map<String, Object> resp =
        gson.fromJson(
            get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {}.getType());
    List<Map<String, String>> body = (List<Map<String, String>>) resp.get("body");
    assertTrue(0 < body.size());

    get.releaseConnection();
    // cleanup
    ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
  }
Exemplo n.º 13
0
 private InterpreterSetting getInterpreterSettingById(String id) {
   InterpreterSetting setting = null;
   for (InterpreterSetting i : factory.getInterpreterSettings(note.getId())) {
     if (id.startsWith(i.getId())) {
       setting = i;
       break;
     }
   }
   return setting;
 }
Exemplo n.º 14
0
 public List<Map<String, Object>> getJobListByParagraphId(String paragraphID) {
   String gotNoteId = null;
   List<Note> notes = getAllNotes();
   for (Note note : notes) {
     Paragraph p = note.getParagraph(paragraphID);
     if (p != null) {
       gotNoteId = note.getId();
     }
   }
   return getJobListBymNotebookId(gotNoteId);
 }
Exemplo n.º 15
0
 /**
  * Clone note REST API
  *
  * @param
  * @return JSON with status.CREATED
  * @throws IOException, CloneNotSupportedException, IllegalArgumentException
  */
 @POST
 @Path("{notebookId}")
 @ZeppelinApi
 public Response cloneNote(@PathParam("notebookId") String notebookId, String message)
     throws IOException, CloneNotSupportedException, IllegalArgumentException {
   LOG.info("clone notebook by JSON {}", message);
   NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class);
   String newNoteName = request.getName();
   AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
   Note newNote = notebook.cloneNote(notebookId, newNoteName, subject);
   notebookServer.broadcastNote(newNote);
   notebookServer.broadcastNoteList(subject);
   return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build();
 }
Exemplo n.º 16
0
  public void initializeJobListenerForParagraph(Paragraph paragraph) {
    final Note paragraphNote = paragraph.getNote();
    if (paragraphNote.getId().equals(this.getId())) {
      throw new IllegalArgumentException(
          format(
              "The paragraph %s from note %s " + "does not belong to note %s",
              paragraph.getId(), paragraphNote.getId(), this.getId()));
    }

    boolean foundParagraph = false;
    for (Paragraph ownParagraph : paragraphs) {
      if (paragraph.getId().equals(ownParagraph.getId())) {
        paragraph.setListener(this.jobListenerFactory.getParagraphJobListener(this));
        foundParagraph = true;
      }
    }

    if (!foundParagraph) {
      throw new IllegalArgumentException(
          format(
              "Cannot find paragraph %s " + "from note %s",
              paragraph.getId(), paragraphNote.getId()));
    }
  }
Exemplo n.º 17
0
 public List<InterpreterCompletion> getInterpreterCompletion() {
   List<InterpreterCompletion> completion = new LinkedList();
   for (InterpreterSetting intp : factory.getInterpreterSettings(note.getId())) {
     List<InterpreterInfo> intInfo = intp.getInterpreterInfos();
     if (intInfo.size() > 1) {
       for (InterpreterInfo info : intInfo) {
         String name = intp.getName() + "." + info.getName();
         completion.add(new InterpreterCompletion(name, name));
       }
     } else {
       completion.add(new InterpreterCompletion(intp.getName(), intp.getName()));
     }
   }
   return completion;
 }
Exemplo n.º 18
0
  public void bindInterpretersToNote(String id, List<String> interpreterSettingIds)
      throws IOException {
    Note note = getNote(id);
    if (note != null) {
      List<InterpreterSetting> currentBindings = replFactory.getInterpreterSettings(id);
      for (InterpreterSetting setting : currentBindings) {
        if (!interpreterSettingIds.contains(setting.getId())) {
          fireUnbindInterpreter(note, setting);
        }
      }

      replFactory.setInterpreters(note.getId(), interpreterSettingIds);
      // comment out while note.getNoteReplLoader().setInterpreters(...) do the same
      // replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds);
    }
  }
Exemplo n.º 19
0
  @Override
  public void save(Note note, AuthenticationInfo subject) throws IOException {
    GsonBuilder gsonBuilder = new GsonBuilder();
    gsonBuilder.setPrettyPrinting();
    Gson gson = gsonBuilder.create();
    String json = gson.toJson(note);
    String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json";

    File file = File.createTempFile("note", "json");
    try {
      Writer writer = new OutputStreamWriter(new FileOutputStream(file));
      writer.write(json);
      writer.close();
      s3client.putObject(new PutObjectRequest(bucketName, key, file));
    } catch (AmazonClientException ace) {
      throw new IOException("Unable to store note in S3: " + ace, ace);
    } finally {
      FileUtils.deleteQuietly(file);
    }
  }
Exemplo n.º 20
0
  private InterpreterContext getInterpreterContext(InterpreterOutput output) {
    AngularObjectRegistry registry = null;
    ResourcePool resourcePool = null;

    if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
      InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
      registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
      resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
    }

    List<InterpreterContextRunner> runners = new LinkedList<>();
    for (Paragraph p : note.getParagraphs()) {
      runners.add(new ParagraphRunner(note, note.getId(), p.getId()));
    }

    final Paragraph self = this;

    Credentials credentials = note.getCredentials();
    if (authenticationInfo != null) {
      UserCredentials userCredentials =
          credentials.getUserCredentials(authenticationInfo.getUser());
      authenticationInfo.setUserCredentials(userCredentials);
    }

    InterpreterContext interpreterContext =
        new InterpreterContext(
            note.getId(),
            getId(),
            getRequiredReplName(),
            this.getTitle(),
            this.getText(),
            this.getAuthenticationInfo(),
            this.getConfig(),
            this.settings,
            registry,
            resourcePool,
            runners,
            output);
    return interpreterContext;
  }
Exemplo n.º 21
0
 public Interpreter getRepl(String name) {
   return factory.getInterpreter(user, note.getId(), name);
 }
Exemplo n.º 22
0
 private boolean noteHasInterpreters() {
   return !factory.getInterpreterSettings(note.getId()).isEmpty();
 }
Exemplo n.º 23
0
  public InterpreterContextRunner getInterpreterContextRunner() {

    return new ParagraphRunner(note, note.getId(), getId());
  }
Exemplo n.º 24
0
  public List<Map<String, Object>> getJobListByUnixTime(
      boolean needsReload, long lastUpdateServerUnixTime, AuthenticationInfo subject) {
    final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";

    if (needsReload) {
      try {
        reloadAllNotes(subject);
      } catch (IOException e) {
        logger.error("Fail to reload notes from repository");
      }
    }

    List<Note> notes = getAllNotes();
    List<Map<String, Object>> notesInfo = new LinkedList<>();
    for (Note note : notes) {
      boolean isNotebookRunning = false;
      boolean isUpdateNotebook = false;
      long lastRunningUnixTime = 0;
      Map<String, Object> info = new HashMap<>();

      // set notebook ID
      info.put("notebookId", note.getId());

      // set notebook Name
      String notebookName = note.getName();
      if (notebookName != null && !notebookName.equals("")) {
        info.put("notebookName", note.getName());
      } else {
        info.put("notebookName", "Note " + note.getId());
      }

      // set notebook type ( cron or normal )
      if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD)
          && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) {
        info.put("notebookType", "cron");
      } else {
        info.put("notebookType", "normal");
      }

      // set paragraphs
      List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
      for (Paragraph paragraph : note.getParagraphs()) {
        // check paragraph's status.
        if (paragraph.getStatus().isRunning()) {
          isNotebookRunning = true;
          isUpdateNotebook = true;
        }

        // get data for the job manager.
        Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph);
        lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph);

        // is update notebook for last server update time.
        if (lastRunningUnixTime > lastUpdateServerUnixTime) {
          isUpdateNotebook = true;
        }
        paragraphsInfo.add(paragraphItem);
      }

      // set interpreter bind type
      String interpreterGroupName = null;
      if (replFactory.getInterpreterSettings(note.getId()) != null
          && replFactory.getInterpreterSettings(note.getId()).size() >= 1) {
        interpreterGroupName = replFactory.getInterpreterSettings(note.getId()).get(0).getName();
      }

      // not update and not running -> pass
      if (!isUpdateNotebook && !isNotebookRunning) {
        continue;
      }

      // notebook json object root information.
      info.put("interpreter", interpreterGroupName);
      info.put("isRunningJob", isNotebookRunning);
      info.put("unixTimeLastRun", lastRunningUnixTime);
      info.put("paragraphs", paragraphsInfo);
      notesInfo.add(info);
    }

    return notesInfo;
  }
Exemplo n.º 25
0
  @SuppressWarnings("rawtypes")
  private Note loadNoteFromRepo(String id, AuthenticationInfo subject) {
    Note note = null;
    try {
      note = notebookRepo.get(id, subject);
    } catch (IOException e) {
      logger.error("Failed to load " + id, e);
    }
    if (note == null) {
      return null;
    }

    // Manually inject ALL dependencies, as DI constructor was NOT used
    note.setIndex(this.notebookIndex);
    note.setCredentials(this.credentials);

    note.setInterpreterFactory(replFactory);

    note.setJobListenerFactory(jobListenerFactory);
    note.setNotebookRepo(notebookRepo);

    Map<String, SnapshotAngularObject> angularObjectSnapshot = new HashMap<>();

    // restore angular object --------------
    Date lastUpdatedDate = new Date(0);
    for (Paragraph p : note.getParagraphs()) {
      p.setNote(note);
      if (p.getDateFinished() != null && lastUpdatedDate.before(p.getDateFinished())) {
        lastUpdatedDate = p.getDateFinished();
      }
    }

    Map<String, List<AngularObject>> savedObjects = note.getAngularObjects();

    if (savedObjects != null) {
      for (String intpGroupName : savedObjects.keySet()) {
        List<AngularObject> objectList = savedObjects.get(intpGroupName);

        for (AngularObject object : objectList) {
          SnapshotAngularObject snapshot = angularObjectSnapshot.get(object.getName());
          if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) {
            angularObjectSnapshot.put(
                object.getName(),
                new SnapshotAngularObject(intpGroupName, object, lastUpdatedDate));
          }
        }
      }
    }

    note.setNoteEventListener(this);

    synchronized (notes) {
      notes.put(note.getId(), note);
      refreshCron(note.getId());
    }

    for (String name : angularObjectSnapshot.keySet()) {
      SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
      List<InterpreterSetting> settings = replFactory.get();
      for (InterpreterSetting setting : settings) {
        InterpreterGroup intpGroup = setting.getInterpreterGroup(note.getId());
        if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
          AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
          String noteId = snapshot.getAngularObject().getNoteId();
          String paragraphId = snapshot.getAngularObject().getParagraphId();
          // at this point, remote interpreter process is not created.
          // so does not make sense add it to the remote.
          //
          // therefore instead of addAndNotifyRemoteProcess(), need to use add()
          // that results add angularObject only in ZeppelinServer side not remoteProcessSide
          registry.add(name, snapshot.getAngularObject().get(), noteId, paragraphId);
        }
      }
    }

    return note;
  }