Пример #1
0
  /**
   * Clone paragraph and add it to note.
   *
   * @param srcParagraph source paragraph
   */
  void addCloneParagraph(Paragraph srcParagraph) {

    // Keep paragraph original ID
    final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory);

    Map<String, Object> config = new HashMap<>(srcParagraph.getConfig());
    Map<String, Object> param = new HashMap<>(srcParagraph.settings.getParams());
    Map<String, Input> form = new HashMap<>(srcParagraph.settings.getForms());

    newParagraph.setConfig(config);
    newParagraph.settings.setParams(param);
    newParagraph.settings.setForms(form);
    newParagraph.setText(srcParagraph.getText());
    newParagraph.setTitle(srcParagraph.getTitle());

    try {
      Gson gson = new Gson();
      String resultJson = gson.toJson(srcParagraph.getReturn());
      InterpreterResult result = gson.fromJson(resultJson, InterpreterResult.class);
      newParagraph.setReturn(result, null);
    } catch (Exception e) {
      // 'result' part of Note consists of exception, instead of actual interpreter results
      logger.warn(
          "Paragraph " + srcParagraph.getId() + " has a result with exception. " + e.getMessage());
    }

    synchronized (paragraphs) {
      paragraphs.add(newParagraph);
    }
    if (noteEventListener != null) {
      noteEventListener.onParagraphCreate(newParagraph);
    }
  }
  @Test
  public void testSyncUpdateMain() throws IOException {

    /* create note */
    Note note = notebookSync.createNote();
    Paragraph p1 = note.addParagraph();
    Map config = p1.getConfig();
    config.put("enabled", true);
    p1.setConfig(config);
    p1.setText("hello world");

    /* new paragraph exists in note instance */
    assertEquals(1, note.getParagraphs().size());

    /* new paragraph not yet saved into storages */
    assertEquals(
        0, notebookRepoSync.get(0, notebookRepoSync.list(0).get(0).getId()).getParagraphs().size());
    assertEquals(
        0, notebookRepoSync.get(1, notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());

    /* save to storage under index 0 (first storage) */
    notebookRepoSync.save(0, note);

    /* check paragraph saved to first storage */
    assertEquals(
        1, notebookRepoSync.get(0, notebookRepoSync.list(0).get(0).getId()).getParagraphs().size());
    /* check paragraph isn't saved to second storage */
    assertEquals(
        0, notebookRepoSync.get(1, notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());
    /* apply sync */
    notebookRepoSync.sync();
    /* check whether added to second storage */
    assertEquals(
        1, notebookRepoSync.get(1, notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());
    /* check whether same paragraph id */
    assertEquals(
        p1.getId(),
        notebookRepoSync
            .get(0, notebookRepoSync.list(0).get(0).getId())
            .getLastParagraph()
            .getId());
    assertEquals(
        p1.getId(),
        notebookRepoSync
            .get(1, notebookRepoSync.list(1).get(0).getId())
            .getLastParagraph()
            .getId());
  }
Пример #3
0
  /**
   * Run a single paragraph.
   *
   * @param paragraphId ID of paragraph
   */
  public void run(String paragraphId) {
    Paragraph p = getParagraph(paragraphId);

    if (p.isBlankParagraph()) {
      logger.info("skip to run blank paragraph. {}", p.getId());
      return;
    }

    p.setListener(jobListenerFactory.getParagraphJobListener(this));
    String requiredReplName = p.getRequiredReplName();
    Interpreter intp = factory.getInterpreter(p.getUser(), getId(), requiredReplName);

    if (intp == null) {
      String intpExceptionMsg =
          p.getJobName() + "'s Interpreter " + requiredReplName + " not found";
      InterpreterException intpException = new InterpreterException(intpExceptionMsg);
      InterpreterResult intpResult =
          new InterpreterResult(InterpreterResult.Code.ERROR, intpException.getMessage());
      p.setReturn(intpResult, intpException);
      p.setStatus(Job.Status.ERROR);
      throw intpException;
    }
    if (p.getConfig().get("enabled") == null || (Boolean) p.getConfig().get("enabled")) {
      p.setAuthenticationInfo(p.getAuthenticationInfo());
      intp.getScheduler().submit(p);
    }
  }
Пример #4
0
  private InterpreterContext getInterpreterContext() {
    AngularObjectRegistry registry = null;

    if (!getNoteReplLoader().getInterpreterSettings().isEmpty()) {
      InterpreterSetting intpGroup = getNoteReplLoader().getInterpreterSettings().get(0);
      registry = intpGroup.getInterpreterGroup().getAngularObjectRegistry();
    }

    List<InterpreterContextRunner> runners = new LinkedList<InterpreterContextRunner>();
    for (Paragraph p : note.getParagraphs()) {
      runners.add(new ParagraphRunner(note, note.id(), p.getId()));
    }

    InterpreterContext interpreterContext =
        new InterpreterContext(
            note.id(),
            getId(),
            this.getTitle(),
            this.getText(),
            this.getConfig(),
            this.settings,
            registry,
            runners);
    return interpreterContext;
  }
  /**
   * Run paragraph job REST API
   *
   * @param message - JSON with params if user wants to update dynamic form's value null, empty
   *     string, empty json if user doesn't want to update
   * @return JSON with status.OK
   * @throws IOException, IllegalArgumentException
   */
  @POST
  @Path("job/{notebookId}/{paragraphId}")
  @ZeppelinApi
  public Response runParagraph(
      @PathParam("notebookId") String notebookId,
      @PathParam("paragraphId") String paragraphId,
      String message)
      throws IOException, IllegalArgumentException {
    LOG.info("run paragraph job {} {} {}", notebookId, paragraphId, message);

    Note note = notebook.getNote(notebookId);
    if (note == null) {
      return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
    }

    Paragraph paragraph = note.getParagraph(paragraphId);
    if (paragraph == null) {
      return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build();
    }

    // handle params if presented
    if (!StringUtils.isEmpty(message)) {
      RunParagraphWithParametersRequest request =
          gson.fromJson(message, RunParagraphWithParametersRequest.class);
      Map<String, Object> paramsForUpdating = request.getParams();
      if (paramsForUpdating != null) {
        paragraph.settings.getParams().putAll(paramsForUpdating);
        AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
        note.persist(subject);
      }
    }

    note.run(paragraph.getId());
    return new JsonResponse<>(Status.OK).build();
  }
  /**
   * Insert paragraph REST API
   *
   * @param message - JSON containing paragraph's information
   * @return JSON with status.OK
   * @throws IOException
   */
  @POST
  @Path("{notebookId}/paragraph")
  @ZeppelinApi
  public Response insertParagraph(@PathParam("notebookId") String notebookId, String message)
      throws IOException {
    LOG.info("insert paragraph {} {}", notebookId, message);

    Note note = notebook.getNote(notebookId);
    if (note == null) {
      return new JsonResponse(Status.NOT_FOUND, "note not found.").build();
    }

    NewParagraphRequest request = gson.fromJson(message, NewParagraphRequest.class);

    Paragraph p;
    Double indexDouble = request.getIndex();
    if (indexDouble == null) {
      p = note.addParagraph();
    } else {
      p = note.insertParagraph(indexDouble.intValue());
    }
    p.setTitle(request.getTitle());
    p.setText(request.getText());

    AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
    note.persist(subject);
    notebookServer.broadcastNote(note);
    return new JsonResponse(Status.CREATED, "", p.getId()).build();
  }
Пример #7
0
  private InterpreterContext getInterpreterContext() {
    AngularObjectRegistry registry = null;
    ResourcePool resourcePool = null;

    if (!getNoteReplLoader().getInterpreterSettings().isEmpty()) {
      InterpreterSetting intpGroup = getNoteReplLoader().getInterpreterSettings().get(0);
      registry = intpGroup.getInterpreterGroup().getAngularObjectRegistry();
      resourcePool = intpGroup.getInterpreterGroup().getResourcePool();
    }

    List<InterpreterContextRunner> runners = new LinkedList<InterpreterContextRunner>();
    for (Paragraph p : note.getParagraphs()) {
      runners.add(new ParagraphRunner(note, note.id(), p.getId()));
    }

    final Paragraph self = this;
    InterpreterContext interpreterContext =
        new InterpreterContext(
            note.id(),
            getId(),
            this.getTitle(),
            this.getText(),
            this.getConfig(),
            this.settings,
            registry,
            resourcePool,
            runners,
            new InterpreterOutput(
                new InterpreterOutputListener() {
                  @Override
                  public void onAppend(InterpreterOutput out, byte[] line) {
                    updateParagraphResult(out);
                    ((ParagraphJobListener) getListener())
                        .onOutputAppend(self, out, new String(line));
                  }

                  @Override
                  public void onUpdate(InterpreterOutput out, byte[] output) {
                    updateParagraphResult(out);
                    ((ParagraphJobListener) getListener())
                        .onOutputUpdate(self, out, new String(output));
                  }

                  private void updateParagraphResult(InterpreterOutput out) {
                    // update paragraph result
                    Throwable t = null;
                    String message = null;
                    try {
                      message = new String(out.toByteArray());
                    } catch (IOException e) {
                      logger().error(e.getMessage(), e);
                      t = e;
                    }
                    setReturn(new InterpreterResult(Code.SUCCESS, out.getType(), message), t);
                  }
                }));
    return interpreterContext;
  }
Пример #8
0
  public void removeNote(String id, AuthenticationInfo subject) {
    Note note;

    synchronized (notes) {
      note = notes.remove(id);
    }
    replFactory.removeNoteInterpreterSettingBinding(id);
    notebookIndex.deleteIndexDocs(note);
    notebookAuthorization.removeNote(id);

    // remove from all interpreter instance's angular object registry
    for (InterpreterSetting settings : replFactory.get()) {
      AngularObjectRegistry registry = settings.getInterpreterGroup(id).getAngularObjectRegistry();
      if (registry instanceof RemoteAngularObjectRegistry) {
        // remove paragraph scope object
        for (Paragraph p : note.getParagraphs()) {
          ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, p.getId());

          // remove app scope object
          List<ApplicationState> appStates = p.getAllApplicationStates();
          if (appStates != null) {
            for (ApplicationState app : appStates) {
              ((RemoteAngularObjectRegistry) registry)
                  .removeAllAndNotifyRemoteProcess(id, app.getId());
            }
          }
        }
        // remove notebook scope object
        ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, null);
      } else {
        // remove paragraph scope object
        for (Paragraph p : note.getParagraphs()) {
          registry.removeAll(id, p.getId());

          // remove app scope object
          List<ApplicationState> appStates = p.getAllApplicationStates();
          if (appStates != null) {
            for (ApplicationState app : appStates) {
              registry.removeAll(id, app.getId());
            }
          }
        }
        // remove notebook scope object
        registry.removeAll(id, null);
      }
    }

    ResourcePoolUtils.removeResourcesBelongsToNote(id);

    fireNoteRemoveEvent(note);

    try {
      note.unpersist(subject);
    } catch (IOException e) {
      logger.error(e.toString(), e);
    }
  }
  @Test
  public void testInterpreterRestart() throws IOException, InterruptedException {
    // create new note
    Note note = ZeppelinServer.notebook.createNote(anonymous);
    note.addParagraph();
    Paragraph p = note.getLastParagraph();
    Map config = p.getConfig();
    config.put("enabled", true);

    // run markdown paragraph
    p.setConfig(config);
    p.setText("%md markdown");
    p.setAuthenticationInfo(anonymous);
    note.run(p.getId());
    while (p.getStatus() != Status.FINISHED) {
      Thread.sleep(100);
    }
    assertEquals("<p>markdown</p>\n", p.getResult().message());

    // restart interpreter
    for (InterpreterSetting setting :
        ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
      if (setting.getName().equals("md")) {
        // Call Restart Interpreter REST API
        PutMethod put = httpPut("/interpreter/setting/restart/" + setting.getId(), "");
        assertThat("test interpreter restart:", put, isAllowed());
        put.releaseConnection();
        break;
      }
    }

    // run markdown paragraph, again
    p = note.addParagraph();
    p.setConfig(config);
    p.setText("%md markdown restarted");
    p.setAuthenticationInfo(anonymous);
    note.run(p.getId());
    while (p.getStatus() != Status.FINISHED) {
      Thread.sleep(100);
    }
    assertEquals("<p>markdown restarted</p>\n", p.getResult().message());
    // cleanup
    ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
  }
Пример #10
0
 public Paragraph getParagraph(String paragraphId) {
   synchronized (paragraphs) {
     for (Paragraph p : paragraphs) {
       if (p.getId().equals(paragraphId)) {
         return p;
       }
     }
   }
   return null;
 }
Пример #11
0
 public Map<String, String> generateSingleParagraphInfo(String paragraphId) {
   synchronized (paragraphs) {
     for (Paragraph p : paragraphs) {
       if (p.getId().equals(paragraphId)) {
         return populateParagraphInfo(p);
       }
     }
     return new HashMap<>();
   }
 }
Пример #12
0
  private Map<String, Object> getParagraphForJobManagerItem(Paragraph paragraph) {
    Map<String, Object> paragraphItem = new HashMap<>();

    // set paragraph id
    paragraphItem.put("id", paragraph.getId());

    // set paragraph name
    String paragraphName = paragraph.getTitle();
    if (paragraphName != null) {
      paragraphItem.put("name", paragraphName);
    } else {
      paragraphItem.put("name", paragraph.getId());
    }

    // set status for paragraph.
    paragraphItem.put("status", paragraph.getStatus().toString());

    return paragraphItem;
  }
Пример #13
0
 /**
  * Clear paragraph output by id.
  *
  * @param paragraphId ID of paragraph
  * @return Paragraph
  */
 public Paragraph clearParagraphOutput(String paragraphId) {
   synchronized (paragraphs) {
     for (Paragraph p : paragraphs) {
       if (p.getId().equals(paragraphId)) {
         p.setReturn(null, null);
         return p;
       }
     }
   }
   return null;
 }
Пример #14
0
 /**
  * Clear paragraph output by id.
  *
  * @param paragraphId
  * @return
  */
 public Paragraph clearParagraphOutput(String paragraphId) {
   synchronized (paragraphs) {
     for (int i = 0; i < paragraphs.size(); i++) {
       Paragraph p = paragraphs.get(i);
       if (p.getId().equals(paragraphId)) {
         p.setReturn(null, null);
         return p;
       }
     }
   }
   return null;
 }
Пример #15
0
 /**
  * Remove paragraph by id.
  *
  * @param paragraphId
  * @return
  */
 public Paragraph removeParagraph(String paragraphId) {
   synchronized (paragraphs) {
     for (int i = 0; i < paragraphs.size(); i++) {
       Paragraph p = paragraphs.get(i);
       if (p.getId().equals(paragraphId)) {
         paragraphs.remove(i);
         return p;
       }
     }
   }
   return null;
 }
  @Test
  public void testRestartInterpreterPerNote() throws IOException, InterruptedException {
    // create new note
    Note note = ZeppelinServer.notebook.createNote(anonymous);
    note.addParagraph();
    Paragraph p = note.getLastParagraph();
    Map config = p.getConfig();
    config.put("enabled", true);

    // run markdown paragraph.
    p.setConfig(config);
    p.setText("%md markdown");
    p.setAuthenticationInfo(anonymous);
    note.run(p.getId());
    while (p.getStatus() != Status.FINISHED) {
      Thread.sleep(100);
    }
    assertEquals("<p>markdown</p>\n", p.getResult().message());

    // get md interpreter
    InterpreterSetting mdIntpSetting = null;
    for (InterpreterSetting setting :
        ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
      if (setting.getName().equals("md")) {
        mdIntpSetting = setting;
        break;
      }
    }

    String jsonRequest = "{\"noteId\":\"" + note.getId() + "\"}";

    // Restart isolated mode of Interpreter for note.
    mdIntpSetting.getOption().setPerNote(InterpreterOption.ISOLATED);
    PutMethod put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
    assertThat("isolated interpreter restart:", put, isAllowed());
    put.releaseConnection();

    // Restart scoped mode of Interpreter for note.
    mdIntpSetting.getOption().setPerNote(InterpreterOption.SCOPED);
    put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
    assertThat("scoped interpreter restart:", put, isAllowed());
    put.releaseConnection();

    // Restart shared mode of Interpreter for note.
    mdIntpSetting.getOption().setPerNote(InterpreterOption.SHARED);
    put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
    assertThat("shared interpreter restart:", put, isAllowed());
    put.releaseConnection();

    ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
  }
Пример #17
0
 /** Run all paragraphs sequentially. */
 public synchronized void runAll() {
   String cronExecutingUser = (String) getConfig().get("cronExecutingUser");
   if (null == cronExecutingUser) {
     cronExecutingUser = "******";
   }
   for (Paragraph p : getParagraphs()) {
     if (!p.isEnabled()) {
       continue;
     }
     AuthenticationInfo authenticationInfo = new AuthenticationInfo();
     authenticationInfo.setUser(cronExecutingUser);
     p.setAuthenticationInfo(authenticationInfo);
     run(p.getId());
   }
 }
Пример #18
0
  public void initializeJobListenerForParagraph(Paragraph paragraph) {
    final Note paragraphNote = paragraph.getNote();
    if (paragraphNote.getId().equals(this.getId())) {
      throw new IllegalArgumentException(
          format(
              "The paragraph %s from note %s " + "does not belong to note %s",
              paragraph.getId(), paragraphNote.getId(), this.getId()));
    }

    boolean foundParagraph = false;
    for (Paragraph ownParagraph : paragraphs) {
      if (paragraph.getId().equals(ownParagraph.getId())) {
        paragraph.setListener(this.jobListenerFactory.getParagraphJobListener(this));
        foundParagraph = true;
      }
    }

    if (!foundParagraph) {
      throw new IllegalArgumentException(
          format(
              "Cannot find paragraph %s " + "from note %s",
              paragraph.getId(), paragraphNote.getId()));
    }
  }
Пример #19
0
 private Map<String, String> populateParagraphInfo(Paragraph p) {
   Map<String, String> info = new HashMap<>();
   info.put("id", p.getId());
   info.put("status", p.getStatus().toString());
   if (p.getDateStarted() != null) {
     info.put("started", p.getDateStarted().toString());
   }
   if (p.getDateFinished() != null) {
     info.put("finished", p.getDateFinished().toString());
   }
   if (p.getStatus().isRunning()) {
     info.put("progress", String.valueOf(p.progress()));
   } else {
     info.put("progress", String.valueOf(100));
   }
   return info;
 }
Пример #20
0
  /**
   * Remove paragraph by id.
   *
   * @param paragraphId ID of paragraph
   * @return a paragraph that was deleted, or <code>null</code> otherwise
   */
  public Paragraph removeParagraph(String user, String paragraphId) {
    removeAllAngularObjectInParagraph(user, paragraphId);
    ResourcePoolUtils.removeResourcesBelongsToParagraph(getId(), paragraphId);
    synchronized (paragraphs) {
      Iterator<Paragraph> i = paragraphs.iterator();
      while (i.hasNext()) {
        Paragraph p = i.next();
        if (p.getId().equals(paragraphId)) {
          index.deleteIndexDoc(this, p);
          i.remove();

          if (noteEventListener != null) {
            noteEventListener.onParagraphRemove(p);
          }
          return p;
        }
      }
    }
    return null;
  }
Пример #21
0
  private InterpreterContext getInterpreterContext(InterpreterOutput output) {
    AngularObjectRegistry registry = null;
    ResourcePool resourcePool = null;

    if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
      InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
      registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
      resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
    }

    List<InterpreterContextRunner> runners = new LinkedList<>();
    for (Paragraph p : note.getParagraphs()) {
      runners.add(new ParagraphRunner(note, note.getId(), p.getId()));
    }

    final Paragraph self = this;

    Credentials credentials = note.getCredentials();
    if (authenticationInfo != null) {
      UserCredentials userCredentials =
          credentials.getUserCredentials(authenticationInfo.getUser());
      authenticationInfo.setUserCredentials(userCredentials);
    }

    InterpreterContext interpreterContext =
        new InterpreterContext(
            note.getId(),
            getId(),
            getRequiredReplName(),
            this.getTitle(),
            this.getText(),
            this.getAuthenticationInfo(),
            this.getConfig(),
            this.settings,
            registry,
            resourcePool,
            runners,
            output);
    return interpreterContext;
  }