Example #1
0
  private Note getNote(String key) throws IOException {
    GsonBuilder gsonBuilder = new GsonBuilder();
    gsonBuilder.setPrettyPrinting();
    Gson gson =
        gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create();

    S3Object s3object;
    try {
      s3object = s3client.getObject(new GetObjectRequest(bucketName, key));
    } catch (AmazonClientException ace) {
      throw new IOException("Unable to retrieve object from S3: " + ace, ace);
    }

    Note note;
    try (InputStream ins = s3object.getObjectContent()) {
      String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
      note = gson.fromJson(json, Note.class);
    }

    for (Paragraph p : note.getParagraphs()) {
      if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) {
        p.setStatus(Status.ABORT);
      }
    }

    return note;
  }
Example #2
0
 private Map<String, String> populateParagraphInfo(Paragraph p) {
   Map<String, String> info = new HashMap<>();
   info.put("id", p.getId());
   info.put("status", p.getStatus().toString());
   if (p.getDateStarted() != null) {
     info.put("started", p.getDateStarted().toString());
   }
   if (p.getDateFinished() != null) {
     info.put("finished", p.getDateFinished().toString());
   }
   if (p.getStatus().isRunning()) {
     info.put("progress", String.valueOf(p.progress()));
   } else {
     info.put("progress", String.valueOf(100));
   }
   return info;
 }
  public List<Map<String, Object>> getJobListBymNotebookId(String notebookID) {
    final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";
    long lastRunningUnixTime = 0;
    boolean isNotebookRunning = false;
    Note jobNote = getNote(notebookID);
    List<Map<String, Object>> notesInfo = new LinkedList<>();
    if (jobNote == null) {
      return notesInfo;
    }

    Map<String, Object> info = new HashMap<>();

    info.put("notebookId", jobNote.getId());
    String notebookName = jobNote.getName();
    if (notebookName != null && !notebookName.equals("")) {
      info.put("notebookName", jobNote.getName());
    } else {
      info.put("notebookName", "Note " + jobNote.getId());
    }
    // set notebook type ( cron or normal )
    if (jobNote.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD)
        && !jobNote.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) {
      info.put("notebookType", "cron");
    } else {
      info.put("notebookType", "normal");
    }

    // set paragraphs
    List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
    for (Paragraph paragraph : jobNote.getParagraphs()) {
      // check paragraph's status.
      if (paragraph.getStatus().isRunning()) {
        isNotebookRunning = true;
      }

      // get data for the job manager.
      Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph);
      lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph);

      paragraphsInfo.add(paragraphItem);
    }

    // set interpreter bind type
    String interpreterGroupName = null;
    if (replFactory.getInterpreterSettings(jobNote.getId()) != null
        && replFactory.getInterpreterSettings(jobNote.getId()).size() >= 1) {
      interpreterGroupName = replFactory.getInterpreterSettings(jobNote.getId()).get(0).getName();
    }

    // notebook json object root information.
    info.put("interpreter", interpreterGroupName);
    info.put("isRunningJob", isNotebookRunning);
    info.put("unixTimeLastRun", lastRunningUnixTime);
    info.put("paragraphs", paragraphsInfo);
    notesInfo.add(info);

    return notesInfo;
  };
  @Test
  public void testInterpreterRestart() throws IOException, InterruptedException {
    // create new note
    Note note = ZeppelinServer.notebook.createNote(anonymous);
    note.addParagraph();
    Paragraph p = note.getLastParagraph();
    Map config = p.getConfig();
    config.put("enabled", true);

    // run markdown paragraph
    p.setConfig(config);
    p.setText("%md markdown");
    p.setAuthenticationInfo(anonymous);
    note.run(p.getId());
    while (p.getStatus() != Status.FINISHED) {
      Thread.sleep(100);
    }
    assertEquals("<p>markdown</p>\n", p.getResult().message());

    // restart interpreter
    for (InterpreterSetting setting :
        ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
      if (setting.getName().equals("md")) {
        // Call Restart Interpreter REST API
        PutMethod put = httpPut("/interpreter/setting/restart/" + setting.getId(), "");
        assertThat("test interpreter restart:", put, isAllowed());
        put.releaseConnection();
        break;
      }
    }

    // run markdown paragraph, again
    p = note.addParagraph();
    p.setConfig(config);
    p.setText("%md markdown restarted");
    p.setAuthenticationInfo(anonymous);
    note.run(p.getId());
    while (p.getStatus() != Status.FINISHED) {
      Thread.sleep(100);
    }
    assertEquals("<p>markdown restarted</p>\n", p.getResult().message());
    // cleanup
    ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
  }
  @Test
  public void testRestartInterpreterPerNote() throws IOException, InterruptedException {
    // create new note
    Note note = ZeppelinServer.notebook.createNote(anonymous);
    note.addParagraph();
    Paragraph p = note.getLastParagraph();
    Map config = p.getConfig();
    config.put("enabled", true);

    // run markdown paragraph.
    p.setConfig(config);
    p.setText("%md markdown");
    p.setAuthenticationInfo(anonymous);
    note.run(p.getId());
    while (p.getStatus() != Status.FINISHED) {
      Thread.sleep(100);
    }
    assertEquals("<p>markdown</p>\n", p.getResult().message());

    // get md interpreter
    InterpreterSetting mdIntpSetting = null;
    for (InterpreterSetting setting :
        ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
      if (setting.getName().equals("md")) {
        mdIntpSetting = setting;
        break;
      }
    }

    String jsonRequest = "{\"noteId\":\"" + note.getId() + "\"}";

    // Restart isolated mode of Interpreter for note.
    mdIntpSetting.getOption().setPerNote(InterpreterOption.ISOLATED);
    PutMethod put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
    assertThat("isolated interpreter restart:", put, isAllowed());
    put.releaseConnection();

    // Restart scoped mode of Interpreter for note.
    mdIntpSetting.getOption().setPerNote(InterpreterOption.SCOPED);
    put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
    assertThat("scoped interpreter restart:", put, isAllowed());
    put.releaseConnection();

    // Restart shared mode of Interpreter for note.
    mdIntpSetting.getOption().setPerNote(InterpreterOption.SHARED);
    put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
    assertThat("shared interpreter restart:", put, isAllowed());
    put.releaseConnection();

    ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
  }
  private Map<String, Object> getParagraphForJobManagerItem(Paragraph paragraph) {
    Map<String, Object> paragraphItem = new HashMap<>();

    // set paragraph id
    paragraphItem.put("id", paragraph.getId());

    // set paragraph name
    String paragraphName = paragraph.getTitle();
    if (paragraphName != null) {
      paragraphItem.put("name", paragraphName);
    } else {
      paragraphItem.put("name", paragraph.getId());
    }

    // set status for paragraph.
    paragraphItem.put("status", paragraph.getStatus().toString());

    return paragraphItem;
  }
  public List<Map<String, Object>> getJobListByUnixTime(
      boolean needsReload, long lastUpdateServerUnixTime, AuthenticationInfo subject) {
    final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";

    if (needsReload) {
      try {
        reloadAllNotes(subject);
      } catch (IOException e) {
        logger.error("Fail to reload notes from repository");
      }
    }

    List<Note> notes = getAllNotes();
    List<Map<String, Object>> notesInfo = new LinkedList<>();
    for (Note note : notes) {
      boolean isNotebookRunning = false;
      boolean isUpdateNotebook = false;
      long lastRunningUnixTime = 0;
      Map<String, Object> info = new HashMap<>();

      // set notebook ID
      info.put("notebookId", note.getId());

      // set notebook Name
      String notebookName = note.getName();
      if (notebookName != null && !notebookName.equals("")) {
        info.put("notebookName", note.getName());
      } else {
        info.put("notebookName", "Note " + note.getId());
      }

      // set notebook type ( cron or normal )
      if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD)
          && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) {
        info.put("notebookType", "cron");
      } else {
        info.put("notebookType", "normal");
      }

      // set paragraphs
      List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
      for (Paragraph paragraph : note.getParagraphs()) {
        // check paragraph's status.
        if (paragraph.getStatus().isRunning()) {
          isNotebookRunning = true;
          isUpdateNotebook = true;
        }

        // get data for the job manager.
        Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph);
        lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph);

        // is update notebook for last server update time.
        if (lastRunningUnixTime > lastUpdateServerUnixTime) {
          isUpdateNotebook = true;
        }
        paragraphsInfo.add(paragraphItem);
      }

      // set interpreter bind type
      String interpreterGroupName = null;
      if (replFactory.getInterpreterSettings(note.getId()) != null
          && replFactory.getInterpreterSettings(note.getId()).size() >= 1) {
        interpreterGroupName = replFactory.getInterpreterSettings(note.getId()).get(0).getName();
      }

      // not update and not running -> pass
      if (!isUpdateNotebook && !isNotebookRunning) {
        continue;
      }

      // notebook json object root information.
      info.put("interpreter", interpreterGroupName);
      info.put("isRunningJob", isNotebookRunning);
      info.put("unixTimeLastRun", lastRunningUnixTime);
      info.put("paragraphs", paragraphsInfo);
      notesInfo.add(info);
    }

    return notesInfo;
  }