Пример #1
0
  public List<Map<String, Object>> getJobListBymNotebookId(String notebookID) {
    final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";
    long lastRunningUnixTime = 0;
    boolean isNotebookRunning = false;
    Note jobNote = getNote(notebookID);
    List<Map<String, Object>> notesInfo = new LinkedList<>();
    if (jobNote == null) {
      return notesInfo;
    }

    Map<String, Object> info = new HashMap<>();

    info.put("notebookId", jobNote.getId());
    String notebookName = jobNote.getName();
    if (notebookName != null && !notebookName.equals("")) {
      info.put("notebookName", jobNote.getName());
    } else {
      info.put("notebookName", "Note " + jobNote.getId());
    }
    // set notebook type ( cron or normal )
    if (jobNote.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD)
        && !jobNote.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) {
      info.put("notebookType", "cron");
    } else {
      info.put("notebookType", "normal");
    }

    // set paragraphs
    List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
    for (Paragraph paragraph : jobNote.getParagraphs()) {
      // check paragraph's status.
      if (paragraph.getStatus().isRunning()) {
        isNotebookRunning = true;
      }

      // get data for the job manager.
      Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph);
      lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph);

      paragraphsInfo.add(paragraphItem);
    }

    // set interpreter bind type
    String interpreterGroupName = null;
    if (replFactory.getInterpreterSettings(jobNote.getId()) != null
        && replFactory.getInterpreterSettings(jobNote.getId()).size() >= 1) {
      interpreterGroupName = replFactory.getInterpreterSettings(jobNote.getId()).get(0).getName();
    }

    // notebook json object root information.
    info.put("interpreter", interpreterGroupName);
    info.put("isRunningJob", isNotebookRunning);
    info.put("unixTimeLastRun", lastRunningUnixTime);
    info.put("paragraphs", paragraphsInfo);
    notesInfo.add(info);

    return notesInfo;
  };
Пример #2
0
  public void removeNote(String id, AuthenticationInfo subject) {
    Note note;

    synchronized (notes) {
      note = notes.remove(id);
    }
    replFactory.removeNoteInterpreterSettingBinding(id);
    notebookIndex.deleteIndexDocs(note);
    notebookAuthorization.removeNote(id);

    // remove from all interpreter instance's angular object registry
    for (InterpreterSetting settings : replFactory.get()) {
      AngularObjectRegistry registry = settings.getInterpreterGroup(id).getAngularObjectRegistry();
      if (registry instanceof RemoteAngularObjectRegistry) {
        // remove paragraph scope object
        for (Paragraph p : note.getParagraphs()) {
          ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, p.getId());

          // remove app scope object
          List<ApplicationState> appStates = p.getAllApplicationStates();
          if (appStates != null) {
            for (ApplicationState app : appStates) {
              ((RemoteAngularObjectRegistry) registry)
                  .removeAllAndNotifyRemoteProcess(id, app.getId());
            }
          }
        }
        // remove notebook scope object
        ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, null);
      } else {
        // remove paragraph scope object
        for (Paragraph p : note.getParagraphs()) {
          registry.removeAll(id, p.getId());

          // remove app scope object
          List<ApplicationState> appStates = p.getAllApplicationStates();
          if (appStates != null) {
            for (ApplicationState app : appStates) {
              registry.removeAll(id, app.getId());
            }
          }
        }
        // remove notebook scope object
        registry.removeAll(id, null);
      }
    }

    ResourcePoolUtils.removeResourcesBelongsToNote(id);

    fireNoteRemoveEvent(note);

    try {
      note.unpersist(subject);
    } catch (IOException e) {
      logger.error(e.toString(), e);
    }
  }
Пример #3
0
  /**
   * Run a single paragraph.
   *
   * @param paragraphId ID of paragraph
   */
  public void run(String paragraphId) {
    Paragraph p = getParagraph(paragraphId);

    if (p.isBlankParagraph()) {
      logger.info("skip to run blank paragraph. {}", p.getId());
      return;
    }

    p.setListener(jobListenerFactory.getParagraphJobListener(this));
    String requiredReplName = p.getRequiredReplName();
    Interpreter intp = factory.getInterpreter(p.getUser(), getId(), requiredReplName);

    if (intp == null) {
      String intpExceptionMsg =
          p.getJobName() + "'s Interpreter " + requiredReplName + " not found";
      InterpreterException intpException = new InterpreterException(intpExceptionMsg);
      InterpreterResult intpResult =
          new InterpreterResult(InterpreterResult.Code.ERROR, intpException.getMessage());
      p.setReturn(intpResult, intpException);
      p.setStatus(Job.Status.ERROR);
      throw intpException;
    }
    if (p.getConfig().get("enabled") == null || (Boolean) p.getConfig().get("enabled")) {
      p.setAuthenticationInfo(p.getAuthenticationInfo());
      intp.getScheduler().submit(p);
    }
  }
Пример #4
0
  public void bindInterpretersToNote(String id, List<String> interpreterSettingIds)
      throws IOException {
    Note note = getNote(id);
    if (note != null) {
      List<InterpreterSetting> currentBindings = replFactory.getInterpreterSettings(id);
      for (InterpreterSetting setting : currentBindings) {
        if (!interpreterSettingIds.contains(setting.getId())) {
          fireUnbindInterpreter(note, setting);
        }
      }

      replFactory.setInterpreters(note.getId(), interpreterSettingIds);
      // comment out while note.getNoteReplLoader().setInterpreters(...) do the same
      // replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds);
    }
  }
Пример #5
0
 /**
  * Create new note.
  *
  * @return
  * @throws IOException
  */
 public Note createNote() throws IOException {
   if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) {
     return createNote(replFactory.getDefaultInterpreterSettingList());
   } else {
     return createNote(null);
   }
 }
Пример #6
0
 public List<InterpreterSetting> getBindedInterpreterSettings(String id) {
   Note note = getNote(id);
   if (note != null) {
     return replFactory.getInterpreterSettings(note.getId());
   } else {
     return new LinkedList<>();
   }
 }
Пример #7
0
 private boolean isValidInterpreter(String replName) {
   try {
     return factory.getInterpreter(user, note.getId(), replName) != null;
   } catch (InterpreterException e) {
     // ignore this exception, it would be recaught when running paragraph.
     return false;
   }
 }
Пример #8
0
 public void bindInterpretersToNote(String id, List<String> interpreterSettingIds)
     throws IOException {
   Note note = getNote(id);
   if (note != null) {
     note.getNoteReplLoader().setInterpreters(interpreterSettingIds);
     replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds);
   }
 }
Пример #9
0
 private InterpreterSetting getInterpreterSettingById(String id) {
   InterpreterSetting setting = null;
   for (InterpreterSetting i : factory.getInterpreterSettings(note.getId())) {
     if (id.startsWith(i.getId())) {
       setting = i;
       break;
     }
   }
   return setting;
 }
Пример #10
0
 /**
  * Create new note.
  *
  * @throws IOException
  */
 public Note createNote(AuthenticationInfo subject) throws IOException {
   Note note;
   if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) {
     note = createNote(replFactory.getDefaultInterpreterSettingList(), subject);
   } else {
     note = createNote(null, subject);
   }
   notebookIndex.addIndexDoc(note);
   return note;
 }
Пример #11
0
  private InterpreterContext getInterpreterContext(InterpreterOutput output) {
    AngularObjectRegistry registry = null;
    ResourcePool resourcePool = null;

    if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
      InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
      registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
      resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
    }

    List<InterpreterContextRunner> runners = new LinkedList<>();
    for (Paragraph p : note.getParagraphs()) {
      runners.add(new ParagraphRunner(note, note.getId(), p.getId()));
    }

    final Paragraph self = this;

    Credentials credentials = note.getCredentials();
    if (authenticationInfo != null) {
      UserCredentials userCredentials =
          credentials.getUserCredentials(authenticationInfo.getUser());
      authenticationInfo.setUserCredentials(userCredentials);
    }

    InterpreterContext interpreterContext =
        new InterpreterContext(
            note.getId(),
            getId(),
            getRequiredReplName(),
            this.getTitle(),
            this.getText(),
            this.getAuthenticationInfo(),
            this.getConfig(),
            this.settings,
            registry,
            resourcePool,
            runners,
            output);
    return interpreterContext;
  }
Пример #12
0
  private void snapshotAngularObjectRegistry(String user) {
    angularObjects = new HashMap<>();

    List<InterpreterSetting> settings = factory.getInterpreterSettings(getId());
    if (settings == null || settings.size() == 0) {
      return;
    }

    for (InterpreterSetting setting : settings) {
      InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
      AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
      angularObjects.put(intpGroup.getId(), registry.getAllWithGlobal(id));
    }
  }
Пример #13
0
 public List<InterpreterCompletion> getInterpreterCompletion() {
   List<InterpreterCompletion> completion = new LinkedList();
   for (InterpreterSetting intp : factory.getInterpreterSettings(note.getId())) {
     List<InterpreterInfo> intInfo = intp.getInterpreterInfos();
     if (intInfo.size() > 1) {
       for (InterpreterInfo info : intInfo) {
         String name = intp.getName() + "." + info.getName();
         completion.add(new InterpreterCompletion(name, name));
       }
     } else {
       completion.add(new InterpreterCompletion(intp.getName(), intp.getName()));
     }
   }
   return completion;
 }
Пример #14
0
  public void removeNote(String id) {
    Note note;
    synchronized (notes) {
      note = notes.remove(id);
    }

    // remove from all interpreter instance's angular object registry
    for (InterpreterSetting settings : replFactory.get()) {
      AngularObjectRegistry registry = settings.getInterpreterGroup().getAngularObjectRegistry();
      if (registry instanceof RemoteAngularObjectRegistry) {
        ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id);
      } else {
        registry.removeAll(id);
      }
    }

    try {
      note.unpersist();
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
Пример #15
0
  private void removeAllAngularObjectInParagraph(String user, String paragraphId) {
    angularObjects = new HashMap<>();

    List<InterpreterSetting> settings = factory.getInterpreterSettings(getId());
    if (settings == null || settings.size() == 0) {
      return;
    }

    for (InterpreterSetting setting : settings) {
      InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
      AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();

      if (registry instanceof RemoteAngularObjectRegistry) {
        // remove paragraph scope object
        ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, paragraphId);

        // remove app scope object
        List<ApplicationState> appStates = getParagraph(paragraphId).getAllApplicationStates();
        if (appStates != null) {
          for (ApplicationState app : appStates) {
            ((RemoteAngularObjectRegistry) registry)
                .removeAllAndNotifyRemoteProcess(id, app.getId());
          }
        }
      } else {
        registry.removeAll(id, paragraphId);

        // remove app scope object
        List<ApplicationState> appStates = getParagraph(paragraphId).getAllApplicationStates();
        if (appStates != null) {
          for (ApplicationState app : appStates) {
            registry.removeAll(id, app.getId());
          }
        }
      }
    }
  }
Пример #16
0
 public Interpreter getRepl(String name) {
   return factory.getInterpreter(user, note.getId(), name);
 }
Пример #17
0
  private Note loadNoteFromRepo(String id) {
    Note note = null;
    try {
      note = notebookRepo.get(id);
    } catch (IOException e) {
      logger.error("Failed to load " + id, e);
    }
    if (note == null) {
      return null;
    }

    // set NoteInterpreterLoader
    NoteInterpreterLoader noteInterpreterLoader = new NoteInterpreterLoader(replFactory);
    note.setReplLoader(noteInterpreterLoader);
    noteInterpreterLoader.setNoteId(note.id());

    // set JobListenerFactory
    note.setJobListenerFactory(jobListenerFactory);

    // set notebookRepo
    note.setNotebookRepo(notebookRepo);

    Map<String, SnapshotAngularObject> angularObjectSnapshot =
        new HashMap<String, SnapshotAngularObject>();

    // restore angular object --------------
    Date lastUpdatedDate = new Date(0);
    for (Paragraph p : note.getParagraphs()) {
      p.setNote(note);
      if (p.getDateFinished() != null && lastUpdatedDate.before(p.getDateFinished())) {
        lastUpdatedDate = p.getDateFinished();
      }
    }

    Map<String, List<AngularObject>> savedObjects = note.getAngularObjects();

    if (savedObjects != null) {
      for (String intpGroupName : savedObjects.keySet()) {
        List<AngularObject> objectList = savedObjects.get(intpGroupName);

        for (AngularObject savedObject : objectList) {
          SnapshotAngularObject snapshot = angularObjectSnapshot.get(savedObject.getName());
          if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) {
            angularObjectSnapshot.put(
                savedObject.getName(),
                new SnapshotAngularObject(intpGroupName, savedObject, lastUpdatedDate));
          }
        }
      }
    }

    synchronized (notes) {
      notes.put(note.id(), note);
      refreshCron(note.id());
    }

    for (String name : angularObjectSnapshot.keySet()) {
      SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
      List<InterpreterSetting> settings = replFactory.get();
      for (InterpreterSetting setting : settings) {
        InterpreterGroup intpGroup = setting.getInterpreterGroup();
        if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
          AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
          String noteId = snapshot.getAngularObject().getNoteId();
          // at this point, remote interpreter process is not created.
          // so does not make sense add it to the remote.
          //
          // therefore instead of addAndNotifyRemoteProcess(), need to use add()
          // that results add angularObject only in ZeppelinServer side not remoteProcessSide
          registry.add(name, snapshot.getAngularObject().get(), noteId);
        }
      }
    }
    return note;
  }
Пример #18
0
 private boolean noteHasInterpreters() {
   return !factory.getInterpreterSettings(note.getId()).isEmpty();
 }
Пример #19
0
  public List<Map<String, Object>> getJobListByUnixTime(
      boolean needsReload, long lastUpdateServerUnixTime, AuthenticationInfo subject) {
    final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";

    if (needsReload) {
      try {
        reloadAllNotes(subject);
      } catch (IOException e) {
        logger.error("Fail to reload notes from repository");
      }
    }

    List<Note> notes = getAllNotes();
    List<Map<String, Object>> notesInfo = new LinkedList<>();
    for (Note note : notes) {
      boolean isNotebookRunning = false;
      boolean isUpdateNotebook = false;
      long lastRunningUnixTime = 0;
      Map<String, Object> info = new HashMap<>();

      // set notebook ID
      info.put("notebookId", note.getId());

      // set notebook Name
      String notebookName = note.getName();
      if (notebookName != null && !notebookName.equals("")) {
        info.put("notebookName", note.getName());
      } else {
        info.put("notebookName", "Note " + note.getId());
      }

      // set notebook type ( cron or normal )
      if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD)
          && !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) {
        info.put("notebookType", "cron");
      } else {
        info.put("notebookType", "normal");
      }

      // set paragraphs
      List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
      for (Paragraph paragraph : note.getParagraphs()) {
        // check paragraph's status.
        if (paragraph.getStatus().isRunning()) {
          isNotebookRunning = true;
          isUpdateNotebook = true;
        }

        // get data for the job manager.
        Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph);
        lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph);

        // is update notebook for last server update time.
        if (lastRunningUnixTime > lastUpdateServerUnixTime) {
          isUpdateNotebook = true;
        }
        paragraphsInfo.add(paragraphItem);
      }

      // set interpreter bind type
      String interpreterGroupName = null;
      if (replFactory.getInterpreterSettings(note.getId()) != null
          && replFactory.getInterpreterSettings(note.getId()).size() >= 1) {
        interpreterGroupName = replFactory.getInterpreterSettings(note.getId()).get(0).getName();
      }

      // not update and not running -> pass
      if (!isUpdateNotebook && !isNotebookRunning) {
        continue;
      }

      // notebook json object root information.
      info.put("interpreter", interpreterGroupName);
      info.put("isRunningJob", isNotebookRunning);
      info.put("unixTimeLastRun", lastRunningUnixTime);
      info.put("paragraphs", paragraphsInfo);
      notesInfo.add(info);
    }

    return notesInfo;
  }
Пример #20
0
  @SuppressWarnings("rawtypes")
  private Note loadNoteFromRepo(String id, AuthenticationInfo subject) {
    Note note = null;
    try {
      note = notebookRepo.get(id, subject);
    } catch (IOException e) {
      logger.error("Failed to load " + id, e);
    }
    if (note == null) {
      return null;
    }

    // Manually inject ALL dependencies, as DI constructor was NOT used
    note.setIndex(this.notebookIndex);
    note.setCredentials(this.credentials);

    note.setInterpreterFactory(replFactory);

    note.setJobListenerFactory(jobListenerFactory);
    note.setNotebookRepo(notebookRepo);

    Map<String, SnapshotAngularObject> angularObjectSnapshot = new HashMap<>();

    // restore angular object --------------
    Date lastUpdatedDate = new Date(0);
    for (Paragraph p : note.getParagraphs()) {
      p.setNote(note);
      if (p.getDateFinished() != null && lastUpdatedDate.before(p.getDateFinished())) {
        lastUpdatedDate = p.getDateFinished();
      }
    }

    Map<String, List<AngularObject>> savedObjects = note.getAngularObjects();

    if (savedObjects != null) {
      for (String intpGroupName : savedObjects.keySet()) {
        List<AngularObject> objectList = savedObjects.get(intpGroupName);

        for (AngularObject object : objectList) {
          SnapshotAngularObject snapshot = angularObjectSnapshot.get(object.getName());
          if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) {
            angularObjectSnapshot.put(
                object.getName(),
                new SnapshotAngularObject(intpGroupName, object, lastUpdatedDate));
          }
        }
      }
    }

    note.setNoteEventListener(this);

    synchronized (notes) {
      notes.put(note.getId(), note);
      refreshCron(note.getId());
    }

    for (String name : angularObjectSnapshot.keySet()) {
      SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
      List<InterpreterSetting> settings = replFactory.get();
      for (InterpreterSetting setting : settings) {
        InterpreterGroup intpGroup = setting.getInterpreterGroup(note.getId());
        if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
          AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
          String noteId = snapshot.getAngularObject().getNoteId();
          String paragraphId = snapshot.getAngularObject().getParagraphId();
          // at this point, remote interpreter process is not created.
          // so does not make sense add it to the remote.
          //
          // therefore instead of addAndNotifyRemoteProcess(), need to use add()
          // that results add angularObject only in ZeppelinServer side not remoteProcessSide
          registry.add(name, snapshot.getAngularObject().get(), noteId, paragraphId);
        }
      }
    }

    return note;
  }
Пример #21
0
 private String getDefaultInterpreterName() {
   InterpreterSetting setting = factory.getDefaultInterpreterSetting(getId());
   return null != setting ? setting.getName() : StringUtils.EMPTY;
 }