public List<Map<String, Object>> getJobListBymNotebookId(String notebookID) {
final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";
long lastRunningUnixTime = 0;
boolean isNotebookRunning = false;
Note jobNote = getNote(notebookID);
List<Map<String, Object>> notesInfo = new LinkedList<>();
if (jobNote == null) {
return notesInfo;
}
Map<String, Object> info = new HashMap<>();
info.put("notebookId", jobNote.getId());
String notebookName = jobNote.getName();
if (notebookName != null && !notebookName.equals("")) {
info.put("notebookName", jobNote.getName());
} else {
info.put("notebookName", "Note " + jobNote.getId());
}
// set notebook type ( cron or normal )
if (jobNote.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD)
&& !jobNote.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) {
info.put("notebookType", "cron");
} else {
info.put("notebookType", "normal");
}
// set paragraphs
List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
for (Paragraph paragraph : jobNote.getParagraphs()) {
// check paragraph's status.
if (paragraph.getStatus().isRunning()) {
isNotebookRunning = true;
}
// get data for the job manager.
Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph);
lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph);
paragraphsInfo.add(paragraphItem);
}
// set interpreter bind type
String interpreterGroupName = null;
if (replFactory.getInterpreterSettings(jobNote.getId()) != null
&& replFactory.getInterpreterSettings(jobNote.getId()).size() >= 1) {
interpreterGroupName = replFactory.getInterpreterSettings(jobNote.getId()).get(0).getName();
}
// notebook json object root information.
info.put("interpreter", interpreterGroupName);
info.put("isRunningJob", isNotebookRunning);
info.put("unixTimeLastRun", lastRunningUnixTime);
info.put("paragraphs", paragraphsInfo);
notesInfo.add(info);
return notesInfo;
};
public void removeNote(String id, AuthenticationInfo subject) {
Note note;
synchronized (notes) {
note = notes.remove(id);
}
replFactory.removeNoteInterpreterSettingBinding(id);
notebookIndex.deleteIndexDocs(note);
notebookAuthorization.removeNote(id);
// remove from all interpreter instance's angular object registry
for (InterpreterSetting settings : replFactory.get()) {
AngularObjectRegistry registry = settings.getInterpreterGroup(id).getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
// remove paragraph scope object
for (Paragraph p : note.getParagraphs()) {
((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, p.getId());
// remove app scope object
List<ApplicationState> appStates = p.getAllApplicationStates();
if (appStates != null) {
for (ApplicationState app : appStates) {
((RemoteAngularObjectRegistry) registry)
.removeAllAndNotifyRemoteProcess(id, app.getId());
}
}
}
// remove notebook scope object
((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, null);
} else {
// remove paragraph scope object
for (Paragraph p : note.getParagraphs()) {
registry.removeAll(id, p.getId());
// remove app scope object
List<ApplicationState> appStates = p.getAllApplicationStates();
if (appStates != null) {
for (ApplicationState app : appStates) {
registry.removeAll(id, app.getId());
}
}
}
// remove notebook scope object
registry.removeAll(id, null);
}
}
ResourcePoolUtils.removeResourcesBelongsToNote(id);
fireNoteRemoveEvent(note);
try {
note.unpersist(subject);
} catch (IOException e) {
logger.error(e.toString(), e);
}
}
/**
* Run a single paragraph.
*
* @param paragraphId ID of paragraph
*/
public void run(String paragraphId) {
Paragraph p = getParagraph(paragraphId);
if (p.isBlankParagraph()) {
logger.info("skip to run blank paragraph. {}", p.getId());
return;
}
p.setListener(jobListenerFactory.getParagraphJobListener(this));
String requiredReplName = p.getRequiredReplName();
Interpreter intp = factory.getInterpreter(p.getUser(), getId(), requiredReplName);
if (intp == null) {
String intpExceptionMsg =
p.getJobName() + "'s Interpreter " + requiredReplName + " not found";
InterpreterException intpException = new InterpreterException(intpExceptionMsg);
InterpreterResult intpResult =
new InterpreterResult(InterpreterResult.Code.ERROR, intpException.getMessage());
p.setReturn(intpResult, intpException);
p.setStatus(Job.Status.ERROR);
throw intpException;
}
if (p.getConfig().get("enabled") == null || (Boolean) p.getConfig().get("enabled")) {
p.setAuthenticationInfo(p.getAuthenticationInfo());
intp.getScheduler().submit(p);
}
}
public void bindInterpretersToNote(String id, List<String> interpreterSettingIds)
throws IOException {
Note note = getNote(id);
if (note != null) {
List<InterpreterSetting> currentBindings = replFactory.getInterpreterSettings(id);
for (InterpreterSetting setting : currentBindings) {
if (!interpreterSettingIds.contains(setting.getId())) {
fireUnbindInterpreter(note, setting);
}
}
replFactory.setInterpreters(note.getId(), interpreterSettingIds);
// comment out while note.getNoteReplLoader().setInterpreters(...) do the same
// replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds);
}
}
/**
* Create new note.
*
* @return
* @throws IOException
*/
public Note createNote() throws IOException {
if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) {
return createNote(replFactory.getDefaultInterpreterSettingList());
} else {
return createNote(null);
}
}
public List<InterpreterSetting> getBindedInterpreterSettings(String id) {
Note note = getNote(id);
if (note != null) {
return replFactory.getInterpreterSettings(note.getId());
} else {
return new LinkedList<>();
}
}
private boolean isValidInterpreter(String replName) {
try {
return factory.getInterpreter(user, note.getId(), replName) != null;
} catch (InterpreterException e) {
// ignore this exception, it would be recaught when running paragraph.
return false;
}
}
public void bindInterpretersToNote(String id, List<String> interpreterSettingIds)
throws IOException {
Note note = getNote(id);
if (note != null) {
note.getNoteReplLoader().setInterpreters(interpreterSettingIds);
replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds);
}
}
private InterpreterSetting getInterpreterSettingById(String id) {
InterpreterSetting setting = null;
for (InterpreterSetting i : factory.getInterpreterSettings(note.getId())) {
if (id.startsWith(i.getId())) {
setting = i;
break;
}
}
return setting;
}
/**
* Create new note.
*
* @throws IOException
*/
public Note createNote(AuthenticationInfo subject) throws IOException {
Note note;
if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) {
note = createNote(replFactory.getDefaultInterpreterSettingList(), subject);
} else {
note = createNote(null, subject);
}
notebookIndex.addIndexDoc(note);
return note;
}
private InterpreterContext getInterpreterContext(InterpreterOutput output) {
AngularObjectRegistry registry = null;
ResourcePool resourcePool = null;
if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<>();
for (Paragraph p : note.getParagraphs()) {
runners.add(new ParagraphRunner(note, note.getId(), p.getId()));
}
final Paragraph self = this;
Credentials credentials = note.getCredentials();
if (authenticationInfo != null) {
UserCredentials userCredentials =
credentials.getUserCredentials(authenticationInfo.getUser());
authenticationInfo.setUserCredentials(userCredentials);
}
InterpreterContext interpreterContext =
new InterpreterContext(
note.getId(),
getId(),
getRequiredReplName(),
this.getTitle(),
this.getText(),
this.getAuthenticationInfo(),
this.getConfig(),
this.settings,
registry,
resourcePool,
runners,
output);
return interpreterContext;
}
private void snapshotAngularObjectRegistry(String user) {
angularObjects = new HashMap<>();
List<InterpreterSetting> settings = factory.getInterpreterSettings(getId());
if (settings == null || settings.size() == 0) {
return;
}
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
angularObjects.put(intpGroup.getId(), registry.getAllWithGlobal(id));
}
}
public List<InterpreterCompletion> getInterpreterCompletion() {
List<InterpreterCompletion> completion = new LinkedList();
for (InterpreterSetting intp : factory.getInterpreterSettings(note.getId())) {
List<InterpreterInfo> intInfo = intp.getInterpreterInfos();
if (intInfo.size() > 1) {
for (InterpreterInfo info : intInfo) {
String name = intp.getName() + "." + info.getName();
completion.add(new InterpreterCompletion(name, name));
}
} else {
completion.add(new InterpreterCompletion(intp.getName(), intp.getName()));
}
}
return completion;
}
public void removeNote(String id) {
Note note;
synchronized (notes) {
note = notes.remove(id);
}
// remove from all interpreter instance's angular object registry
for (InterpreterSetting settings : replFactory.get()) {
AngularObjectRegistry registry = settings.getInterpreterGroup().getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id);
} else {
registry.removeAll(id);
}
}
try {
note.unpersist();
} catch (IOException e) {
e.printStackTrace();
}
}
private void removeAllAngularObjectInParagraph(String user, String paragraphId) {
angularObjects = new HashMap<>();
List<InterpreterSetting> settings = factory.getInterpreterSettings(getId());
if (settings == null || settings.size() == 0) {
return;
}
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
// remove paragraph scope object
((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, paragraphId);
// remove app scope object
List<ApplicationState> appStates = getParagraph(paragraphId).getAllApplicationStates();
if (appStates != null) {
for (ApplicationState app : appStates) {
((RemoteAngularObjectRegistry) registry)
.removeAllAndNotifyRemoteProcess(id, app.getId());
}
}
} else {
registry.removeAll(id, paragraphId);
// remove app scope object
List<ApplicationState> appStates = getParagraph(paragraphId).getAllApplicationStates();
if (appStates != null) {
for (ApplicationState app : appStates) {
registry.removeAll(id, app.getId());
}
}
}
}
}
public Interpreter getRepl(String name) {
return factory.getInterpreter(user, note.getId(), name);
}
private Note loadNoteFromRepo(String id) {
Note note = null;
try {
note = notebookRepo.get(id);
} catch (IOException e) {
logger.error("Failed to load " + id, e);
}
if (note == null) {
return null;
}
// set NoteInterpreterLoader
NoteInterpreterLoader noteInterpreterLoader = new NoteInterpreterLoader(replFactory);
note.setReplLoader(noteInterpreterLoader);
noteInterpreterLoader.setNoteId(note.id());
// set JobListenerFactory
note.setJobListenerFactory(jobListenerFactory);
// set notebookRepo
note.setNotebookRepo(notebookRepo);
Map<String, SnapshotAngularObject> angularObjectSnapshot =
new HashMap<String, SnapshotAngularObject>();
// restore angular object --------------
Date lastUpdatedDate = new Date(0);
for (Paragraph p : note.getParagraphs()) {
p.setNote(note);
if (p.getDateFinished() != null && lastUpdatedDate.before(p.getDateFinished())) {
lastUpdatedDate = p.getDateFinished();
}
}
Map<String, List<AngularObject>> savedObjects = note.getAngularObjects();
if (savedObjects != null) {
for (String intpGroupName : savedObjects.keySet()) {
List<AngularObject> objectList = savedObjects.get(intpGroupName);
for (AngularObject savedObject : objectList) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(savedObject.getName());
if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) {
angularObjectSnapshot.put(
savedObject.getName(),
new SnapshotAngularObject(intpGroupName, savedObject, lastUpdatedDate));
}
}
}
}
synchronized (notes) {
notes.put(note.id(), note);
refreshCron(note.id());
}
for (String name : angularObjectSnapshot.keySet()) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
List<InterpreterSetting> settings = replFactory.get();
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup();
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
String noteId = snapshot.getAngularObject().getNoteId();
// at this point, remote interpreter process is not created.
// so does not make sense add it to the remote.
//
// therefore instead of addAndNotifyRemoteProcess(), need to use add()
// that results add angularObject only in ZeppelinServer side not remoteProcessSide
registry.add(name, snapshot.getAngularObject().get(), noteId);
}
}
}
return note;
}
private boolean noteHasInterpreters() {
return !factory.getInterpreterSettings(note.getId()).isEmpty();
}
public List<Map<String, Object>> getJobListByUnixTime(
boolean needsReload, long lastUpdateServerUnixTime, AuthenticationInfo subject) {
final String CRON_TYPE_NOTEBOOK_KEYWORD = "cron";
if (needsReload) {
try {
reloadAllNotes(subject);
} catch (IOException e) {
logger.error("Fail to reload notes from repository");
}
}
List<Note> notes = getAllNotes();
List<Map<String, Object>> notesInfo = new LinkedList<>();
for (Note note : notes) {
boolean isNotebookRunning = false;
boolean isUpdateNotebook = false;
long lastRunningUnixTime = 0;
Map<String, Object> info = new HashMap<>();
// set notebook ID
info.put("notebookId", note.getId());
// set notebook Name
String notebookName = note.getName();
if (notebookName != null && !notebookName.equals("")) {
info.put("notebookName", note.getName());
} else {
info.put("notebookName", "Note " + note.getId());
}
// set notebook type ( cron or normal )
if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD)
&& !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) {
info.put("notebookType", "cron");
} else {
info.put("notebookType", "normal");
}
// set paragraphs
List<Map<String, Object>> paragraphsInfo = new LinkedList<>();
for (Paragraph paragraph : note.getParagraphs()) {
// check paragraph's status.
if (paragraph.getStatus().isRunning()) {
isNotebookRunning = true;
isUpdateNotebook = true;
}
// get data for the job manager.
Map<String, Object> paragraphItem = getParagraphForJobManagerItem(paragraph);
lastRunningUnixTime = getUnixTimeLastRunParagraph(paragraph);
// is update notebook for last server update time.
if (lastRunningUnixTime > lastUpdateServerUnixTime) {
isUpdateNotebook = true;
}
paragraphsInfo.add(paragraphItem);
}
// set interpreter bind type
String interpreterGroupName = null;
if (replFactory.getInterpreterSettings(note.getId()) != null
&& replFactory.getInterpreterSettings(note.getId()).size() >= 1) {
interpreterGroupName = replFactory.getInterpreterSettings(note.getId()).get(0).getName();
}
// not update and not running -> pass
if (!isUpdateNotebook && !isNotebookRunning) {
continue;
}
// notebook json object root information.
info.put("interpreter", interpreterGroupName);
info.put("isRunningJob", isNotebookRunning);
info.put("unixTimeLastRun", lastRunningUnixTime);
info.put("paragraphs", paragraphsInfo);
notesInfo.add(info);
}
return notesInfo;
}
@SuppressWarnings("rawtypes")
private Note loadNoteFromRepo(String id, AuthenticationInfo subject) {
Note note = null;
try {
note = notebookRepo.get(id, subject);
} catch (IOException e) {
logger.error("Failed to load " + id, e);
}
if (note == null) {
return null;
}
// Manually inject ALL dependencies, as DI constructor was NOT used
note.setIndex(this.notebookIndex);
note.setCredentials(this.credentials);
note.setInterpreterFactory(replFactory);
note.setJobListenerFactory(jobListenerFactory);
note.setNotebookRepo(notebookRepo);
Map<String, SnapshotAngularObject> angularObjectSnapshot = new HashMap<>();
// restore angular object --------------
Date lastUpdatedDate = new Date(0);
for (Paragraph p : note.getParagraphs()) {
p.setNote(note);
if (p.getDateFinished() != null && lastUpdatedDate.before(p.getDateFinished())) {
lastUpdatedDate = p.getDateFinished();
}
}
Map<String, List<AngularObject>> savedObjects = note.getAngularObjects();
if (savedObjects != null) {
for (String intpGroupName : savedObjects.keySet()) {
List<AngularObject> objectList = savedObjects.get(intpGroupName);
for (AngularObject object : objectList) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(object.getName());
if (snapshot == null || snapshot.getLastUpdate().before(lastUpdatedDate)) {
angularObjectSnapshot.put(
object.getName(),
new SnapshotAngularObject(intpGroupName, object, lastUpdatedDate));
}
}
}
}
note.setNoteEventListener(this);
synchronized (notes) {
notes.put(note.getId(), note);
refreshCron(note.getId());
}
for (String name : angularObjectSnapshot.keySet()) {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
List<InterpreterSetting> settings = replFactory.get();
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(note.getId());
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
String noteId = snapshot.getAngularObject().getNoteId();
String paragraphId = snapshot.getAngularObject().getParagraphId();
// at this point, remote interpreter process is not created.
// so does not make sense add it to the remote.
//
// therefore instead of addAndNotifyRemoteProcess(), need to use add()
// that results add angularObject only in ZeppelinServer side not remoteProcessSide
registry.add(name, snapshot.getAngularObject().get(), noteId, paragraphId);
}
}
}
return note;
}
private String getDefaultInterpreterName() {
InterpreterSetting setting = factory.getDefaultInterpreterSetting(getId());
return null != setting ? setting.getName() : StringUtils.EMPTY;
}