@Override
protected Object jobRun() throws Throwable {
String replName = getRequiredReplName();
Interpreter repl = getRepl(replName);
logger().info("run paragraph {} using {} " + repl, getId(), replName);
if (repl == null) {
logger().error("Can not find interpreter name " + repl);
throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
}
String script = getScriptBody();
// inject form
if (repl.getFormType() == FormType.NATIVE) {
settings.clear();
} else if (repl.getFormType() == FormType.SIMPLE) {
String scriptBody = getScriptBody();
Map<String, Input> inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built
// from script body
settings.setForms(inputs);
script = Input.getSimpleQuery(settings.getParams(), scriptBody);
}
logger().info("RUN : " + script);
InterpreterResult ret = repl.interpret(script, getInterpreterContext());
return ret;
}
@ZeppelinApi
public Interpreter getInterpreterInTheSameSessionByClassName(String className) {
synchronized (interpreterGroup) {
for (List<Interpreter> interpreters : interpreterGroup.values()) {
boolean belongsToSameNoteGroup = false;
Interpreter interpreterFound = null;
for (Interpreter intp : interpreters) {
if (intp.getClassName().equals(className)) {
interpreterFound = intp;
}
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
if (this == p) {
belongsToSameNoteGroup = true;
}
}
if (belongsToSameNoteGroup) {
return interpreterFound;
}
}
}
return null;
}
/**
* Run a single paragraph.
*
* @param paragraphId ID of paragraph
*/
public void run(String paragraphId) {
Paragraph p = getParagraph(paragraphId);
if (p.isBlankParagraph()) {
logger.info("skip to run blank paragraph. {}", p.getId());
return;
}
p.setListener(jobListenerFactory.getParagraphJobListener(this));
String requiredReplName = p.getRequiredReplName();
Interpreter intp = factory.getInterpreter(p.getUser(), getId(), requiredReplName);
if (intp == null) {
String intpExceptionMsg =
p.getJobName() + "'s Interpreter " + requiredReplName + " not found";
InterpreterException intpException = new InterpreterException(intpExceptionMsg);
InterpreterResult intpResult =
new InterpreterResult(InterpreterResult.Code.ERROR, intpException.getMessage());
p.setReturn(intpResult, intpException);
p.setStatus(Job.Status.ERROR);
throw intpException;
}
if (p.getConfig().get("enabled") == null || (Boolean) p.getConfig().get("enabled")) {
p.setAuthenticationInfo(p.getAuthenticationInfo());
intp.getScheduler().submit(p);
}
}
@Override
protected Object jobRun() throws Throwable {
String replName = getRequiredReplName();
Interpreter repl = getRepl(replName);
logger().info("run paragraph {} using {} " + repl, getId(), replName);
if (repl == null) {
logger().error("Can not find interpreter name " + repl);
throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
}
String script = getScriptBody();
// inject form
if (repl.getFormType() == FormType.NATIVE) {
settings.clear();
} else if (repl.getFormType() == FormType.SIMPLE) {
String scriptBody = getScriptBody();
Map<String, Input> inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built
// from script body
settings.setForms(inputs);
script = Input.getSimpleQuery(settings.getParams(), scriptBody);
}
logger().debug("RUN : " + script);
try {
InterpreterContext context = getInterpreterContext();
InterpreterContext.set(context);
InterpreterResult ret = repl.interpret(script, context);
if (Code.KEEP_PREVIOUS_RESULT == ret.code()) {
return getReturn();
}
String message = "";
context.out.flush();
InterpreterResult.Type outputType = context.out.getType();
byte[] interpreterOutput = context.out.toByteArray();
context.out.clear();
if (interpreterOutput != null && interpreterOutput.length > 0) {
message = new String(interpreterOutput);
}
if (message.isEmpty()) {
return ret;
} else {
String interpreterResultMessage = ret.message();
if (interpreterResultMessage != null && !interpreterResultMessage.isEmpty()) {
message += interpreterResultMessage;
return new InterpreterResult(ret.code(), ret.type(), message);
} else {
return new InterpreterResult(ret.code(), outputType, message);
}
}
} finally {
InterpreterContext.remove();
}
}
@Override
public int progress() {
String replName = getRequiredReplName();
Interpreter repl = getRepl(replName);
if (repl != null) {
return repl.getProgress(getInterpreterContext());
} else {
return 0;
}
}
/**
* Run a single paragraph.
*
* @param paragraphId
*/
public void run(String paragraphId) {
Paragraph p = getParagraph(paragraphId);
p.setNoteReplLoader(replLoader);
p.setListener(jobListenerFactory.getParagraphJobListener(this));
Interpreter intp = replLoader.get(p.getRequiredReplName());
if (intp == null) {
throw new InterpreterException("Interpreter " + p.getRequiredReplName() + " not found");
}
intp.getScheduler().submit(p);
}
/**
* Run all paragraphs sequentially.
*
* @param jobListener
*/
public void runAll() {
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
p.setNoteReplLoader(replLoader);
p.setListener(jobListenerFactory.getParagraphJobListener(this));
Interpreter intp = replLoader.get(p.getRequiredReplName());
intp.getScheduler().submit(p);
}
}
}
@Override
protected boolean jobAbort() {
Interpreter repl = getRepl(getRequiredReplName());
Job job = repl.getScheduler().removeFromWaitingQueue(getId());
if (job != null) {
job.setStatus(Status.ABORT);
} else {
repl.cancel(getInterpreterContext());
}
return true;
}
/**
* Get combined property of all interpreters in this group
*
* @return
*/
public Properties getProperty() {
Properties p = new Properties();
for (List<Interpreter> intpGroupForASession : this.values()) {
for (Interpreter intp : intpGroupForASession) {
p.putAll(intp.getProperty());
}
// it's okay to break here while every List<Interpreters> will have the same property set
break;
}
return p;
}
public List<String> completion(String buffer, int cursor) {
String replName = getRequiredReplName(buffer);
if (replName != null) {
cursor -= replName.length() + 1;
}
String body = getScriptBody(buffer);
Interpreter repl = getRepl(replName);
if (repl == null) {
return null;
}
return repl.completion(body, cursor);
}
static {
Interpreter.register(
"pyspark",
"livy",
LivyPySparkInterpreter.class.getName(),
new InterpreterPropertyBuilder().build());
}
static {
Interpreter.register(
"tachyon",
"tachyon",
TachyonInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(TACHYON_MASTER_HOSTNAME, "localhost", "Tachyon master hostname")
.add(TACHYON_MASTER_PORT, "19998", "Tachyon master port")
.build());
}
@Override
protected boolean jobAbort() {
Interpreter repl = getRepl(getRequiredReplName());
if (repl == null) {
// when interpreters are already destroyed
return true;
}
Scheduler scheduler = repl.getScheduler();
if (scheduler == null) {
return true;
}
Job job = scheduler.removeFromWaitingQueue(getId());
if (job != null) {
job.setStatus(Status.ABORT);
} else {
repl.cancel(getInterpreterContext());
}
return true;
}
static {
Interpreter.register(
"flink",
"flink",
FlinkInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(
"host",
"local",
"host name of running JobManager. 'local' runs flink in local mode")
.add("port", "6123", "port of running JobManager")
.build());
}
public List<InterpreterCompletion> completion(String buffer, int cursor) {
String lines[] = buffer.split(System.getProperty("line.separator"));
if (lines.length > 0 && lines[0].startsWith("%") && cursor <= lines[0].trim().length()) {
int idx = lines[0].indexOf(' ');
if (idx < 0 || (idx > 0 && cursor <= idx)) {
return getInterpreterCompletion();
}
}
String replName = getRequiredReplName(buffer);
if (replName != null && cursor > replName.length()) {
cursor -= replName.length() + 1;
}
String body = getScriptBody(buffer);
Interpreter repl = getRepl(replName);
if (repl == null) {
return null;
}
List completion = repl.completion(body, cursor);
return completion;
}
@ZeppelinApi
public Properties getProperty() {
Properties p = new Properties();
p.putAll(property);
RegisteredInterpreter registeredInterpreter =
Interpreter.findRegisteredInterpreterByClassName(getClassName());
if (null != registeredInterpreter) {
Map<String, InterpreterProperty> defaultProperties = registeredInterpreter.getProperties();
for (String k : defaultProperties.keySet()) {
if (!p.containsKey(k)) {
String value = defaultProperties.get(k).getValue();
if (value != null) {
p.put(k, defaultProperties.get(k).getValue());
}
}
}
}
return p;
}
static {
Interpreter.register("md", Markdown.class.getName());
}
@Override
protected boolean jobAbort() {
Interpreter repl = getRepl(getRequiredReplName());
repl.cancel(getInterpreterContext());
return true;
}
@Override
protected Object jobRun() throws Throwable {
String replName = getRequiredReplName();
Interpreter repl = getRepl(replName);
logger.info("run paragraph {} using {} " + repl, getId(), replName);
if (repl == null) {
logger.error("Can not find interpreter name " + repl);
throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
}
InterpreterSetting intp = getInterpreterSettingById(repl.getInterpreterGroup().getId());
while (intp.getStatus()
.equals(
org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) {
Thread.sleep(200);
}
if (this.noteHasUser() && this.noteHasInterpreters()) {
if (intp != null
&& interpreterHasUser(intp)
&& isUserAuthorizedToAccessInterpreter(intp.getOption()) == false) {
logger.error("{} has no permission for {} ", authenticationInfo.getUser(), repl);
return new InterpreterResult(
Code.ERROR,
authenticationInfo.getUser() + " has no permission for " + getRequiredReplName());
}
}
String script = getScriptBody();
// inject form
if (repl.getFormType() == FormType.NATIVE) {
settings.clear();
} else if (repl.getFormType() == FormType.SIMPLE) {
String scriptBody = getScriptBody();
Map<String, Input> inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built
// from script body
final AngularObjectRegistry angularRegistry =
repl.getInterpreterGroup().getAngularObjectRegistry();
scriptBody = extractVariablesFromAngularRegistry(scriptBody, inputs, angularRegistry);
settings.setForms(inputs);
script = Input.getSimpleQuery(settings.getParams(), scriptBody);
}
logger.debug("RUN : " + script);
try {
InterpreterContext context = getInterpreterContext();
InterpreterContext.set(context);
InterpreterResult ret = repl.interpret(script, context);
if (Code.KEEP_PREVIOUS_RESULT == ret.code()) {
return getReturn();
}
context.out.flush();
List<InterpreterResultMessage> resultMessages = context.out.toInterpreterResultMessage();
resultMessages.addAll(ret.message());
return new InterpreterResult(ret.code(), resultMessages);
} finally {
InterpreterContext.remove();
}
}