@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
InterpreterResult result;
if ("getId".equals(st)) {
// get unique id of this interpreter instance
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "" + this.hashCode());
} else if (st.startsWith("sleep")) {
try {
Thread.sleep(Integer.parseInt(st.split(" ")[1]));
} catch (InterruptedException e) {
// nothing to do
}
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl2: " + st);
} else {
result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl2: " + st);
}
if (context.getResourcePool() != null) {
context
.getResourcePool()
.put(context.getNoteId(), context.getParagraphId(), "result", result);
}
return result;
}
@Override
public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
try {
if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
try {
userSessionMap.put(
interpreterContext.getAuthenticationInfo().getUser(),
livyHelper.createSession(interpreterContext, "pyspark"));
} catch (Exception e) {
LOGGER.error("Exception in LivyPySparkInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
return livyHelper.interpret(line, interpreterContext, userSessionMap);
} catch (Exception e) {
LOGGER.error("Exception in LivyPySparkInterpreter while interpret ", e);
return new InterpreterResult(
InterpreterResult.Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
protected Object jobRun() throws Throwable {
String replName = getRequiredReplName();
Interpreter repl = getRepl(replName);
logger().info("run paragraph {} using {} " + repl, getId(), replName);
if (repl == null) {
logger().error("Can not find interpreter name " + repl);
throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
}
String script = getScriptBody();
// inject form
if (repl.getFormType() == FormType.NATIVE) {
settings.clear();
} else if (repl.getFormType() == FormType.SIMPLE) {
String scriptBody = getScriptBody();
Map<String, Input> inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built
// from script body
settings.setForms(inputs);
script = Input.getSimpleQuery(settings.getParams(), scriptBody);
}
logger().debug("RUN : " + script);
try {
InterpreterContext context = getInterpreterContext();
InterpreterContext.set(context);
InterpreterResult ret = repl.interpret(script, context);
if (Code.KEEP_PREVIOUS_RESULT == ret.code()) {
return getReturn();
}
return ret;
} finally {
InterpreterContext.remove();
}
}
@Override
protected Object jobRun() throws Throwable {
String replName = getRequiredReplName();
Interpreter repl = getRepl(replName);
logger().info("run paragraph {} using {} " + repl, getId(), replName);
if (repl == null) {
logger().error("Can not find interpreter name " + repl);
throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
}
String script = getScriptBody();
// inject form
if (repl.getFormType() == FormType.NATIVE) {
settings.clear();
} else if (repl.getFormType() == FormType.SIMPLE) {
String scriptBody = getScriptBody();
Map<String, Input> inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built
// from script body
settings.setForms(inputs);
script = Input.getSimpleQuery(settings.getParams(), scriptBody);
}
logger().debug("RUN : " + script);
try {
InterpreterContext context = getInterpreterContext();
InterpreterContext.set(context);
InterpreterResult ret = repl.interpret(script, context);
if (Code.KEEP_PREVIOUS_RESULT == ret.code()) {
return getReturn();
}
String message = "";
context.out.flush();
InterpreterResult.Type outputType = context.out.getType();
byte[] interpreterOutput = context.out.toByteArray();
context.out.clear();
if (interpreterOutput != null && interpreterOutput.length > 0) {
message = new String(interpreterOutput);
}
if (message.isEmpty()) {
return ret;
} else {
String interpreterResultMessage = ret.message();
if (interpreterResultMessage != null && !interpreterResultMessage.isEmpty()) {
message += interpreterResultMessage;
return new InterpreterResult(ret.code(), ret.type(), message);
} else {
return new InterpreterResult(ret.code(), outputType, message);
}
}
} finally {
InterpreterContext.remove();
}
}
@Override
public void run(ApplicationArgument arg, InterpreterContext context)
throws ApplicationException, IOException {
// get TableData
TableData tableData =
(TableData)
context.getResourcePool().get(arg.getResource().location(), arg.getResource().name());
if (tableData == null) {
context.out.write("No table data found");
return;
}
if (tableData.getColumnDef().length < 2) {
context.out.write("Minimum 2 columns are required. xaxis, yaxis");
return;
}
// dbar elementId
String elementId = "dbar" + context.getParagraphId();
// create element
context.out.write("<div id=\"" + elementId + "\" style=\"height:400px;\"><svg></svg></div>");
// include library
context.out.write("<script>\n");
// write data
int numRows = tableData.length();
String jsonData = "[ { key: \"data\", values: [";
for (int i = 0; i < numRows; i++) {
try {
jsonData +=
"{ label:\"" + tableData.getData(i, 0) + "\", value:" + tableData.getData(i, 1) + "}";
if (i != numRows) {
jsonData += ",";
}
} catch (Exception e) {
continue;
}
}
jsonData += "]}]";
context.out.write("nv.addGraph(function() {\n");
context.out.write("var elementId = \"" + elementId + "\";");
context.out.write("var data = " + jsonData + ";");
context.out.writeResource("dbar/draw.js");
context.out.write("});\n");
context.out.write("</script>");
}
@Test
public void testCreateDataFrame() {
if (getSparkVersionNumber() >= 13) {
repl.interpret("case class Person(name:String, age:Int)\n", context);
repl.interpret(
"val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n",
context);
repl.interpret("people.toDF.count", context);
assertEquals(
new Long(4),
context
.getResourcePool()
.get(
context.getNoteId(),
context.getParagraphId(),
WellKnownResourceName.ZeppelinReplResult.toString())
.get());
}
}
@Override
public void cancel(InterpreterContext context) {
livyHelper.cancelHTTP(context.getParagraphId());
}
@Override
protected Object jobRun() throws Throwable {
String replName = getRequiredReplName();
Interpreter repl = getRepl(replName);
logger.info("run paragraph {} using {} " + repl, getId(), replName);
if (repl == null) {
logger.error("Can not find interpreter name " + repl);
throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
}
InterpreterSetting intp = getInterpreterSettingById(repl.getInterpreterGroup().getId());
while (intp.getStatus()
.equals(
org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) {
Thread.sleep(200);
}
if (this.noteHasUser() && this.noteHasInterpreters()) {
if (intp != null
&& interpreterHasUser(intp)
&& isUserAuthorizedToAccessInterpreter(intp.getOption()) == false) {
logger.error("{} has no permission for {} ", authenticationInfo.getUser(), repl);
return new InterpreterResult(
Code.ERROR,
authenticationInfo.getUser() + " has no permission for " + getRequiredReplName());
}
}
String script = getScriptBody();
// inject form
if (repl.getFormType() == FormType.NATIVE) {
settings.clear();
} else if (repl.getFormType() == FormType.SIMPLE) {
String scriptBody = getScriptBody();
Map<String, Input> inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built
// from script body
final AngularObjectRegistry angularRegistry =
repl.getInterpreterGroup().getAngularObjectRegistry();
scriptBody = extractVariablesFromAngularRegistry(scriptBody, inputs, angularRegistry);
settings.setForms(inputs);
script = Input.getSimpleQuery(settings.getParams(), scriptBody);
}
logger.debug("RUN : " + script);
try {
InterpreterContext context = getInterpreterContext();
InterpreterContext.set(context);
InterpreterResult ret = repl.interpret(script, context);
if (Code.KEEP_PREVIOUS_RESULT == ret.code()) {
return getReturn();
}
context.out.flush();
List<InterpreterResultMessage> resultMessages = context.out.toInterpreterResultMessage();
resultMessages.addAll(ret.message());
return new InterpreterResult(ret.code(), resultMessages);
} finally {
InterpreterContext.remove();
}
}