@Override
public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
try {
if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
try {
userSessionMap.put(
interpreterContext.getAuthenticationInfo().getUser(),
livyHelper.createSession(interpreterContext, "pyspark"));
} catch (Exception e) {
LOGGER.error("Exception in LivyPySparkInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
return livyHelper.interpret(line, interpreterContext, userSessionMap);
} catch (Exception e) {
LOGGER.error("Exception in LivyPySparkInterpreter while interpret ", e);
return new InterpreterResult(
InterpreterResult.Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
protected Object jobRun() throws Throwable {
String replName = getRequiredReplName();
Interpreter repl = getRepl(replName);
logger().info("run paragraph {} using {} " + repl, getId(), replName);
if (repl == null) {
logger().error("Can not find interpreter name " + repl);
throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
}
String script = getScriptBody();
// inject form
if (repl.getFormType() == FormType.NATIVE) {
settings.clear();
} else if (repl.getFormType() == FormType.SIMPLE) {
String scriptBody = getScriptBody();
Map<String, Input> inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built
// from script body
settings.setForms(inputs);
script = Input.getSimpleQuery(settings.getParams(), scriptBody);
}
logger().debug("RUN : " + script);
try {
InterpreterContext context = getInterpreterContext();
InterpreterContext.set(context);
InterpreterResult ret = repl.interpret(script, context);
if (Code.KEEP_PREVIOUS_RESULT == ret.code()) {
return getReturn();
}
return ret;
} finally {
InterpreterContext.remove();
}
}
@Override
protected Object jobRun() throws Throwable {
String replName = getRequiredReplName();
Interpreter repl = getRepl(replName);
logger().info("run paragraph {} using {} " + repl, getId(), replName);
if (repl == null) {
logger().error("Can not find interpreter name " + repl);
throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
}
String script = getScriptBody();
// inject form
if (repl.getFormType() == FormType.NATIVE) {
settings.clear();
} else if (repl.getFormType() == FormType.SIMPLE) {
String scriptBody = getScriptBody();
Map<String, Input> inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built
// from script body
settings.setForms(inputs);
script = Input.getSimpleQuery(settings.getParams(), scriptBody);
}
logger().debug("RUN : " + script);
try {
InterpreterContext context = getInterpreterContext();
InterpreterContext.set(context);
InterpreterResult ret = repl.interpret(script, context);
if (Code.KEEP_PREVIOUS_RESULT == ret.code()) {
return getReturn();
}
String message = "";
context.out.flush();
InterpreterResult.Type outputType = context.out.getType();
byte[] interpreterOutput = context.out.toByteArray();
context.out.clear();
if (interpreterOutput != null && interpreterOutput.length > 0) {
message = new String(interpreterOutput);
}
if (message.isEmpty()) {
return ret;
} else {
String interpreterResultMessage = ret.message();
if (interpreterResultMessage != null && !interpreterResultMessage.isEmpty()) {
message += interpreterResultMessage;
return new InterpreterResult(ret.code(), ret.type(), message);
} else {
return new InterpreterResult(ret.code(), outputType, message);
}
}
} finally {
InterpreterContext.remove();
}
}
public static IRubyObject INTERPRET_METHOD(
ThreadContext context,
CFG cfg,
InterpreterContext interp,
String name,
RubyModule implClass,
boolean isTraceable) {
Ruby runtime = interp.getRuntime();
boolean syntheticMethod = name == null || name.equals("");
try {
String className = implClass.getName();
if (!syntheticMethod)
ThreadContext.pushBacktrace(context, className, name, context.getFile(), context.getLine());
if (isTraceable) methodPreTrace(runtime, context, name, implClass);
return interpret(context, cfg, interp);
} finally {
if (isTraceable) {
try {
methodPostTrace(runtime, context, name, implClass);
} finally {
if (!syntheticMethod) ThreadContext.popBacktrace(context);
}
} else {
if (!syntheticMethod) ThreadContext.popBacktrace(context);
}
}
}
@Test
public void testCreateDataFrame() {
if (getSparkVersionNumber() >= 13) {
repl.interpret("case class Person(name:String, age:Int)\n", context);
repl.interpret(
"val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n",
context);
repl.interpret("people.toDF.count", context);
assertEquals(
new Long(4),
context
.getResourcePool()
.get(
context.getNoteId(),
context.getParagraphId(),
WellKnownResourceName.ZeppelinReplResult.toString())
.get());
}
}
@Override
public String interpret(InterpreterContext ic) {
return ic.getHexadecimalFormat(i);
}
public static IRubyObject interpret(ThreadContext context, CFG cfg, InterpreterContext interp) {
Ruby runtime = context.getRuntime();
boolean inClosure = (cfg.getScope() instanceof IRClosure);
boolean passThroughBreak = false;
try {
interp.setMethodExitLabel(
cfg.getExitBB().getLabel()); // used by return and break instructions!
Instr[] instrs = cfg.prepareForInterpretation();
int n = instrs.length;
int ipc = 0;
Instr lastInstr = null;
while (ipc < n) {
interpInstrsCount++;
lastInstr = instrs[ipc];
if (isDebug()) LOG.debug("I: {}", lastInstr);
try {
Label jumpTarget = lastInstr.interpret(interp);
ipc = (jumpTarget == null) ? ipc + 1 : jumpTarget.getTargetPC();
}
// SSS FIXME: This only catches Ruby exceptions
// What about Java exceptions?
catch (org.jruby.exceptions.RaiseException re) {
ipc = cfg.getRescuerPC(lastInstr);
if (ipc == -1) throw re; // No one rescued exception, pass it on!
interp.setException(re.getException());
}
}
// If a closure, and lastInstr was a return, have to return from the nearest method!
IRubyObject rv = (IRubyObject) interp.getReturnValue();
if (lastInstr instanceof ReturnInstr && inClosure && !interp.inLambda()) {
throw new IRReturnJump(((ReturnInstr) lastInstr).methodToReturnFrom, rv);
}
// If a closure, and lastInstr was a break, have to return from the nearest closure!
else if (lastInstr instanceof BREAK_Instr) {
if (!inClosure) throw runtime.newLocalJumpError(Reason.BREAK, rv, "unexpected break");
passThroughBreak = true;
RuntimeHelpers.breakJump(context, rv);
}
return rv;
} catch (JumpException.BreakJump bj) {
if (passThroughBreak) throw bj;
return (IRubyObject) bj.getValue();
} catch (IRReturnJump rj) {
// - If we are in a lambda, stop propagating
// - If not in a lambda
// - if in a closure, pass it along
// - if not in a closure, we got this return jump from a closure further up the call stack.
// So, continue popping the call stack till we get to the right method
if (!interp.inLambda() && (inClosure || (rj.methodToReturnFrom != cfg.getScope())))
throw rj; // pass it along
return (IRubyObject) rj.returnValue;
} finally {
if (interp.getFrame() != null) {
context.popFrame();
interp.setFrame(null);
}
if (interp.hasAllocatedDynamicScope()) context.postMethodScopeOnly();
}
}
public IRubyObject interpret(
ThreadContext context,
IRubyObject self,
InterpreterContext interpreterContext,
RubyModule implClass,
String name,
IRubyObject[] args,
Block block,
Block.Type blockType) {
Instr[] instrs = interpreterContext.getInstructions();
Object[] temp = interpreterContext.allocateTemporaryVariables();
int n = instrs.length;
int ipc = 0;
Object exception = null;
if (interpreterContext.receivesKeywordArguments())
IRRuntimeHelpers.frobnicateKwargsArgument(
context, interpreterContext.getRequiredArgsCount(), args);
StaticScope currScope = interpreterContext.getStaticScope();
DynamicScope currDynScope = context.getCurrentScope();
boolean acceptsKeywordArgument = interpreterContext.receivesKeywordArguments();
Stack<Integer> rescuePCs = null;
// Init profiling this scope
boolean debug = IRRuntimeHelpers.isDebug();
boolean profile = IRRuntimeHelpers.inProfileMode();
Integer scopeVersion = profile ? Profiler.initProfiling(interpreterContext.getScope()) : 0;
// Enter the looooop!
while (ipc < n) {
Instr instr = instrs[ipc];
Operation operation = instr.getOperation();
if (debug) {
Interpreter.LOG.info("I: {" + ipc + "} ", instr + "; <#RPCs=" + rescuePCs.size() + ">");
Interpreter.interpInstrsCount++;
} else if (profile) {
Profiler.instrTick(operation);
Interpreter.interpInstrsCount++;
}
ipc++;
try {
switch (operation.opClass) {
case ARG_OP:
receiveArg(
context,
instr,
operation,
args,
acceptsKeywordArgument,
currDynScope,
temp,
exception,
block);
break;
case CALL_OP:
if (profile)
Profiler.updateCallSite(instr, interpreterContext.getScope(), scopeVersion);
processCall(context, instr, operation, currDynScope, currScope, temp, self);
break;
case RET_OP:
return processReturnOp(
context, instr, operation, currDynScope, temp, self, blockType, currScope);
case BRANCH_OP:
switch (operation) {
case JUMP:
JumpInstr jump = ((JumpInstr) instr);
if (jump.exitsExcRegion()) {
rescuePCs.pop();
}
ipc = jump.getJumpTarget().getTargetPC();
break;
default:
ipc =
instr.interpretAndGetNewIPC(context, currDynScope, currScope, self, temp, ipc);
break;
}
break;
case BOOK_KEEPING_OP:
switch (operation) {
case PUSH_BINDING:
// IMPORTANT: Preserve this update of currDynScope.
// This affects execution of all instructions in this scope
// which will now use the updated value of currDynScope.
currDynScope = interpreterContext.newDynamicScope(context);
context.pushScope(currDynScope);
case EXC_REGION_START:
if (rescuePCs == null) rescuePCs = new Stack<>();
rescuePCs.push(
((ExceptionRegionStartMarkerInstr) instr)
.getFirstRescueBlockLabel()
.getTargetPC());
break;
case EXC_REGION_END:
rescuePCs.pop();
break;
default:
processBookKeepingOp(
context, instr, operation, name, args, self, block, blockType, implClass);
}
break;
case OTHER_OP:
processOtherOp(
context, instr, operation, currDynScope, currScope, temp, self, blockType);
break;
}
} catch (Throwable t) {
if (debug) extractToMethodToAvoidC2Crash(instr, t);
if (rescuePCs == null
|| rescuePCs.empty()
|| (t instanceof IRBreakJump && instr instanceof BreakInstr)
|| (t instanceof IRReturnJump && instr instanceof NonlocalReturnInstr)) {
ipc = -1;
} else {
ipc = rescuePCs.pop();
}
if (debug) {
Interpreter.LOG.info(
"in : "
+ interpreterContext.getScope()
+ ", caught Java throwable: "
+ t
+ "; excepting instr: "
+ instr);
Interpreter.LOG.info("ipc for rescuer: " + ipc);
}
if (ipc == -1) {
Helpers.throwException(t);
} else {
exception = t;
}
}
}
// Control should never get here!
throw context.runtime.newRuntimeError("BUG: interpreter fell through to end unexpectedly");
}
@Override
public void cancel(InterpreterContext context) {
livyHelper.cancelHTTP(context.getParagraphId());
}
@Override
protected Object jobRun() throws Throwable {
String replName = getRequiredReplName();
Interpreter repl = getRepl(replName);
logger.info("run paragraph {} using {} " + repl, getId(), replName);
if (repl == null) {
logger.error("Can not find interpreter name " + repl);
throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
}
InterpreterSetting intp = getInterpreterSettingById(repl.getInterpreterGroup().getId());
while (intp.getStatus()
.equals(
org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) {
Thread.sleep(200);
}
if (this.noteHasUser() && this.noteHasInterpreters()) {
if (intp != null
&& interpreterHasUser(intp)
&& isUserAuthorizedToAccessInterpreter(intp.getOption()) == false) {
logger.error("{} has no permission for {} ", authenticationInfo.getUser(), repl);
return new InterpreterResult(
Code.ERROR,
authenticationInfo.getUser() + " has no permission for " + getRequiredReplName());
}
}
String script = getScriptBody();
// inject form
if (repl.getFormType() == FormType.NATIVE) {
settings.clear();
} else if (repl.getFormType() == FormType.SIMPLE) {
String scriptBody = getScriptBody();
Map<String, Input> inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built
// from script body
final AngularObjectRegistry angularRegistry =
repl.getInterpreterGroup().getAngularObjectRegistry();
scriptBody = extractVariablesFromAngularRegistry(scriptBody, inputs, angularRegistry);
settings.setForms(inputs);
script = Input.getSimpleQuery(settings.getParams(), scriptBody);
}
logger.debug("RUN : " + script);
try {
InterpreterContext context = getInterpreterContext();
InterpreterContext.set(context);
InterpreterResult ret = repl.interpret(script, context);
if (Code.KEEP_PREVIOUS_RESULT == ret.code()) {
return getReturn();
}
context.out.flush();
List<InterpreterResultMessage> resultMessages = context.out.toInterpreterResultMessage();
resultMessages.addAll(ret.message());
return new InterpreterResult(ret.code(), resultMessages);
} finally {
InterpreterContext.remove();
}
}