@Override
public void open() {
out = new ByteArrayOutputStream();
flinkConf = new org.apache.flink.configuration.Configuration();
Properties intpProperty = getProperty();
for (Object k : intpProperty.keySet()) {
String key = (String) k;
String val = toString(intpProperty.get(key));
flinkConf.setString(key, val);
}
if (localMode()) {
startFlinkMiniCluster();
}
flinkIloop = new FlinkILoop(getHost(), getPort(), (BufferedReader) null, new PrintWriter(out));
flinkIloop.settings_$eq(createSettings());
flinkIloop.createInterpreter();
imain = flinkIloop.intp();
// prepare bindings
imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
binder = (Map<String, Object>) getValue("_binder");
// import libraries
imain.interpret("import scala.tools.nsc.io._");
imain.interpret("import Properties.userHome");
imain.interpret("import scala.compat.Platform.EOL");
imain.interpret("import org.apache.flink.api.scala._");
imain.interpret("import org.apache.flink.api.common.functions._");
imain.bindValue("env", flinkIloop.scalaEnv());
}
@Override
public void open() {
out = new ByteArrayOutputStream();
flinkConf = new org.apache.flink.configuration.Configuration();
Properties intpProperty = getProperty();
for (Object k : intpProperty.keySet()) {
String key = (String) k;
String val = toString(intpProperty.get(key));
flinkConf.setString(key, val);
}
if (localMode()) {
startFlinkMiniCluster();
}
flinkIloop =
new FlinkILoop(
getHost(), getPort(), flinkConf, (BufferedReader) null, new PrintWriter(out));
flinkIloop.settings_$eq(createSettings());
flinkIloop.createInterpreter();
imain = flinkIloop.intp();
org.apache.flink.api.scala.ExecutionEnvironment benv = flinkIloop.scalaBenv();
// new ExecutionEnvironment(remoteBenv)
org.apache.flink.streaming.api.scala.StreamExecutionEnvironment senv = flinkIloop.scalaSenv();
senv.getConfig().disableSysoutLogging();
benv.getConfig().disableSysoutLogging();
// prepare bindings
imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
Map<String, Object> binder = (Map<String, Object>) getLastObject();
// import libraries
imain.interpret("import scala.tools.nsc.io._");
imain.interpret("import Properties.userHome");
imain.interpret("import scala.compat.Platform.EOL");
imain.interpret("import org.apache.flink.api.scala._");
imain.interpret("import org.apache.flink.api.common.functions._");
binder.put("benv", benv);
imain.interpret(
"val benv = _binder.get(\"benv\").asInstanceOf[" + benv.getClass().getName() + "]");
binder.put("senv", senv);
imain.interpret(
"val senv = _binder.get(\"senv\").asInstanceOf[" + senv.getClass().getName() + "]");
}
@Override
public void close() {
flinkIloop.closeInterpreter();
if (localMode()) {
stopFlinkMiniCluster();
}
}
public Object getValue(String name) {
IMain imain = flinkIloop.intp();
Object ret = imain.valueOfTerm(name);
if (ret instanceof None) {
return null;
} else if (ret instanceof Some) {
return ((Some) ret).get();
} else {
return ret;
}
}
public InterpreterResult interpret(String[] lines, InterpreterContext context) {
IMain imain = flinkIloop.intp();
String[] linesToRun = new String[lines.length + 1];
for (int i = 0; i < lines.length; i++) {
linesToRun[i] = lines[i];
}
linesToRun[lines.length] = "print(\"\")";
Console.setOut(out);
out.reset();
Code r = null;
String incomplete = "";
for (String s : linesToRun) {
scala.tools.nsc.interpreter.Results.Result res = null;
try {
res = imain.interpret(incomplete + s);
} catch (Exception e) {
logger.info("Interpreter exception", e);
return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
r = getResultCode(res);
if (r == Code.ERROR) {
return new InterpreterResult(r, out.toString());
} else if (r == Code.INCOMPLETE) {
incomplete += s + "\n";
} else {
incomplete = "";
}
}
if (r == Code.INCOMPLETE) {
return new InterpreterResult(r, "Incomplete expression");
} else {
return new InterpreterResult(r, out.toString());
}
}
/**
* compiles jars from files in the shell virtual directory on the fly, sends and executes it in
* the remote environment
*
* @param jobName name of the job as string
* @return Result of the computation
* @throws Exception
*/
@Override
public JobExecutionResult execute(String jobName) throws Exception {
Plan p = createProgramPlan(jobName);
String jarFile = flinkILoop.writeFilesToDisk().getAbsolutePath();
// get "external jars, and add the shell command jar, pass to executor
List<String> alljars = new ArrayList<String>();
// get external (library) jars
String[] extJars = this.flinkILoop.getExternalJars();
if (!ArrayUtils.isEmpty(extJars)) {
alljars.addAll(Arrays.asList(extJars));
}
// add shell commands
alljars.add(jarFile);
String[] alljarsArr = new String[alljars.size()];
alljarsArr = alljars.toArray(alljarsArr);
PlanExecutor executor = PlanExecutor.createRemoteExecutor(host, port, alljarsArr);
executor.setPrintStatusDuringExecution(p.getExecutionConfig().isSysoutLoggingEnabled());
return executor.executePlan(p);
}
public InterpreterResult interpret(String[] lines, InterpreterContext context) {
final IMain imain = flinkIloop.intp();
String[] linesToRun = new String[lines.length + 1];
for (int i = 0; i < lines.length; i++) {
linesToRun[i] = lines[i];
}
linesToRun[lines.length] = "print(\"\")";
System.setOut(new PrintStream(out));
out.reset();
Code r = null;
String incomplete = "";
boolean inComment = false;
for (int l = 0; l < linesToRun.length; l++) {
final String s = linesToRun[l];
// check if next line starts with "." (but not ".." or "./") it is treated as an invocation
if (l + 1 < linesToRun.length) {
String nextLine = linesToRun[l + 1].trim();
boolean continuation = false;
if (nextLine.isEmpty()
|| nextLine.startsWith("//") // skip empty line or comment
|| nextLine.startsWith("}")
|| nextLine.startsWith("object")) { // include "} object" for Scala companion object
continuation = true;
} else if (!inComment && nextLine.startsWith("/*")) {
inComment = true;
continuation = true;
} else if (inComment && nextLine.lastIndexOf("*/") >= 0) {
inComment = false;
continuation = true;
} else if (nextLine.length() > 1
&& nextLine.charAt(0) == '.'
&& nextLine.charAt(1) != '.' // ".."
&& nextLine.charAt(1) != '/') { // "./"
continuation = true;
} else if (inComment) {
continuation = true;
}
if (continuation) {
incomplete += s + "\n";
continue;
}
}
final String currentCommand = incomplete;
scala.tools.nsc.interpreter.Results.Result res = null;
try {
res =
Console.withOut(
System.out,
new AbstractFunction0<Results.Result>() {
@Override
public Results.Result apply() {
return imain.interpret(currentCommand + s);
}
});
} catch (Exception e) {
logger.info("Interpreter exception", e);
return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
r = getResultCode(res);
if (r == Code.ERROR) {
return new InterpreterResult(r, out.toString());
} else if (r == Code.INCOMPLETE) {
incomplete += s + "\n";
} else {
incomplete = "";
}
}
if (r == Code.INCOMPLETE) {
return new InterpreterResult(r, "Incomplete expression");
} else {
return new InterpreterResult(r, out.toString());
}
}