Exemplo n.º 1
0
  @Override
  public void open() {
    out = new ByteArrayOutputStream();
    flinkConf = new org.apache.flink.configuration.Configuration();
    Properties intpProperty = getProperty();
    for (Object k : intpProperty.keySet()) {
      String key = (String) k;
      String val = toString(intpProperty.get(key));
      flinkConf.setString(key, val);
    }

    if (localMode()) {
      startFlinkMiniCluster();
    }

    flinkIloop = new FlinkILoop(getHost(), getPort(), (BufferedReader) null, new PrintWriter(out));
    flinkIloop.settings_$eq(createSettings());
    flinkIloop.createInterpreter();

    imain = flinkIloop.intp();

    // prepare bindings
    imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
    binder = (Map<String, Object>) getValue("_binder");

    // import libraries
    imain.interpret("import scala.tools.nsc.io._");
    imain.interpret("import Properties.userHome");
    imain.interpret("import scala.compat.Platform.EOL");

    imain.interpret("import org.apache.flink.api.scala._");
    imain.interpret("import org.apache.flink.api.common.functions._");
    imain.bindValue("env", flinkIloop.scalaEnv());
  }
  @Override
  public void open() {
    out = new ByteArrayOutputStream();
    flinkConf = new org.apache.flink.configuration.Configuration();
    Properties intpProperty = getProperty();
    for (Object k : intpProperty.keySet()) {
      String key = (String) k;
      String val = toString(intpProperty.get(key));
      flinkConf.setString(key, val);
    }

    if (localMode()) {
      startFlinkMiniCluster();
    }

    flinkIloop =
        new FlinkILoop(
            getHost(), getPort(), flinkConf, (BufferedReader) null, new PrintWriter(out));

    flinkIloop.settings_$eq(createSettings());
    flinkIloop.createInterpreter();

    imain = flinkIloop.intp();

    org.apache.flink.api.scala.ExecutionEnvironment benv = flinkIloop.scalaBenv();
    // new ExecutionEnvironment(remoteBenv)
    org.apache.flink.streaming.api.scala.StreamExecutionEnvironment senv = flinkIloop.scalaSenv();

    senv.getConfig().disableSysoutLogging();
    benv.getConfig().disableSysoutLogging();

    // prepare bindings
    imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
    Map<String, Object> binder = (Map<String, Object>) getLastObject();

    // import libraries
    imain.interpret("import scala.tools.nsc.io._");
    imain.interpret("import Properties.userHome");
    imain.interpret("import scala.compat.Platform.EOL");

    imain.interpret("import org.apache.flink.api.scala._");
    imain.interpret("import org.apache.flink.api.common.functions._");

    binder.put("benv", benv);
    imain.interpret(
        "val benv = _binder.get(\"benv\").asInstanceOf[" + benv.getClass().getName() + "]");

    binder.put("senv", senv);
    imain.interpret(
        "val senv = _binder.get(\"senv\").asInstanceOf[" + senv.getClass().getName() + "]");
  }
  @Override
  public void close() {
    flinkIloop.closeInterpreter();

    if (localMode()) {
      stopFlinkMiniCluster();
    }
  }
Exemplo n.º 4
0
 public Object getValue(String name) {
   IMain imain = flinkIloop.intp();
   Object ret = imain.valueOfTerm(name);
   if (ret instanceof None) {
     return null;
   } else if (ret instanceof Some) {
     return ((Some) ret).get();
   } else {
     return ret;
   }
 }
Exemplo n.º 5
0
  public InterpreterResult interpret(String[] lines, InterpreterContext context) {
    IMain imain = flinkIloop.intp();

    String[] linesToRun = new String[lines.length + 1];
    for (int i = 0; i < lines.length; i++) {
      linesToRun[i] = lines[i];
    }
    linesToRun[lines.length] = "print(\"\")";

    Console.setOut(out);
    out.reset();
    Code r = null;

    String incomplete = "";
    for (String s : linesToRun) {
      scala.tools.nsc.interpreter.Results.Result res = null;
      try {
        res = imain.interpret(incomplete + s);
      } catch (Exception e) {
        logger.info("Interpreter exception", e);
        return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
      }

      r = getResultCode(res);

      if (r == Code.ERROR) {
        return new InterpreterResult(r, out.toString());
      } else if (r == Code.INCOMPLETE) {
        incomplete += s + "\n";
      } else {
        incomplete = "";
      }
    }

    if (r == Code.INCOMPLETE) {
      return new InterpreterResult(r, "Incomplete expression");
    } else {
      return new InterpreterResult(r, out.toString());
    }
  }
  /**
   * compiles jars from files in the shell virtual directory on the fly, sends and executes it in
   * the remote environment
   *
   * @param jobName name of the job as string
   * @return Result of the computation
   * @throws Exception
   */
  @Override
  public JobExecutionResult execute(String jobName) throws Exception {
    Plan p = createProgramPlan(jobName);

    String jarFile = flinkILoop.writeFilesToDisk().getAbsolutePath();

    // get "external jars, and add the shell command jar, pass to executor
    List<String> alljars = new ArrayList<String>();
    // get external (library) jars
    String[] extJars = this.flinkILoop.getExternalJars();

    if (!ArrayUtils.isEmpty(extJars)) {
      alljars.addAll(Arrays.asList(extJars));
    }
    // add shell commands
    alljars.add(jarFile);
    String[] alljarsArr = new String[alljars.size()];
    alljarsArr = alljars.toArray(alljarsArr);
    PlanExecutor executor = PlanExecutor.createRemoteExecutor(host, port, alljarsArr);

    executor.setPrintStatusDuringExecution(p.getExecutionConfig().isSysoutLoggingEnabled());
    return executor.executePlan(p);
  }
  public InterpreterResult interpret(String[] lines, InterpreterContext context) {
    final IMain imain = flinkIloop.intp();

    String[] linesToRun = new String[lines.length + 1];
    for (int i = 0; i < lines.length; i++) {
      linesToRun[i] = lines[i];
    }
    linesToRun[lines.length] = "print(\"\")";

    System.setOut(new PrintStream(out));
    out.reset();
    Code r = null;

    String incomplete = "";
    boolean inComment = false;

    for (int l = 0; l < linesToRun.length; l++) {
      final String s = linesToRun[l];
      // check if next line starts with "." (but not ".." or "./") it is treated as an invocation
      if (l + 1 < linesToRun.length) {
        String nextLine = linesToRun[l + 1].trim();
        boolean continuation = false;
        if (nextLine.isEmpty()
            || nextLine.startsWith("//") // skip empty line or comment
            || nextLine.startsWith("}")
            || nextLine.startsWith("object")) { // include "} object" for Scala companion object
          continuation = true;
        } else if (!inComment && nextLine.startsWith("/*")) {
          inComment = true;
          continuation = true;
        } else if (inComment && nextLine.lastIndexOf("*/") >= 0) {
          inComment = false;
          continuation = true;
        } else if (nextLine.length() > 1
            && nextLine.charAt(0) == '.'
            && nextLine.charAt(1) != '.' // ".."
            && nextLine.charAt(1) != '/') { // "./"
          continuation = true;
        } else if (inComment) {
          continuation = true;
        }
        if (continuation) {
          incomplete += s + "\n";
          continue;
        }
      }

      final String currentCommand = incomplete;

      scala.tools.nsc.interpreter.Results.Result res = null;
      try {
        res =
            Console.withOut(
                System.out,
                new AbstractFunction0<Results.Result>() {
                  @Override
                  public Results.Result apply() {
                    return imain.interpret(currentCommand + s);
                  }
                });
      } catch (Exception e) {
        logger.info("Interpreter exception", e);
        return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
      }

      r = getResultCode(res);

      if (r == Code.ERROR) {
        return new InterpreterResult(r, out.toString());
      } else if (r == Code.INCOMPLETE) {
        incomplete += s + "\n";
      } else {
        incomplete = "";
      }
    }

    if (r == Code.INCOMPLETE) {
      return new InterpreterResult(r, "Incomplete expression");
    } else {
      return new InterpreterResult(r, out.toString());
    }
  }