Exemplo n.º 1
0
  @Override
  protected Object jobRun() throws Throwable {
    String replName = getRequiredReplName();
    Interpreter repl = getRepl(replName);
    logger().info("run paragraph {} using {} " + repl, getId(), replName);
    if (repl == null) {
      logger().error("Can not find interpreter name " + repl);
      throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
    }

    String script = getScriptBody();
    // inject form
    if (repl.getFormType() == FormType.NATIVE) {
      settings.clear();
    } else if (repl.getFormType() == FormType.SIMPLE) {
      String scriptBody = getScriptBody();
      Map<String, Input> inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built
      // from script body
      settings.setForms(inputs);
      script = Input.getSimpleQuery(settings.getParams(), scriptBody);
    }
    logger().info("RUN : " + script);
    InterpreterResult ret = repl.interpret(script, getInterpreterContext());
    return ret;
  }
Exemplo n.º 2
0
  @ZeppelinApi
  public Interpreter getInterpreterInTheSameSessionByClassName(String className) {
    synchronized (interpreterGroup) {
      for (List<Interpreter> interpreters : interpreterGroup.values()) {
        boolean belongsToSameNoteGroup = false;
        Interpreter interpreterFound = null;
        for (Interpreter intp : interpreters) {
          if (intp.getClassName().equals(className)) {
            interpreterFound = intp;
          }

          Interpreter p = intp;
          while (p instanceof WrappedInterpreter) {
            p = ((WrappedInterpreter) p).getInnerInterpreter();
          }
          if (this == p) {
            belongsToSameNoteGroup = true;
          }
        }

        if (belongsToSameNoteGroup) {
          return interpreterFound;
        }
      }
    }
    return null;
  }
Exemplo n.º 3
0
  /**
   * Run a single paragraph.
   *
   * @param paragraphId ID of paragraph
   */
  public void run(String paragraphId) {
    Paragraph p = getParagraph(paragraphId);

    if (p.isBlankParagraph()) {
      logger.info("skip to run blank paragraph. {}", p.getId());
      return;
    }

    p.setListener(jobListenerFactory.getParagraphJobListener(this));
    String requiredReplName = p.getRequiredReplName();
    Interpreter intp = factory.getInterpreter(p.getUser(), getId(), requiredReplName);

    if (intp == null) {
      String intpExceptionMsg =
          p.getJobName() + "'s Interpreter " + requiredReplName + " not found";
      InterpreterException intpException = new InterpreterException(intpExceptionMsg);
      InterpreterResult intpResult =
          new InterpreterResult(InterpreterResult.Code.ERROR, intpException.getMessage());
      p.setReturn(intpResult, intpException);
      p.setStatus(Job.Status.ERROR);
      throw intpException;
    }
    if (p.getConfig().get("enabled") == null || (Boolean) p.getConfig().get("enabled")) {
      p.setAuthenticationInfo(p.getAuthenticationInfo());
      intp.getScheduler().submit(p);
    }
  }
Exemplo n.º 4
0
  @Override
  protected Object jobRun() throws Throwable {
    String replName = getRequiredReplName();
    Interpreter repl = getRepl(replName);
    logger().info("run paragraph {} using {} " + repl, getId(), replName);
    if (repl == null) {
      logger().error("Can not find interpreter name " + repl);
      throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
    }

    String script = getScriptBody();
    // inject form
    if (repl.getFormType() == FormType.NATIVE) {
      settings.clear();
    } else if (repl.getFormType() == FormType.SIMPLE) {
      String scriptBody = getScriptBody();
      Map<String, Input> inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built
      // from script body
      settings.setForms(inputs);
      script = Input.getSimpleQuery(settings.getParams(), scriptBody);
    }
    logger().debug("RUN : " + script);
    try {
      InterpreterContext context = getInterpreterContext();
      InterpreterContext.set(context);
      InterpreterResult ret = repl.interpret(script, context);

      if (Code.KEEP_PREVIOUS_RESULT == ret.code()) {
        return getReturn();
      }

      String message = "";

      context.out.flush();
      InterpreterResult.Type outputType = context.out.getType();
      byte[] interpreterOutput = context.out.toByteArray();
      context.out.clear();

      if (interpreterOutput != null && interpreterOutput.length > 0) {
        message = new String(interpreterOutput);
      }

      if (message.isEmpty()) {
        return ret;
      } else {
        String interpreterResultMessage = ret.message();
        if (interpreterResultMessage != null && !interpreterResultMessage.isEmpty()) {
          message += interpreterResultMessage;
          return new InterpreterResult(ret.code(), ret.type(), message);
        } else {
          return new InterpreterResult(ret.code(), outputType, message);
        }
      }
    } finally {
      InterpreterContext.remove();
    }
  }
Exemplo n.º 5
0
 @Override
 public int progress() {
   String replName = getRequiredReplName();
   Interpreter repl = getRepl(replName);
   if (repl != null) {
     return repl.getProgress(getInterpreterContext());
   } else {
     return 0;
   }
 }
Exemplo n.º 6
0
 /**
  * Run a single paragraph.
  *
  * @param paragraphId
  */
 public void run(String paragraphId) {
   Paragraph p = getParagraph(paragraphId);
   p.setNoteReplLoader(replLoader);
   p.setListener(jobListenerFactory.getParagraphJobListener(this));
   Interpreter intp = replLoader.get(p.getRequiredReplName());
   if (intp == null) {
     throw new InterpreterException("Interpreter " + p.getRequiredReplName() + " not found");
   }
   intp.getScheduler().submit(p);
 }
Exemplo n.º 7
0
 /**
  * Run all paragraphs sequentially.
  *
  * @param jobListener
  */
 public void runAll() {
   synchronized (paragraphs) {
     for (Paragraph p : paragraphs) {
       p.setNoteReplLoader(replLoader);
       p.setListener(jobListenerFactory.getParagraphJobListener(this));
       Interpreter intp = replLoader.get(p.getRequiredReplName());
       intp.getScheduler().submit(p);
     }
   }
 }
Exemplo n.º 8
0
 @Override
 protected boolean jobAbort() {
   Interpreter repl = getRepl(getRequiredReplName());
   Job job = repl.getScheduler().removeFromWaitingQueue(getId());
   if (job != null) {
     job.setStatus(Status.ABORT);
   } else {
     repl.cancel(getInterpreterContext());
   }
   return true;
 }
Exemplo n.º 9
0
  /**
   * Get combined property of all interpreters in this group
   *
   * @return
   */
  public Properties getProperty() {
    Properties p = new Properties();

    for (List<Interpreter> intpGroupForASession : this.values()) {
      for (Interpreter intp : intpGroupForASession) {
        p.putAll(intp.getProperty());
      }
      // it's okay to break here while every List<Interpreters> will have the same property set
      break;
    }
    return p;
  }
Exemplo n.º 10
0
  public List<String> completion(String buffer, int cursor) {
    String replName = getRequiredReplName(buffer);
    if (replName != null) {
      cursor -= replName.length() + 1;
    }
    String body = getScriptBody(buffer);
    Interpreter repl = getRepl(replName);
    if (repl == null) {
      return null;
    }

    return repl.completion(body, cursor);
  }
 static {
   Interpreter.register(
       "pyspark",
       "livy",
       LivyPySparkInterpreter.class.getName(),
       new InterpreterPropertyBuilder().build());
 }
 static {
   Interpreter.register(
       "tachyon",
       "tachyon",
       TachyonInterpreter.class.getName(),
       new InterpreterPropertyBuilder()
           .add(TACHYON_MASTER_HOSTNAME, "localhost", "Tachyon master hostname")
           .add(TACHYON_MASTER_PORT, "19998", "Tachyon master port")
           .build());
 }
Exemplo n.º 13
0
  @Override
  protected boolean jobAbort() {
    Interpreter repl = getRepl(getRequiredReplName());
    if (repl == null) {
      // when interpreters are already destroyed
      return true;
    }

    Scheduler scheduler = repl.getScheduler();
    if (scheduler == null) {
      return true;
    }

    Job job = scheduler.removeFromWaitingQueue(getId());
    if (job != null) {
      job.setStatus(Status.ABORT);
    } else {
      repl.cancel(getInterpreterContext());
    }
    return true;
  }
Exemplo n.º 14
0
 static {
   Interpreter.register(
       "flink",
       "flink",
       FlinkInterpreter.class.getName(),
       new InterpreterPropertyBuilder()
           .add(
               "host",
               "local",
               "host name of running JobManager. 'local' runs flink in local mode")
           .add("port", "6123", "port of running JobManager")
           .build());
 }
Exemplo n.º 15
0
  public List<InterpreterCompletion> completion(String buffer, int cursor) {
    String lines[] = buffer.split(System.getProperty("line.separator"));
    if (lines.length > 0 && lines[0].startsWith("%") && cursor <= lines[0].trim().length()) {

      int idx = lines[0].indexOf(' ');
      if (idx < 0 || (idx > 0 && cursor <= idx)) {
        return getInterpreterCompletion();
      }
    }

    String replName = getRequiredReplName(buffer);
    if (replName != null && cursor > replName.length()) {
      cursor -= replName.length() + 1;
    }

    String body = getScriptBody(buffer);
    Interpreter repl = getRepl(replName);
    if (repl == null) {
      return null;
    }

    List completion = repl.completion(body, cursor);
    return completion;
  }
Exemplo n.º 16
0
  @ZeppelinApi
  public Properties getProperty() {
    Properties p = new Properties();
    p.putAll(property);

    RegisteredInterpreter registeredInterpreter =
        Interpreter.findRegisteredInterpreterByClassName(getClassName());
    if (null != registeredInterpreter) {
      Map<String, InterpreterProperty> defaultProperties = registeredInterpreter.getProperties();
      for (String k : defaultProperties.keySet()) {
        if (!p.containsKey(k)) {
          String value = defaultProperties.get(k).getValue();
          if (value != null) {
            p.put(k, defaultProperties.get(k).getValue());
          }
        }
      }
    }

    return p;
  }
Exemplo n.º 17
0
 static {
   Interpreter.register("md", Markdown.class.getName());
 }
Exemplo n.º 18
0
 @Override
 protected boolean jobAbort() {
   Interpreter repl = getRepl(getRequiredReplName());
   repl.cancel(getInterpreterContext());
   return true;
 }
Exemplo n.º 19
0
  @Override
  protected Object jobRun() throws Throwable {
    String replName = getRequiredReplName();
    Interpreter repl = getRepl(replName);
    logger.info("run paragraph {} using {} " + repl, getId(), replName);
    if (repl == null) {
      logger.error("Can not find interpreter name " + repl);
      throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
    }
    InterpreterSetting intp = getInterpreterSettingById(repl.getInterpreterGroup().getId());
    while (intp.getStatus()
        .equals(
            org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) {
      Thread.sleep(200);
    }
    if (this.noteHasUser() && this.noteHasInterpreters()) {
      if (intp != null
          && interpreterHasUser(intp)
          && isUserAuthorizedToAccessInterpreter(intp.getOption()) == false) {
        logger.error("{} has no permission for {} ", authenticationInfo.getUser(), repl);
        return new InterpreterResult(
            Code.ERROR,
            authenticationInfo.getUser() + " has no permission for " + getRequiredReplName());
      }
    }

    String script = getScriptBody();
    // inject form
    if (repl.getFormType() == FormType.NATIVE) {
      settings.clear();
    } else if (repl.getFormType() == FormType.SIMPLE) {
      String scriptBody = getScriptBody();
      Map<String, Input> inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built
      // from script body

      final AngularObjectRegistry angularRegistry =
          repl.getInterpreterGroup().getAngularObjectRegistry();

      scriptBody = extractVariablesFromAngularRegistry(scriptBody, inputs, angularRegistry);

      settings.setForms(inputs);
      script = Input.getSimpleQuery(settings.getParams(), scriptBody);
    }
    logger.debug("RUN : " + script);
    try {
      InterpreterContext context = getInterpreterContext();
      InterpreterContext.set(context);
      InterpreterResult ret = repl.interpret(script, context);

      if (Code.KEEP_PREVIOUS_RESULT == ret.code()) {
        return getReturn();
      }

      context.out.flush();
      List<InterpreterResultMessage> resultMessages = context.out.toInterpreterResultMessage();
      resultMessages.addAll(ret.message());
      return new InterpreterResult(ret.code(), resultMessages);
    } finally {
      InterpreterContext.remove();
    }
  }