Пример #1
0
  @Test
  public void testSparkSql() {
    repl.interpret("case class Person(name:String, age:Int)\n", context);
    repl.interpret(
        "val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n",
        context);
    assertEquals(Code.SUCCESS, repl.interpret("people.take(3)", context).code());

    if (getSparkVersionNumber()
        <= 11) { // spark 1.2 or later does not allow create multiple SparkContext in the same jvm
      // by default.
      // create new interpreter
      SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties());
      repl2.setInterpreterGroup(intpGroup);
      intpGroup.get("note").add(repl2);
      repl2.open();

      repl2.interpret("case class Man(name:String, age:Int)", context);
      repl2.interpret(
          "val man = sc.parallelize(Seq(Man(\"moon\", 33), Man(\"jobs\", 51), Man(\"gates\", 51), Man(\"park\", 34)))",
          context);
      assertEquals(Code.SUCCESS, repl2.interpret("man.take(3)", context).code());
      repl2.close();
    }
  }
Пример #2
0
  @Test
  public void shareSingleSparkContext() throws InterruptedException {
    // create another SparkInterpreter
    SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties());
    repl2.setInterpreterGroup(intpGroup);
    intpGroup.get("note").add(repl2);
    repl2.open();

    assertEquals(
        Code.SUCCESS, repl.interpret("print(sc.parallelize(1 to 10).count())", context).code());
    assertEquals(
        Code.SUCCESS, repl2.interpret("print(sc.parallelize(1 to 10).count())", context).code());

    repl2.close();
  }
Пример #3
0
  @Test
  public void testEnableImplicitImport() {
    if (getSparkVersionNumber() >= 13) {
      // Set option of importing implicits to "true", and initialize new Spark repl
      Properties p = getSparkTestProperties();
      p.setProperty("zeppelin.spark.importImplicit", "true");
      SparkInterpreter repl2 = new SparkInterpreter(p);
      repl2.setInterpreterGroup(intpGroup);
      intpGroup.get("note").add(repl2);

      repl2.open();
      String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")";
      assertEquals(Code.SUCCESS, repl2.interpret(ddl, context).code());
      repl2.close();
    }
  }
Пример #4
0
  @Test
  public void testDisableImplicitImport() {
    if (getSparkVersionNumber() >= 13) {
      // Set option of importing implicits to "false", and initialize new Spark repl
      // this test should return error status when creating DataFrame from sequence
      Properties p = getSparkTestProperties();
      p.setProperty("zeppelin.spark.importImplicit", "false");
      SparkInterpreter repl2 = new SparkInterpreter(p);
      repl2.setInterpreterGroup(intpGroup);
      intpGroup.get("note").add(repl2);

      repl2.open();
      String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")";
      assertEquals(Code.ERROR, repl2.interpret(ddl, context).code());
      repl2.close();
    }
  }
Пример #5
0
  @Before
  public void setUp() throws Exception {
    tmpDir =
        new File(
            System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
    System.setProperty("zeppelin.dep.localrepo", tmpDir.getAbsolutePath() + "/local-repo");

    tmpDir.mkdirs();

    if (repl == null) {
      intpGroup = new InterpreterGroup();
      intpGroup.put("note", new LinkedList<Interpreter>());
      repl = new SparkInterpreter(getSparkTestProperties());
      repl.setInterpreterGroup(intpGroup);
      intpGroup.get("note").add(repl);
      repl.open();
    }

    context =
        new InterpreterContext(
            "note",
            "id",
            "title",
            "text",
            new AuthenticationInfo(),
            new HashMap<String, Object>(),
            new GUI(),
            new AngularObjectRegistry(intpGroup.getId(), null),
            new LocalResourcePool("id"),
            new LinkedList<InterpreterContextRunner>(),
            new InterpreterOutput(
                new InterpreterOutputListener() {
                  @Override
                  public void onAppend(InterpreterOutput out, byte[] line) {}

                  @Override
                  public void onUpdate(InterpreterOutput out, byte[] output) {}
                }));
  }