@Test
  public void testSparkSql() {
    repl.interpret("case class Person(name:String, age:Int)\n", context);
    repl.interpret(
        "val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n",
        context);
    assertEquals(Code.SUCCESS, repl.interpret("people.take(3)", context).code());

    if (getSparkVersionNumber()
        <= 11) { // spark 1.2 or later does not allow create multiple SparkContext in the same jvm
      // by default.
      // create new interpreter
      SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties());
      repl2.setInterpreterGroup(intpGroup);
      intpGroup.get("note").add(repl2);
      repl2.open();

      repl2.interpret("case class Man(name:String, age:Int)", context);
      repl2.interpret(
          "val man = sc.parallelize(Seq(Man(\"moon\", 33), Man(\"jobs\", 51), Man(\"gates\", 51), Man(\"park\", 34)))",
          context);
      assertEquals(Code.SUCCESS, repl2.interpret("man.take(3)", context).code());
      repl2.close();
    }
  }
 @Test
 public void emptyConfigurationVariablesOnlyForNonSparkProperties() {
   Properties intpProperty = repl.getProperty();
   SparkConf sparkConf = repl.getSparkContext().getConf();
   for (Object oKey : intpProperty.keySet()) {
     String key = (String) oKey;
     String value = (String) intpProperty.get(key);
     LOGGER.debug(String.format("[%s]: [%s]", key, value));
     if (key.startsWith("spark.") && value.isEmpty()) {
       assertTrue(
           String.format("configuration starting from 'spark.' should not be empty. [%s]", key),
           !sparkConf.contains(key) || !sparkConf.get(key).isEmpty());
     }
   }
 }
 @Test
 public void testZShow() {
   String code = "";
   repl.interpret("case class Person(name:String, age:Int)\n", context);
   repl.interpret(
       "val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n",
       context);
   if (getSparkVersionNumber() < 13) {
     repl.interpret("people.registerTempTable(\"people\")", context);
     code = "z.show(sqlc.sql(\"select * from people\"))";
   } else {
     code = "z.show(people.toDF)";
   }
   assertEquals(Code.SUCCESS, repl.interpret(code, context).code());
 }
 @Test
 public void testReferencingUndefinedVal() {
   InterpreterResult result =
       repl.interpret(
           "def category(min: Int) = {" + "    if (0 <= value) \"error\"" + "}", context);
   assertEquals(Code.ERROR, result.code());
 }
  @Test
  public void testEnableImplicitImport() {
    if (getSparkVersionNumber() >= 13) {
      // Set option of importing implicits to "true", and initialize new Spark repl
      Properties p = getSparkTestProperties();
      p.setProperty("zeppelin.spark.importImplicit", "true");
      SparkInterpreter repl2 = new SparkInterpreter(p);
      repl2.setInterpreterGroup(intpGroup);
      intpGroup.get("note").add(repl2);

      repl2.open();
      String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")";
      assertEquals(Code.SUCCESS, repl2.interpret(ddl, context).code());
      repl2.close();
    }
  }
  @Test
  public void testDisableImplicitImport() {
    if (getSparkVersionNumber() >= 13) {
      // Set option of importing implicits to "false", and initialize new Spark repl
      // this test should return error status when creating DataFrame from sequence
      Properties p = getSparkTestProperties();
      p.setProperty("zeppelin.spark.importImplicit", "false");
      SparkInterpreter repl2 = new SparkInterpreter(p);
      repl2.setInterpreterGroup(intpGroup);
      intpGroup.get("note").add(repl2);

      repl2.open();
      String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")";
      assertEquals(Code.ERROR, repl2.interpret(ddl, context).code());
      repl2.close();
    }
  }
  @Test
  public void testBasicIntp() {
    assertEquals(
        InterpreterResult.Code.SUCCESS, repl.interpret("val a = 1\nval b = 2", context).code());

    // when interpret incomplete expression
    InterpreterResult incomplete = repl.interpret("val a = \"\"\"", context);
    assertEquals(InterpreterResult.Code.INCOMPLETE, incomplete.code());
    assertTrue(incomplete.message().length() > 0); // expecting some error
    // message

    /*
     * assertEquals(1, repl.getValue("a")); assertEquals(2, repl.getValue("b"));
     * repl.interpret("val ver = sc.version");
     * assertNotNull(repl.getValue("ver")); assertEquals("HELLO\n",
     * repl.interpret("println(\"HELLO\")").message());
     */
  }
  /**
   * Get spark version number as a numerical value. eg. 1.1.x => 11, 1.2.x => 12, 1.3.x => 13 ...
   */
  public static int getSparkVersionNumber() {
    if (repl == null) {
      return 0;
    }

    String[] split = repl.getSparkContext().version().split("\\.");
    int version = Integer.parseInt(split[0]) * 10 + Integer.parseInt(split[1]);
    return version;
  }
 @Test
 public void testCreateDataFrame() {
   if (getSparkVersionNumber() >= 13) {
     repl.interpret("case class Person(name:String, age:Int)\n", context);
     repl.interpret(
         "val people = sc.parallelize(Seq(Person(\"moon\", 33), Person(\"jobs\", 51), Person(\"gates\", 51), Person(\"park\", 34)))\n",
         context);
     repl.interpret("people.toDF.count", context);
     assertEquals(
         new Long(4),
         context
             .getResourcePool()
             .get(
                 context.getNoteId(),
                 context.getParagraphId(),
                 WellKnownResourceName.ZeppelinReplResult.toString())
             .get());
   }
 }
示例#10
0
  @Before
  public void setUp() throws Exception {
    tmpDir =
        new File(
            System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
    System.setProperty("zeppelin.dep.localrepo", tmpDir.getAbsolutePath() + "/local-repo");

    tmpDir.mkdirs();

    if (repl == null) {
      intpGroup = new InterpreterGroup();
      intpGroup.put("note", new LinkedList<Interpreter>());
      repl = new SparkInterpreter(getSparkTestProperties());
      repl.setInterpreterGroup(intpGroup);
      intpGroup.get("note").add(repl);
      repl.open();
    }

    context =
        new InterpreterContext(
            "note",
            "id",
            "title",
            "text",
            new AuthenticationInfo(),
            new HashMap<String, Object>(),
            new GUI(),
            new AngularObjectRegistry(intpGroup.getId(), null),
            new LocalResourcePool("id"),
            new LinkedList<InterpreterContextRunner>(),
            new InterpreterOutput(
                new InterpreterOutputListener() {
                  @Override
                  public void onAppend(InterpreterOutput out, byte[] line) {}

                  @Override
                  public void onUpdate(InterpreterOutput out, byte[] output) {}
                }));
  }
示例#11
0
  @Test
  public void shareSingleSparkContext() throws InterruptedException {
    // create another SparkInterpreter
    SparkInterpreter repl2 = new SparkInterpreter(getSparkTestProperties());
    repl2.setInterpreterGroup(intpGroup);
    intpGroup.get("note").add(repl2);
    repl2.open();

    assertEquals(
        Code.SUCCESS, repl.interpret("print(sc.parallelize(1 to 10).count())", context).code());
    assertEquals(
        Code.SUCCESS, repl2.interpret("print(sc.parallelize(1 to 10).count())", context).code());

    repl2.close();
  }
示例#12
0
 @Test
 public void testCompletion() {
   List<InterpreterCompletion> completions = repl.completion("sc.", "sc.".length());
   assertTrue(completions.size() > 0);
 }
示例#13
0
 @Test
 public void testNextLineComments() {
   assertEquals(
       InterpreterResult.Code.SUCCESS,
       repl.interpret("\"123\"\n/*comment here\n*/.toInt", context).code());
 }
示例#14
0
 @Test
 public void testNextLineCompanionObject() {
   String code =
       "class Counter {\nvar value: Long = 0\n}\n // comment\n\n object Counter {\n def apply(x: Long) = new Counter()\n}";
   assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret(code, context).code());
 }
示例#15
0
 @Test
 public void testEndWithComment() {
   assertEquals(
       InterpreterResult.Code.SUCCESS, repl.interpret("val c=1\n//comment", context).code());
 }
示例#16
0
 @Test
 public void testListener() {
   SparkContext sc = repl.getSparkContext();
   assertNotNull(SparkInterpreter.setupListeners(sc));
 }
示例#17
0
 @Test
 public void testNextLineInvocation() {
   assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("\"123\"\n.toInt", context).code());
 }