@Test public void testCreateDataFrameFromJavaBeans() { Bean bean = new Bean(); JavaRDD<Bean> rdd = jsc.parallelize(Arrays.asList(bean)); DataFrame df = context.createDataFrame(rdd, Bean.class); validateDataFrameWithBeans(bean, df); }
@Test public void testCreateDataFrameFromLocalJavaBeans() { Bean bean = new Bean(); List<Bean> data = Arrays.asList(bean); DataFrame df = context.createDataFrame(data, Bean.class); validateDataFrameWithBeans(bean, df); }
@Test public void testCreateDataFromFromList() { StructType schema = createStructType(Arrays.asList(createStructField("i", IntegerType, true))); List<Row> rows = Arrays.asList(RowFactory.create(0)); DataFrame df = context.createDataFrame(rows, schema); Row[] result = df.collect(); Assert.assertEquals(1, result.length); }
@Test public void testJavaBeanEncoder() { OuterScopes.addOuterScope(this); SimpleJavaBean obj1 = new SimpleJavaBean(); obj1.setA(true); obj1.setB(3); obj1.setC(new byte[] {1, 2}); obj1.setD(new String[] {"hello", null}); obj1.setE(Arrays.asList("a", "b")); obj1.setF(Arrays.asList(100L, null, 200L)); SimpleJavaBean obj2 = new SimpleJavaBean(); obj2.setA(false); obj2.setB(30); obj2.setC(new byte[] {3, 4}); obj2.setD(new String[] {null, "world"}); obj2.setE(Arrays.asList("x", "y")); obj2.setF(Arrays.asList(300L, null, 400L)); List<SimpleJavaBean> data = Arrays.asList(obj1, obj2); Dataset<SimpleJavaBean> ds = context.createDataset(data, Encoders.bean(SimpleJavaBean.class)); Assert.assertEquals(data, ds.collectAsList()); NestedJavaBean obj3 = new NestedJavaBean(); obj3.setA(obj1); List<NestedJavaBean> data2 = Arrays.asList(obj3); Dataset<NestedJavaBean> ds2 = context.createDataset(data2, Encoders.bean(NestedJavaBean.class)); Assert.assertEquals(data2, ds2.collectAsList()); Row row1 = new GenericRow( new Object[] { true, 3, new byte[] {1, 2}, new String[] {"hello", null}, Arrays.asList("a", "b"), Arrays.asList(100L, null, 200L) }); Row row2 = new GenericRow( new Object[] { false, 30, new byte[] {3, 4}, new String[] {null, "world"}, Arrays.asList("x", "y"), Arrays.asList(300L, null, 400L) }); StructType schema = new StructType() .add("a", BooleanType, false) .add("b", IntegerType, false) .add("c", BinaryType) .add("d", createArrayType(StringType)) .add("e", createArrayType(StringType)) .add("f", createArrayType(LongType)); Dataset<SimpleJavaBean> ds3 = context .createDataFrame(Arrays.asList(row1, row2), schema) .as(Encoders.bean(SimpleJavaBean.class)); Assert.assertEquals(data, ds3.collectAsList()); }