@Test
public void testCreateDataFrameFromJavaBeans() {
Bean bean = new Bean();
JavaRDD<Bean> rdd = jsc.parallelize(Arrays.asList(bean));
DataFrame df = context.createDataFrame(rdd, Bean.class);
validateDataFrameWithBeans(bean, df);
}
@Test
public void testCreateDataFrameFromLocalJavaBeans() {
Bean bean = new Bean();
List<Bean> data = Arrays.asList(bean);
DataFrame df = context.createDataFrame(data, Bean.class);
validateDataFrameWithBeans(bean, df);
}
@Test
public void testCreateDataFromFromList() {
StructType schema = createStructType(Arrays.asList(createStructField("i", IntegerType, true)));
List<Row> rows = Arrays.asList(RowFactory.create(0));
DataFrame df = context.createDataFrame(rows, schema);
Row[] result = df.collect();
Assert.assertEquals(1, result.length);
}
@Test
public void testJavaBeanEncoder() {
OuterScopes.addOuterScope(this);
SimpleJavaBean obj1 = new SimpleJavaBean();
obj1.setA(true);
obj1.setB(3);
obj1.setC(new byte[] {1, 2});
obj1.setD(new String[] {"hello", null});
obj1.setE(Arrays.asList("a", "b"));
obj1.setF(Arrays.asList(100L, null, 200L));
SimpleJavaBean obj2 = new SimpleJavaBean();
obj2.setA(false);
obj2.setB(30);
obj2.setC(new byte[] {3, 4});
obj2.setD(new String[] {null, "world"});
obj2.setE(Arrays.asList("x", "y"));
obj2.setF(Arrays.asList(300L, null, 400L));
List<SimpleJavaBean> data = Arrays.asList(obj1, obj2);
Dataset<SimpleJavaBean> ds = context.createDataset(data, Encoders.bean(SimpleJavaBean.class));
Assert.assertEquals(data, ds.collectAsList());
NestedJavaBean obj3 = new NestedJavaBean();
obj3.setA(obj1);
List<NestedJavaBean> data2 = Arrays.asList(obj3);
Dataset<NestedJavaBean> ds2 = context.createDataset(data2, Encoders.bean(NestedJavaBean.class));
Assert.assertEquals(data2, ds2.collectAsList());
Row row1 =
new GenericRow(
new Object[] {
true,
3,
new byte[] {1, 2},
new String[] {"hello", null},
Arrays.asList("a", "b"),
Arrays.asList(100L, null, 200L)
});
Row row2 =
new GenericRow(
new Object[] {
false,
30,
new byte[] {3, 4},
new String[] {null, "world"},
Arrays.asList("x", "y"),
Arrays.asList(300L, null, 400L)
});
StructType schema =
new StructType()
.add("a", BooleanType, false)
.add("b", IntegerType, false)
.add("c", BinaryType)
.add("d", createArrayType(StringType))
.add("e", createArrayType(StringType))
.add("f", createArrayType(LongType));
Dataset<SimpleJavaBean> ds3 =
context
.createDataFrame(Arrays.asList(row1, row2), schema)
.as(Encoders.bean(SimpleJavaBean.class));
Assert.assertEquals(data, ds3.collectAsList());
}