Пример #1
0
 @Test
 public void testCreateDataFrameFromJavaBeans() {
   Bean bean = new Bean();
   JavaRDD<Bean> rdd = jsc.parallelize(Arrays.asList(bean));
   DataFrame df = context.createDataFrame(rdd, Bean.class);
   validateDataFrameWithBeans(bean, df);
 }
Пример #2
0
 @Test
 public void testCreateDataFrameFromLocalJavaBeans() {
   Bean bean = new Bean();
   List<Bean> data = Arrays.asList(bean);
   DataFrame df = context.createDataFrame(data, Bean.class);
   validateDataFrameWithBeans(bean, df);
 }
Пример #3
0
 @Test
 public void testCreateDataFromFromList() {
   StructType schema = createStructType(Arrays.asList(createStructField("i", IntegerType, true)));
   List<Row> rows = Arrays.asList(RowFactory.create(0));
   DataFrame df = context.createDataFrame(rows, schema);
   Row[] result = df.collect();
   Assert.assertEquals(1, result.length);
 }
Пример #4
0
  @Test
  public void testJavaBeanEncoder() {
    OuterScopes.addOuterScope(this);
    SimpleJavaBean obj1 = new SimpleJavaBean();
    obj1.setA(true);
    obj1.setB(3);
    obj1.setC(new byte[] {1, 2});
    obj1.setD(new String[] {"hello", null});
    obj1.setE(Arrays.asList("a", "b"));
    obj1.setF(Arrays.asList(100L, null, 200L));
    SimpleJavaBean obj2 = new SimpleJavaBean();
    obj2.setA(false);
    obj2.setB(30);
    obj2.setC(new byte[] {3, 4});
    obj2.setD(new String[] {null, "world"});
    obj2.setE(Arrays.asList("x", "y"));
    obj2.setF(Arrays.asList(300L, null, 400L));

    List<SimpleJavaBean> data = Arrays.asList(obj1, obj2);
    Dataset<SimpleJavaBean> ds = context.createDataset(data, Encoders.bean(SimpleJavaBean.class));
    Assert.assertEquals(data, ds.collectAsList());

    NestedJavaBean obj3 = new NestedJavaBean();
    obj3.setA(obj1);

    List<NestedJavaBean> data2 = Arrays.asList(obj3);
    Dataset<NestedJavaBean> ds2 = context.createDataset(data2, Encoders.bean(NestedJavaBean.class));
    Assert.assertEquals(data2, ds2.collectAsList());

    Row row1 =
        new GenericRow(
            new Object[] {
              true,
              3,
              new byte[] {1, 2},
              new String[] {"hello", null},
              Arrays.asList("a", "b"),
              Arrays.asList(100L, null, 200L)
            });
    Row row2 =
        new GenericRow(
            new Object[] {
              false,
              30,
              new byte[] {3, 4},
              new String[] {null, "world"},
              Arrays.asList("x", "y"),
              Arrays.asList(300L, null, 400L)
            });
    StructType schema =
        new StructType()
            .add("a", BooleanType, false)
            .add("b", IntegerType, false)
            .add("c", BinaryType)
            .add("d", createArrayType(StringType))
            .add("e", createArrayType(StringType))
            .add("f", createArrayType(LongType));
    Dataset<SimpleJavaBean> ds3 =
        context
            .createDataFrame(Arrays.asList(row1, row2), schema)
            .as(Encoders.bean(SimpleJavaBean.class));
    Assert.assertEquals(data, ds3.collectAsList());
  }