示例#1
0
  @Test
  public void testIdentityMapWithBasicType() throws Exception {
    /*
     * Test identity map with basic type
     */

    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    DataSet<String> ds = CollectionDataSets.getStringDataSet(env);
    DataSet<String> identityMapDs = ds.map(new Mapper1());

    List<String> result = identityMapDs.collect();

    String expected =
        "Hi\n"
            + "Hello\n"
            + "Hello world\n"
            + "Hello world, how are you?\n"
            + "I am fine.\n"
            + "Luke Skywalker\n"
            + "Random comment\n"
            + "LOL\n";

    compareResultAsText(result, expected);
  }
示例#2
0
  @Test
  public void testMapWithBroadcastSet() throws Exception {
    /*
     * Test map with broadcast set
     */

    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    DataSet<Integer> ints = CollectionDataSets.getIntegerDataSet(env);

    DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
    DataSet<Tuple3<Integer, Long, String>> bcMapDs =
        ds.map(new RichMapper1()).withBroadcastSet(ints, "ints");
    List<Tuple3<Integer, Long, String>> result = bcMapDs.collect();

    String expected =
        "55,1,Hi\n"
            + "55,2,Hello\n"
            + "55,2,Hello world\n"
            + "55,3,Hello world, how are you?\n"
            + "55,3,I am fine.\n"
            + "55,3,Luke Skywalker\n"
            + "55,4,Comment#1\n"
            + "55,4,Comment#2\n"
            + "55,4,Comment#3\n"
            + "55,4,Comment#4\n"
            + "55,5,Comment#5\n"
            + "55,5,Comment#6\n"
            + "55,5,Comment#7\n"
            + "55,5,Comment#8\n"
            + "55,5,Comment#9\n"
            + "55,6,Comment#10\n"
            + "55,6,Comment#11\n"
            + "55,6,Comment#12\n"
            + "55,6,Comment#13\n"
            + "55,6,Comment#14\n"
            + "55,6,Comment#15\n";

    compareResultAsTuples(result, expected);
  }
示例#3
0
  @Test(expected = ExpressionException.class)
  public void testOnlyFieldRefInAs() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    TableEnvironment tableEnv = new TableEnvironment();

    DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);

    Table in = tableEnv.fromDataSet(ds, "a, b as c, d");

    DataSet<Row> resultSet = tableEnv.toDataSet(in, Row.class);
    resultSet.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);

    env.execute();

    expected = "sorry bro";
  }
示例#4
0
  @Test(expected = ExpressionException.class)
  public void testAsWithAmbiguousFields() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    TableEnvironment tableEnv = new TableEnvironment();

    DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);

    Table in = tableEnv.fromDataSet(ds, "a, b, c, b");

    DataSet<Row> resultSet = tableEnv.toDataSet(in, Row.class);
    resultSet.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);

    env.execute();

    expected = " today's not your day ";
  }
示例#5
0
  @Test
  public void testPassingConfigurationObject() throws Exception {
    /*
     * Test passing configuration object.
     */

    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.getSmall3TupleDataSet(env);
    Configuration conf = new Configuration();
    conf.setInteger(testKey, testValue);
    DataSet<Tuple3<Integer, Long, String>> bcMapDs = ds.map(new RichMapper2()).withParameters(conf);
    List<Tuple3<Integer, Long, String>> result = bcMapDs.collect();

    String expected = "1,1,Hi\n" + "2,2,Hello\n" + "3,2,Hello world";

    compareResultAsTuples(result, expected);
  }
示例#6
0
  @Test
  public void testSimpleSelectAllWithAs() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    TableEnvironment tableEnv = new TableEnvironment();

    DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);

    Table in = tableEnv.fromDataSet(ds, "a,b,c");

    Table result = in.select("a, b, c");

    DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
    resultSet.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);

    env.execute();

    expected =
        "1,1,Hi\n"
            + "2,2,Hello\n"
            + "3,2,Hello world\n"
            + "4,3,Hello world, "
            + "how are you?\n"
            + "5,3,I am fine.\n"
            + "6,3,Luke Skywalker\n"
            + "7,4,"
            + "Comment#1\n"
            + "8,4,Comment#2\n"
            + "9,4,Comment#3\n"
            + "10,4,Comment#4\n"
            + "11,5,"
            + "Comment#5\n"
            + "12,5,Comment#6\n"
            + "13,5,Comment#7\n"
            + "14,5,Comment#8\n"
            + "15,5,"
            + "Comment#9\n"
            + "16,6,Comment#10\n"
            + "17,6,Comment#11\n"
            + "18,6,Comment#12\n"
            + "19,"
            + "6,Comment#13\n"
            + "20,6,Comment#14\n"
            + "21,6,Comment#15\n";
  }
示例#7
0
  @Test
  public void testSimpleSelectWithNaming() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    TableEnvironment tableEnv = new TableEnvironment();

    DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);

    Table in = tableEnv.fromDataSet(ds);

    Table result = in.select("f0 as a, f1 as b").select("a, b");

    DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
    resultSet.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);

    env.execute();

    expected =
        "1,1\n" + "2,2\n" + "3,2\n" + "4,3\n" + "5,3\n" + "6,3\n" + "7,4\n" + "8,4\n" + "9,4\n"
            + "10,4\n" + "11,5\n" + "12,5\n" + "13,5\n" + "14,5\n" + "15,5\n" + "16,6\n" + "17,6\n"
            + "18,6\n" + "19,6\n" + "20,6\n" + "21,6\n";
  }
示例#8
0
  @Test
  public void testIdentityMapWithTuple() throws Exception {
    /*
     * Test identity map with a tuple
     */

    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
    DataSet<Tuple3<Integer, Long, String>> identityMapDs = ds.map(new Mapper2());

    List<Tuple3<Integer, Long, String>> result = identityMapDs.collect();

    String expected =
        "1,1,Hi\n"
            + "2,2,Hello\n"
            + "3,2,Hello world\n"
            + "4,3,Hello world, how are you?\n"
            + "5,3,I am fine.\n"
            + "6,3,Luke Skywalker\n"
            + "7,4,Comment#1\n"
            + "8,4,Comment#2\n"
            + "9,4,Comment#3\n"
            + "10,4,Comment#4\n"
            + "11,5,Comment#5\n"
            + "12,5,Comment#6\n"
            + "13,5,Comment#7\n"
            + "14,5,Comment#8\n"
            + "15,5,Comment#9\n"
            + "16,6,Comment#10\n"
            + "17,6,Comment#11\n"
            + "18,6,Comment#12\n"
            + "19,6,Comment#13\n"
            + "20,6,Comment#14\n"
            + "21,6,Comment#15\n";

    compareResultAsTuples(result, expected);
  }
示例#9
0
  @Test
  public void test() throws Exception {
    /*
     * Test mapper if UDF returns input object - increment first field of a tuple
     */

    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
    DataSet<Tuple3<Integer, Long, String>> inputObjMapDs = ds.map(new Mapper7());

    List<Tuple3<Integer, Long, String>> result = inputObjMapDs.collect();

    String expected =
        "2,1,Hi\n"
            + "3,2,Hello\n"
            + "4,2,Hello world\n"
            + "5,3,Hello world, how are you?\n"
            + "6,3,I am fine.\n"
            + "7,3,Luke Skywalker\n"
            + "8,4,Comment#1\n"
            + "9,4,Comment#2\n"
            + "10,4,Comment#3\n"
            + "11,4,Comment#4\n"
            + "12,5,Comment#5\n"
            + "13,5,Comment#6\n"
            + "14,5,Comment#7\n"
            + "15,5,Comment#8\n"
            + "16,5,Comment#9\n"
            + "17,6,Comment#10\n"
            + "18,6,Comment#11\n"
            + "19,6,Comment#12\n"
            + "20,6,Comment#13\n"
            + "21,6,Comment#14\n"
            + "22,6,Comment#15\n";

    compareResultAsTuples(result, expected);
  }
示例#10
0
  @Test
  public void testMapperOnCustomLowercaseString() throws Exception {
    /*
     * Test mapper on Custom - lowercase myString
     */

    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
    DataSet<CustomType> customMapDs = ds.map(new Mapper6());

    List<CustomType> result = customMapDs.collect();

    String expected =
        "1,0,hi\n"
            + "2,1,hello\n"
            + "2,2,hello world\n"
            + "3,3,hello world, how are you?\n"
            + "3,4,i am fine.\n"
            + "3,5,luke skywalker\n"
            + "4,6,comment#1\n"
            + "4,7,comment#2\n"
            + "4,8,comment#3\n"
            + "4,9,comment#4\n"
            + "5,10,comment#5\n"
            + "5,11,comment#6\n"
            + "5,12,comment#7\n"
            + "5,13,comment#8\n"
            + "5,14,comment#9\n"
            + "6,15,comment#10\n"
            + "6,16,comment#11\n"
            + "6,17,comment#12\n"
            + "6,18,comment#13\n"
            + "6,19,comment#14\n"
            + "6,20,comment#15\n";

    compareResultAsText(result, expected);
  }
示例#11
0
  @Test
  public void testMapperOnTupleIncrementIntegerFieldReorderSecondAndThirdFields() throws Exception {
    /*
     * Test mapper on tuple - Increment Integer field, reorder second and third fields
     */

    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
    DataSet<Tuple3<Integer, String, Long>> tupleMapDs = ds.map(new Mapper5());

    List<Tuple3<Integer, String, Long>> result = tupleMapDs.collect();

    String expected =
        "2,Hi,1\n"
            + "3,Hello,2\n"
            + "4,Hello world,2\n"
            + "5,Hello world, how are you?,3\n"
            + "6,I am fine.,3\n"
            + "7,Luke Skywalker,3\n"
            + "8,Comment#1,4\n"
            + "9,Comment#2,4\n"
            + "10,Comment#3,4\n"
            + "11,Comment#4,4\n"
            + "12,Comment#5,5\n"
            + "13,Comment#6,5\n"
            + "14,Comment#7,5\n"
            + "15,Comment#8,5\n"
            + "16,Comment#9,5\n"
            + "17,Comment#10,6\n"
            + "18,Comment#11,6\n"
            + "19,Comment#12,6\n"
            + "20,Comment#13,6\n"
            + "21,Comment#14,6\n"
            + "22,Comment#15,6\n";

    compareResultAsTuples(result, expected);
  }
示例#12
0
  @Test
  public void testTypeConversionMapperTupleToBasic() throws Exception {
    /*
     * Test type conversion mapper (Tuple -> Basic)
     */

    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
    DataSet<String> typeConversionMapDs = ds.map(new Mapper4());

    List<String> result = typeConversionMapDs.collect();

    String expected =
        "Hi\n"
            + "Hello\n"
            + "Hello world\n"
            + "Hello world, how are you?\n"
            + "I am fine.\n"
            + "Luke Skywalker\n"
            + "Comment#1\n"
            + "Comment#2\n"
            + "Comment#3\n"
            + "Comment#4\n"
            + "Comment#5\n"
            + "Comment#6\n"
            + "Comment#7\n"
            + "Comment#8\n"
            + "Comment#9\n"
            + "Comment#10\n"
            + "Comment#11\n"
            + "Comment#12\n"
            + "Comment#13\n"
            + "Comment#14\n"
            + "Comment#15\n";

    compareResultAsText(result, expected);
  }
示例#13
0
  @Test
  public void testTypeConversionMapperCustomToTuple() throws Exception {
    /*
     * Test type conversion mapper (Custom -> Tuple)
     */

    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    DataSet<CustomType> ds = CollectionDataSets.getCustomTypeDataSet(env);
    DataSet<Tuple3<Integer, Long, String>> typeConversionMapDs = ds.map(new Mapper3());

    List<Tuple3<Integer, Long, String>> result = typeConversionMapDs.collect();

    String expected =
        "1,0,Hi\n"
            + "2,1,Hello\n"
            + "2,2,Hello world\n"
            + "3,3,Hello world, how are you?\n"
            + "3,4,I am fine.\n"
            + "3,5,Luke Skywalker\n"
            + "4,6,Comment#1\n"
            + "4,7,Comment#2\n"
            + "4,8,Comment#3\n"
            + "4,9,Comment#4\n"
            + "5,10,Comment#5\n"
            + "5,11,Comment#6\n"
            + "5,12,Comment#7\n"
            + "5,13,Comment#8\n"
            + "5,14,Comment#9\n"
            + "6,15,Comment#10\n"
            + "6,16,Comment#11\n"
            + "6,17,Comment#12\n"
            + "6,18,Comment#13\n"
            + "6,19,Comment#14\n"
            + "6,20,Comment#15\n";

    compareResultAsTuples(result, expected);
  }