@Test public void testFieldsAggregate() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo); // should work try { tupleDs.aggregate(Aggregations.SUM, 1); } catch (Exception e) { Assert.fail(); } // should not work: index out of bounds try { tupleDs.aggregate(Aggregations.SUM, 10); Assert.fail(); } catch (IllegalArgumentException iae) { // we're good here } catch (Exception e) { Assert.fail(); } // should not work: not applied to tuple dataset DataSet<Long> longDs = env.fromCollection(emptyLongData, BasicTypeInfo.LONG_TYPE_INFO); try { longDs.aggregate(Aggregations.MIN, 1); Assert.fail(); } catch (InvalidProgramException uoe) { // we're good here } catch (Exception e) { Assert.fail(); } }
@Test(expected = IllegalArgumentException.class) public void testRightOuter7() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // invalid key position ds1.rightOuterJoin(ds2).where(5).equalTo(0).with(new DummyJoin()); }
@Test(expected = CompositeType.InvalidFieldReferenceException.class) public void testRightOuter8() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // invalid key reference ds1.rightOuterJoin(ds2).where(1).equalTo("f5").with(new DummyJoin()); }
@Test(expected = InvalidProgramException.class) public void testRightOuter9() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // key types do not match ds1.rightOuterJoin(ds2).where(0).equalTo(1).with(new DummyJoin()); }
private void testRightOuterStrategies(JoinHint hint) { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // should work ds1.rightOuterJoin(ds2, hint).where(0).equalTo(4).with(new DummyJoin()); }
@Test public void testRightOuter4() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // should work ds1.rightOuterJoin(ds2).where(0).equalTo(new IntKeySelector()).with(new DummyJoin()); }
@Test(expected = IllegalArgumentException.class) public void testJoinKeyFields6() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<CustomType> ds2 = env.fromCollection(customTypeData); // should not work, join key fields on custom type ds1.join(ds2).where(5).equalTo(0); }
@Test(expected = IndexOutOfBoundsException.class) public void testJoinProjection14() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // should not work, index out of range ds1.join(ds2).where(0).equalTo(0).projectFirst(0).projectSecond(5).types(Integer.class); }
@Test(expected = IllegalArgumentException.class) public void testJoinKeyFields5() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // should not work, negative key field position ds1.join(ds2).where(-1).equalTo(-1); }
@Test(expected = InvalidProgramException.class) public void testJoinKeyFields3() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // should not work, incompatible number of join keys ds1.join(ds2).where(0, 1).equalTo(2); }
@Test(expected = IllegalArgumentException.class) public void testJoinProjection12() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // should not work, number of types and fields does not match ds1.join(ds2).where(0).equalTo(0).projectSecond(2).projectFirst(1).types(String.class); }
@Test public void testAggregationTypes() { try { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo); // should work: multiple aggregates tupleDs.aggregate(Aggregations.SUM, 0).and(Aggregations.MIN, 4); // should work: nested aggregates tupleDs.aggregate(Aggregations.MIN, 2).aggregate(Aggregations.SUM, 1); // should not work: average on string try { tupleDs.aggregate(Aggregations.SUM, 2); Assert.fail(); } catch (UnsupportedAggregationTypeException iae) { // we're good here } } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail(e.getMessage()); } }
@Test public void testJoinKeySelectors1() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<CustomType> ds1 = env.fromCollection(customTypeData); DataSet<CustomType> ds2 = env.fromCollection(customTypeData); // should work try { ds1.join(ds2) .where( new KeySelector<CustomType, Long>() { @Override public Long getKey(CustomType value) { return value.myLong; } }) .equalTo( new KeySelector<CustomType, Long>() { @Override public Long getKey(CustomType value) { return value.myLong; } }); } catch (Exception e) { Assert.fail(); } }
@Test public void testJoinKeyFields1() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // should work try { ds1.join(ds2).where(0).equalTo(0); } catch (Exception e) { Assert.fail(); } }
@Test public void testJoinProjection6() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<CustomType> ds1 = env.fromCollection(customTypeData); DataSet<CustomType> ds2 = env.fromCollection(customTypeData); // should work try { ds1.join(ds2) .where( new KeySelector<CustomType, Long>() { @Override public Long getKey(CustomType value) { return value.myLong; } }) .equalTo( new KeySelector<CustomType, Long>() { @Override public Long getKey(CustomType value) { return value.myLong; } }) .projectFirst() .projectSecond() .types(CustomType.class, CustomType.class); } catch (Exception e) { System.out.println("FAILED: " + e); Assert.fail(); } }
@Test(expected = InvalidProgramException.class) public void testJoinKeyExpressions3() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<CustomType> ds1 = env.fromCollection(customTypeData); DataSet<CustomType> ds2 = env.fromCollection(customTypeData); // should not work, incompatible number of join keys ds1.join(ds2).where("myInt", "myString").equalTo("myString"); }
@Test(expected = IllegalArgumentException.class) public void testJoinKeyExpressions4() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<CustomType> ds1 = env.fromCollection(customTypeData); DataSet<CustomType> ds2 = env.fromCollection(customTypeData); // should not work, join key non-existent ds1.join(ds2).where("myNonExistent").equalTo("myInt"); }
@Test(expected = InvalidProgramException.class) public void testJoinKeyMixing4() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<CustomType> ds2 = env.fromCollection(customTypeData); // should not work, more than one key field position ds1.join(ds2) .where(1, 3) .equalTo( new KeySelector<CustomType, Long>() { @Override public Long getKey(CustomType value) { return value.myLong; } }); }
@Test public void testJoinProjection7() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo); // should work try { ds1.join(ds2) .where(0) .equalTo(0) .projectSecond() .projectFirst(1, 4) .types(Tuple5.class, Long.class, Integer.class); } catch (Exception e) { Assert.fail(); } }
@Test public void testJoinKeyExpressions1() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<CustomType> ds1 = env.fromCollection(customTypeData); DataSet<CustomType> ds2 = env.fromCollection(customTypeData); // should work try { ds1.join(ds2).where("myInt").equalTo("myInt"); } catch (Exception e) { Assert.fail(); } }
@Test public void testJoinKeyMixing2() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo); DataSet<CustomType> ds2 = env.fromCollection(customTypeData); // should work try { ds1.join(ds2) .where(3) .equalTo( new KeySelector<CustomType, Long>() { @Override public Long getKey(CustomType value) { return value.myLong; } }); } catch (Exception e) { Assert.fail(); } }
public static DataSet<Edge<Long, Double>> getDefaultEdgeDataSet(ExecutionEnvironment env) { List<Edge<Long, Double>> edges = new ArrayList<Edge<Long, Double>>(); edges.add(new Edge<Long, Double>(1L, 2L, new Double(0))); edges.add(new Edge<Long, Double>(1L, 3L, new Double(0))); edges.add(new Edge<Long, Double>(1L, 4L, new Double(0))); edges.add(new Edge<Long, Double>(1L, 5L, new Double(0))); edges.add(new Edge<Long, Double>(2L, 3L, new Double(0))); edges.add(new Edge<Long, Double>(2L, 4L, new Double(0))); edges.add(new Edge<Long, Double>(2L, 5L, new Double(0))); edges.add(new Edge<Long, Double>(3L, 4L, new Double(0))); edges.add(new Edge<Long, Double>(3L, 5L, new Double(0))); edges.add(new Edge<Long, Double>(4L, 5L, new Double(0))); return env.fromCollection(edges); }