@Test
public void testFieldsAggregate() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs =
env.fromCollection(emptyTupleData, tupleTypeInfo);
// should work
try {
tupleDs.aggregate(Aggregations.SUM, 1);
} catch (Exception e) {
Assert.fail();
}
// should not work: index out of bounds
try {
tupleDs.aggregate(Aggregations.SUM, 10);
Assert.fail();
} catch (IllegalArgumentException iae) {
// we're good here
} catch (Exception e) {
Assert.fail();
}
// should not work: not applied to tuple dataset
DataSet<Long> longDs = env.fromCollection(emptyLongData, BasicTypeInfo.LONG_TYPE_INFO);
try {
longDs.aggregate(Aggregations.MIN, 1);
Assert.fail();
} catch (InvalidProgramException uoe) {
// we're good here
} catch (Exception e) {
Assert.fail();
}
}
@Test(expected = IllegalArgumentException.class)
public void testRightOuter7() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
// invalid key position
ds1.rightOuterJoin(ds2).where(5).equalTo(0).with(new DummyJoin());
}
@Test(expected = CompositeType.InvalidFieldReferenceException.class)
public void testRightOuter8() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
// invalid key reference
ds1.rightOuterJoin(ds2).where(1).equalTo("f5").with(new DummyJoin());
}
@Test(expected = InvalidProgramException.class)
public void testRightOuter9() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
// key types do not match
ds1.rightOuterJoin(ds2).where(0).equalTo(1).with(new DummyJoin());
}
private void testRightOuterStrategies(JoinHint hint) {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
// should work
ds1.rightOuterJoin(ds2, hint).where(0).equalTo(4).with(new DummyJoin());
}
@Test
public void testRightOuter4() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
// should work
ds1.rightOuterJoin(ds2).where(0).equalTo(new IntKeySelector()).with(new DummyJoin());
}
@Test(expected = IllegalArgumentException.class)
public void testJoinKeyFields6() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, join key fields on custom type
ds1.join(ds2).where(5).equalTo(0);
}
@Test(expected = IndexOutOfBoundsException.class)
public void testJoinProjection14() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
// should not work, index out of range
ds1.join(ds2).where(0).equalTo(0).projectFirst(0).projectSecond(5).types(Integer.class);
}
@Test(expected = IllegalArgumentException.class)
public void testJoinKeyFields5() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
// should not work, negative key field position
ds1.join(ds2).where(-1).equalTo(-1);
}
@Test(expected = InvalidProgramException.class)
public void testJoinKeyFields3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
// should not work, incompatible number of join keys
ds1.join(ds2).where(0, 1).equalTo(2);
}
@Test(expected = IllegalArgumentException.class)
public void testJoinProjection12() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
// should not work, number of types and fields does not match
ds1.join(ds2).where(0).equalTo(0).projectSecond(2).projectFirst(1).types(String.class);
}
@Test
public void testAggregationTypes() {
try {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs =
env.fromCollection(emptyTupleData, tupleTypeInfo);
// should work: multiple aggregates
tupleDs.aggregate(Aggregations.SUM, 0).and(Aggregations.MIN, 4);
// should work: nested aggregates
tupleDs.aggregate(Aggregations.MIN, 2).aggregate(Aggregations.SUM, 1);
// should not work: average on string
try {
tupleDs.aggregate(Aggregations.SUM, 2);
Assert.fail();
} catch (UnsupportedAggregationTypeException iae) {
// we're good here
}
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
@Test
public void testJoinKeySelectors1() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should work
try {
ds1.join(ds2)
.where(
new KeySelector<CustomType, Long>() {
@Override
public Long getKey(CustomType value) {
return value.myLong;
}
})
.equalTo(
new KeySelector<CustomType, Long>() {
@Override
public Long getKey(CustomType value) {
return value.myLong;
}
});
} catch (Exception e) {
Assert.fail();
}
}
@Test
public void testJoinKeyFields1() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
// should work
try {
ds1.join(ds2).where(0).equalTo(0);
} catch (Exception e) {
Assert.fail();
}
}
@Test
public void testJoinProjection6() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should work
try {
ds1.join(ds2)
.where(
new KeySelector<CustomType, Long>() {
@Override
public Long getKey(CustomType value) {
return value.myLong;
}
})
.equalTo(
new KeySelector<CustomType, Long>() {
@Override
public Long getKey(CustomType value) {
return value.myLong;
}
})
.projectFirst()
.projectSecond()
.types(CustomType.class, CustomType.class);
} catch (Exception e) {
System.out.println("FAILED: " + e);
Assert.fail();
}
}
@Test(expected = InvalidProgramException.class)
public void testJoinKeyExpressions3() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, incompatible number of join keys
ds1.join(ds2).where("myInt", "myString").equalTo("myString");
}
@Test(expected = IllegalArgumentException.class)
public void testJoinKeyExpressions4() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, join key non-existent
ds1.join(ds2).where("myNonExistent").equalTo("myInt");
}
@Test(expected = InvalidProgramException.class)
public void testJoinKeyMixing4() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should not work, more than one key field position
ds1.join(ds2)
.where(1, 3)
.equalTo(
new KeySelector<CustomType, Long>() {
@Override
public Long getKey(CustomType value) {
return value.myLong;
}
});
}
@Test
public void testJoinProjection7() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
// should work
try {
ds1.join(ds2)
.where(0)
.equalTo(0)
.projectSecond()
.projectFirst(1, 4)
.types(Tuple5.class, Long.class, Integer.class);
} catch (Exception e) {
Assert.fail();
}
}
@Test
public void testJoinKeyExpressions1() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should work
try {
ds1.join(ds2).where("myInt").equalTo("myInt");
} catch (Exception e) {
Assert.fail();
}
}
@Test
public void testJoinKeyMixing2() {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 =
env.fromCollection(emptyTupleData, tupleTypeInfo);
DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
// should work
try {
ds1.join(ds2)
.where(3)
.equalTo(
new KeySelector<CustomType, Long>() {
@Override
public Long getKey(CustomType value) {
return value.myLong;
}
});
} catch (Exception e) {
Assert.fail();
}
}
public static DataSet<Edge<Long, Double>> getDefaultEdgeDataSet(ExecutionEnvironment env) {
List<Edge<Long, Double>> edges = new ArrayList<Edge<Long, Double>>();
edges.add(new Edge<Long, Double>(1L, 2L, new Double(0)));
edges.add(new Edge<Long, Double>(1L, 3L, new Double(0)));
edges.add(new Edge<Long, Double>(1L, 4L, new Double(0)));
edges.add(new Edge<Long, Double>(1L, 5L, new Double(0)));
edges.add(new Edge<Long, Double>(2L, 3L, new Double(0)));
edges.add(new Edge<Long, Double>(2L, 4L, new Double(0)));
edges.add(new Edge<Long, Double>(2L, 5L, new Double(0)));
edges.add(new Edge<Long, Double>(3L, 4L, new Double(0)));
edges.add(new Edge<Long, Double>(3L, 5L, new Double(0)));
edges.add(new Edge<Long, Double>(4L, 5L, new Double(0)));
return env.fromCollection(edges);
}