@Test
public void shouldOptimizeMultipleStatelessRegions() {
final OperatorDef stateful1 =
OperatorDefBuilder.newInstance("stateful1", StatefulOperator.class).build();
final OperatorDef stateful2 =
OperatorDefBuilder.newInstance("stateful2", StatefulOperator.class).build();
final OperatorDef stateless1 =
OperatorDefBuilder.newInstance("stateless1", StatelessOperator.class).build();
final OperatorDef stateless2 =
OperatorDefBuilder.newInstance("stateless2", StatelessOperator.class).build();
final OperatorDef stateless3 =
OperatorDefBuilder.newInstance("stateless3", StatelessOperator.class).build();
final OperatorDef stateful3 =
OperatorDefBuilder.newInstance("stateful3", StatefulOperator.class).build();
final OperatorDef stateful4 =
OperatorDefBuilder.newInstance("stateful4", StatefulOperator.class).build();
final OperatorDef stateless4 =
OperatorDefBuilder.newInstance("stateless4", StatelessOperator2.class).build();
final OperatorDef stateless5 =
OperatorDefBuilder.newInstance("stateless5", StatelessOperator2.class).build();
final FlowDef flow =
new FlowDefBuilder()
.add(stateful1)
.add(stateful2)
.add(stateless1)
.add(stateless2)
.add(stateless3)
.add(stateful3)
.add(stateful4)
.add(stateless4)
.add(stateless5)
.connect("stateful1", "stateless1")
.connect("stateful2", "stateless1")
.connect("stateless1", "stateless2")
.connect("stateless2", "stateless3")
.connect("stateless3", "stateful3")
.connect("stateless3", "stateful4")
.connect("stateless3", "stateless4")
.connect("stateless4", "stateless5")
.build();
final List<RegionDef> regions = regionDefFormer.createRegions(flow);
final Collection<Entry<Port, Port>> connections = new HashSet<>(flow.getConnections());
final Map<String, OperatorDef> operators = flow.getOperatorsMap();
flowOptimizer.duplicateStatelessRegions(operators, connections, regions);
final List<RegionDef> statelessRegions =
regions.stream().filter(r -> r.getRegionType() == STATELESS).collect(toList());
assertEquals(4, statelessRegions.size());
assertStatelessRegions(
stateless1, stateless2, stateless3, operators, connections, statelessRegions, true);
assertStatelessRegions(stateless4, stateless5, operators, connections, statelessRegions);
}
@Test
public void shouldMergePartitionedStatefulAndStatelessRegionsAfterOptimization() {
final OperatorDef partitionedStateful1 =
OperatorDefBuilder.newInstance("partitionedStateful1", PartitionedStatefulOperator.class)
.setExtendingSchema(schema)
.setPartitionFieldNames(partitionFieldNames)
.build();
final OperatorDef partitionedStateful2 =
OperatorDefBuilder.newInstance("partitionedStateful2", PartitionedStatefulOperator.class)
.setExtendingSchema(schema)
.setPartitionFieldNames(partitionFieldNames)
.build();
final OperatorDef stateless =
OperatorDefBuilder.newInstance("stateless", StatelessOperator.class)
.setExtendingSchema(schema)
.build();
final FlowDef flow =
new FlowDefBuilder()
.add(partitionedStateful1)
.add(partitionedStateful2)
.add(stateless)
.connect("partitionedStateful1", "stateless")
.connect("partitionedStateful2", "stateless")
.build();
final List<RegionDef> regions = regionDefFormer.createRegions(flow);
final Map<String, OperatorDef> operators = flow.getOperatorsMap();
final Collection<Entry<Port, Port>> connections = flow.getConnections();
flowOptimizer.duplicateStatelessRegions(operators, connections, regions);
flowOptimizer.mergeRegions(operators, connections, regions);
assertEquals(2, regions.size());
final RegionDef region1 = regions.get(0);
assertEquals(PARTITIONED_STATEFUL, region1.getRegionType());
assertEquals(
asList(PartitionedStatefulOperator.class, StatelessOperator.class),
region1.getOperators().stream().map(OperatorDef::operatorClazz).collect(toList()));
final RegionDef region2 = regions.get(1);
assertEquals(PARTITIONED_STATEFUL, region2.getRegionType());
assertEquals(
asList(PartitionedStatefulOperator.class, StatelessOperator.class),
region2.getOperators().stream().map(OperatorDef::operatorClazz).collect(toList()));
}
@Test
public void
shouldNotMergePartitionedStatefulAndStatelessRegionsWhenPartitionFieldNamesAreNotPresentInStatelessRegion() {
final OperatorDef partitionedStateful =
OperatorDefBuilder.newInstance("partitionedStateful", PartitionedStatefulOperator.class)
.setExtendingSchema(schema)
.setPartitionFieldNames(partitionFieldNames)
.build();
final OperatorRuntimeSchemaBuilder statelessSchema = new OperatorRuntimeSchemaBuilder(1, 1);
final RuntimeSchemaField inputField =
schema.getInputPortSchemaBuilder(0).getFields().iterator().next();
final RuntimeSchemaField outputField =
schema.getOutputPortSchemaBuilder(0).getFields().iterator().next();
statelessSchema.addInputField(0, inputField.getName(), inputField.getType());
statelessSchema.addOutputField(0, outputField.getName(), outputField.getType());
final OperatorDef stateless =
OperatorDefBuilder.newInstance("stateless", StatelessOperator.class)
.setExtendingSchema(statelessSchema)
.build();
final FlowDef flow =
new FlowDefBuilder()
.add(partitionedStateful)
.add(stateless)
.connect("partitionedStateful", "stateless")
.build();
final List<RegionDef> regions = new ArrayList<>();
regions.add(
new RegionDef(
0,
PARTITIONED_STATEFUL,
partitionedStateful.partitionFieldNames(),
singletonList(partitionedStateful)));
regions.add(new RegionDef(1, STATELESS, emptyList(), singletonList(stateless)));
flowOptimizer.mergeRegions(flow.getOperatorsMap(), flow.getConnections(), regions);
assertEquals(2, regions.size());
}
@Test
public void shouldMergeStatefulAndStatelessRegions() {
final OperatorDef stateful =
OperatorDefBuilder.newInstance("stateful", StatefulOperator.class).build();
final OperatorDef stateless =
OperatorDefBuilder.newInstance("stateless", StatelessOperator.class).build();
final FlowDef flow =
new FlowDefBuilder().add(stateful).add(stateless).connect("stateful", "stateless").build();
final List<RegionDef> regions = regionDefFormer.createRegions(flow);
flowOptimizer.mergeRegions(flow.getOperatorsMap(), flow.getConnections(), regions);
assertEquals(1, regions.size());
final RegionDef region = regions.get(0);
assertEquals(STATEFUL, region.getRegionType());
assertEquals(asList(stateful, stateless), region.getOperators());
}