@Test public void shouldOptimizeMultipleStatelessRegions() { final OperatorDef stateful1 = OperatorDefBuilder.newInstance("stateful1", StatefulOperator.class).build(); final OperatorDef stateful2 = OperatorDefBuilder.newInstance("stateful2", StatefulOperator.class).build(); final OperatorDef stateless1 = OperatorDefBuilder.newInstance("stateless1", StatelessOperator.class).build(); final OperatorDef stateless2 = OperatorDefBuilder.newInstance("stateless2", StatelessOperator.class).build(); final OperatorDef stateless3 = OperatorDefBuilder.newInstance("stateless3", StatelessOperator.class).build(); final OperatorDef stateful3 = OperatorDefBuilder.newInstance("stateful3", StatefulOperator.class).build(); final OperatorDef stateful4 = OperatorDefBuilder.newInstance("stateful4", StatefulOperator.class).build(); final OperatorDef stateless4 = OperatorDefBuilder.newInstance("stateless4", StatelessOperator2.class).build(); final OperatorDef stateless5 = OperatorDefBuilder.newInstance("stateless5", StatelessOperator2.class).build(); final FlowDef flow = new FlowDefBuilder() .add(stateful1) .add(stateful2) .add(stateless1) .add(stateless2) .add(stateless3) .add(stateful3) .add(stateful4) .add(stateless4) .add(stateless5) .connect("stateful1", "stateless1") .connect("stateful2", "stateless1") .connect("stateless1", "stateless2") .connect("stateless2", "stateless3") .connect("stateless3", "stateful3") .connect("stateless3", "stateful4") .connect("stateless3", "stateless4") .connect("stateless4", "stateless5") .build(); final List<RegionDef> regions = regionDefFormer.createRegions(flow); final Collection<Entry<Port, Port>> connections = new HashSet<>(flow.getConnections()); final Map<String, OperatorDef> operators = flow.getOperatorsMap(); flowOptimizer.duplicateStatelessRegions(operators, connections, regions); final List<RegionDef> statelessRegions = regions.stream().filter(r -> r.getRegionType() == STATELESS).collect(toList()); assertEquals(4, statelessRegions.size()); assertStatelessRegions( stateless1, stateless2, stateless3, operators, connections, statelessRegions, true); assertStatelessRegions(stateless4, stateless5, operators, connections, statelessRegions); }
@Test public void shouldMergePartitionedStatefulAndStatelessRegionsAfterOptimization() { final OperatorDef partitionedStateful1 = OperatorDefBuilder.newInstance("partitionedStateful1", PartitionedStatefulOperator.class) .setExtendingSchema(schema) .setPartitionFieldNames(partitionFieldNames) .build(); final OperatorDef partitionedStateful2 = OperatorDefBuilder.newInstance("partitionedStateful2", PartitionedStatefulOperator.class) .setExtendingSchema(schema) .setPartitionFieldNames(partitionFieldNames) .build(); final OperatorDef stateless = OperatorDefBuilder.newInstance("stateless", StatelessOperator.class) .setExtendingSchema(schema) .build(); final FlowDef flow = new FlowDefBuilder() .add(partitionedStateful1) .add(partitionedStateful2) .add(stateless) .connect("partitionedStateful1", "stateless") .connect("partitionedStateful2", "stateless") .build(); final List<RegionDef> regions = regionDefFormer.createRegions(flow); final Map<String, OperatorDef> operators = flow.getOperatorsMap(); final Collection<Entry<Port, Port>> connections = flow.getConnections(); flowOptimizer.duplicateStatelessRegions(operators, connections, regions); flowOptimizer.mergeRegions(operators, connections, regions); assertEquals(2, regions.size()); final RegionDef region1 = regions.get(0); assertEquals(PARTITIONED_STATEFUL, region1.getRegionType()); assertEquals( asList(PartitionedStatefulOperator.class, StatelessOperator.class), region1.getOperators().stream().map(OperatorDef::operatorClazz).collect(toList())); final RegionDef region2 = regions.get(1); assertEquals(PARTITIONED_STATEFUL, region2.getRegionType()); assertEquals( asList(PartitionedStatefulOperator.class, StatelessOperator.class), region2.getOperators().stream().map(OperatorDef::operatorClazz).collect(toList())); }
@Test public void shouldNotMergePartitionedStatefulAndStatelessRegionsWhenPartitionFieldNamesAreNotPresentInStatelessRegion() { final OperatorDef partitionedStateful = OperatorDefBuilder.newInstance("partitionedStateful", PartitionedStatefulOperator.class) .setExtendingSchema(schema) .setPartitionFieldNames(partitionFieldNames) .build(); final OperatorRuntimeSchemaBuilder statelessSchema = new OperatorRuntimeSchemaBuilder(1, 1); final RuntimeSchemaField inputField = schema.getInputPortSchemaBuilder(0).getFields().iterator().next(); final RuntimeSchemaField outputField = schema.getOutputPortSchemaBuilder(0).getFields().iterator().next(); statelessSchema.addInputField(0, inputField.getName(), inputField.getType()); statelessSchema.addOutputField(0, outputField.getName(), outputField.getType()); final OperatorDef stateless = OperatorDefBuilder.newInstance("stateless", StatelessOperator.class) .setExtendingSchema(statelessSchema) .build(); final FlowDef flow = new FlowDefBuilder() .add(partitionedStateful) .add(stateless) .connect("partitionedStateful", "stateless") .build(); final List<RegionDef> regions = new ArrayList<>(); regions.add( new RegionDef( 0, PARTITIONED_STATEFUL, partitionedStateful.partitionFieldNames(), singletonList(partitionedStateful))); regions.add(new RegionDef(1, STATELESS, emptyList(), singletonList(stateless))); flowOptimizer.mergeRegions(flow.getOperatorsMap(), flow.getConnections(), regions); assertEquals(2, regions.size()); }
@Test public void shouldMergeStatefulAndStatelessRegions() { final OperatorDef stateful = OperatorDefBuilder.newInstance("stateful", StatefulOperator.class).build(); final OperatorDef stateless = OperatorDefBuilder.newInstance("stateless", StatelessOperator.class).build(); final FlowDef flow = new FlowDefBuilder().add(stateful).add(stateless).connect("stateful", "stateless").build(); final List<RegionDef> regions = regionDefFormer.createRegions(flow); flowOptimizer.mergeRegions(flow.getOperatorsMap(), flow.getConnections(), regions); assertEquals(1, regions.size()); final RegionDef region = regions.get(0); assertEquals(STATEFUL, region.getRegionType()); assertEquals(asList(stateful, stateless), region.getOperators()); }