Ejemplo n.º 1
0
  @Test
  public void shouldOptimizeMultipleStatelessRegions() {
    final OperatorDef stateful1 =
        OperatorDefBuilder.newInstance("stateful1", StatefulOperator.class).build();
    final OperatorDef stateful2 =
        OperatorDefBuilder.newInstance("stateful2", StatefulOperator.class).build();
    final OperatorDef stateless1 =
        OperatorDefBuilder.newInstance("stateless1", StatelessOperator.class).build();
    final OperatorDef stateless2 =
        OperatorDefBuilder.newInstance("stateless2", StatelessOperator.class).build();
    final OperatorDef stateless3 =
        OperatorDefBuilder.newInstance("stateless3", StatelessOperator.class).build();
    final OperatorDef stateful3 =
        OperatorDefBuilder.newInstance("stateful3", StatefulOperator.class).build();
    final OperatorDef stateful4 =
        OperatorDefBuilder.newInstance("stateful4", StatefulOperator.class).build();
    final OperatorDef stateless4 =
        OperatorDefBuilder.newInstance("stateless4", StatelessOperator2.class).build();
    final OperatorDef stateless5 =
        OperatorDefBuilder.newInstance("stateless5", StatelessOperator2.class).build();

    final FlowDef flow =
        new FlowDefBuilder()
            .add(stateful1)
            .add(stateful2)
            .add(stateless1)
            .add(stateless2)
            .add(stateless3)
            .add(stateful3)
            .add(stateful4)
            .add(stateless4)
            .add(stateless5)
            .connect("stateful1", "stateless1")
            .connect("stateful2", "stateless1")
            .connect("stateless1", "stateless2")
            .connect("stateless2", "stateless3")
            .connect("stateless3", "stateful3")
            .connect("stateless3", "stateful4")
            .connect("stateless3", "stateless4")
            .connect("stateless4", "stateless5")
            .build();

    final List<RegionDef> regions = regionDefFormer.createRegions(flow);
    final Collection<Entry<Port, Port>> connections = new HashSet<>(flow.getConnections());

    final Map<String, OperatorDef> operators = flow.getOperatorsMap();
    flowOptimizer.duplicateStatelessRegions(operators, connections, regions);

    final List<RegionDef> statelessRegions =
        regions.stream().filter(r -> r.getRegionType() == STATELESS).collect(toList());
    assertEquals(4, statelessRegions.size());
    assertStatelessRegions(
        stateless1, stateless2, stateless3, operators, connections, statelessRegions, true);
    assertStatelessRegions(stateless4, stateless5, operators, connections, statelessRegions);
  }
Ejemplo n.º 2
0
  @Test
  public void shouldMergePartitionedStatefulAndStatelessRegionsAfterOptimization() {
    final OperatorDef partitionedStateful1 =
        OperatorDefBuilder.newInstance("partitionedStateful1", PartitionedStatefulOperator.class)
            .setExtendingSchema(schema)
            .setPartitionFieldNames(partitionFieldNames)
            .build();

    final OperatorDef partitionedStateful2 =
        OperatorDefBuilder.newInstance("partitionedStateful2", PartitionedStatefulOperator.class)
            .setExtendingSchema(schema)
            .setPartitionFieldNames(partitionFieldNames)
            .build();

    final OperatorDef stateless =
        OperatorDefBuilder.newInstance("stateless", StatelessOperator.class)
            .setExtendingSchema(schema)
            .build();

    final FlowDef flow =
        new FlowDefBuilder()
            .add(partitionedStateful1)
            .add(partitionedStateful2)
            .add(stateless)
            .connect("partitionedStateful1", "stateless")
            .connect("partitionedStateful2", "stateless")
            .build();

    final List<RegionDef> regions = regionDefFormer.createRegions(flow);
    final Map<String, OperatorDef> operators = flow.getOperatorsMap();
    final Collection<Entry<Port, Port>> connections = flow.getConnections();
    flowOptimizer.duplicateStatelessRegions(operators, connections, regions);
    flowOptimizer.mergeRegions(operators, connections, regions);

    assertEquals(2, regions.size());
    final RegionDef region1 = regions.get(0);
    assertEquals(PARTITIONED_STATEFUL, region1.getRegionType());
    assertEquals(
        asList(PartitionedStatefulOperator.class, StatelessOperator.class),
        region1.getOperators().stream().map(OperatorDef::operatorClazz).collect(toList()));
    final RegionDef region2 = regions.get(1);
    assertEquals(PARTITIONED_STATEFUL, region2.getRegionType());
    assertEquals(
        asList(PartitionedStatefulOperator.class, StatelessOperator.class),
        region2.getOperators().stream().map(OperatorDef::operatorClazz).collect(toList()));
  }
Ejemplo n.º 3
0
  @Test
  public void
      shouldNotMergePartitionedStatefulAndStatelessRegionsWhenPartitionFieldNamesAreNotPresentInStatelessRegion() {
    final OperatorDef partitionedStateful =
        OperatorDefBuilder.newInstance("partitionedStateful", PartitionedStatefulOperator.class)
            .setExtendingSchema(schema)
            .setPartitionFieldNames(partitionFieldNames)
            .build();

    final OperatorRuntimeSchemaBuilder statelessSchema = new OperatorRuntimeSchemaBuilder(1, 1);

    final RuntimeSchemaField inputField =
        schema.getInputPortSchemaBuilder(0).getFields().iterator().next();
    final RuntimeSchemaField outputField =
        schema.getOutputPortSchemaBuilder(0).getFields().iterator().next();

    statelessSchema.addInputField(0, inputField.getName(), inputField.getType());
    statelessSchema.addOutputField(0, outputField.getName(), outputField.getType());

    final OperatorDef stateless =
        OperatorDefBuilder.newInstance("stateless", StatelessOperator.class)
            .setExtendingSchema(statelessSchema)
            .build();

    final FlowDef flow =
        new FlowDefBuilder()
            .add(partitionedStateful)
            .add(stateless)
            .connect("partitionedStateful", "stateless")
            .build();

    final List<RegionDef> regions = new ArrayList<>();
    regions.add(
        new RegionDef(
            0,
            PARTITIONED_STATEFUL,
            partitionedStateful.partitionFieldNames(),
            singletonList(partitionedStateful)));
    regions.add(new RegionDef(1, STATELESS, emptyList(), singletonList(stateless)));

    flowOptimizer.mergeRegions(flow.getOperatorsMap(), flow.getConnections(), regions);

    assertEquals(2, regions.size());
  }
Ejemplo n.º 4
0
  @Test
  public void shouldMergeStatefulAndStatelessRegions() {
    final OperatorDef stateful =
        OperatorDefBuilder.newInstance("stateful", StatefulOperator.class).build();
    final OperatorDef stateless =
        OperatorDefBuilder.newInstance("stateless", StatelessOperator.class).build();

    final FlowDef flow =
        new FlowDefBuilder().add(stateful).add(stateless).connect("stateful", "stateless").build();

    final List<RegionDef> regions = regionDefFormer.createRegions(flow);

    flowOptimizer.mergeRegions(flow.getOperatorsMap(), flow.getConnections(), regions);

    assertEquals(1, regions.size());
    final RegionDef region = regions.get(0);
    assertEquals(STATEFUL, region.getRegionType());
    assertEquals(asList(stateful, stateless), region.getOperators());
  }