private void assertStatelessRegions(
final OperatorDef stateless1,
final OperatorDef stateless2,
final Map<String, OperatorDef> optimizedOperators,
final Collection<Entry<Port, Port>> optimizedConnections,
final List<RegionDef> statelessRegions) {
for (RegionDef region : statelessRegions) {
if (region.getOperatorCount() != 2) {
continue;
}
final List<OperatorDef> operators = region.getOperators();
final OperatorDef optimizedStateless1 = operators.get(0);
final OperatorDef optimizedStateless2 = operators.get(1);
assertEquals(StatelessOperator2.class, optimizedStateless1.operatorClazz());
assertEquals(StatelessOperator2.class, optimizedStateless2.operatorClazz());
assertTrue(optimizedOperators.containsKey(optimizedStateless1.id()));
assertTrue(optimizedOperators.containsKey(optimizedStateless2.id()));
assertTrue(
optimizedConnections.contains(
new SimpleEntry<>(
new Port(optimizedStateless1.id(), 0), new Port(optimizedStateless2.id(), 0))));
}
assertFalse(optimizedOperators.containsKey(stateless1.id()));
assertFalse(optimizedOperators.containsKey(stateless2.id()));
}
@Test
public void
shouldNotMergePartitionedStatefulAndStatelessRegionsWhenPartitionFieldNamesAreNotPresentInStatelessRegion() {
final OperatorDef partitionedStateful =
OperatorDefBuilder.newInstance("partitionedStateful", PartitionedStatefulOperator.class)
.setExtendingSchema(schema)
.setPartitionFieldNames(partitionFieldNames)
.build();
final OperatorRuntimeSchemaBuilder statelessSchema = new OperatorRuntimeSchemaBuilder(1, 1);
final RuntimeSchemaField inputField =
schema.getInputPortSchemaBuilder(0).getFields().iterator().next();
final RuntimeSchemaField outputField =
schema.getOutputPortSchemaBuilder(0).getFields().iterator().next();
statelessSchema.addInputField(0, inputField.getName(), inputField.getType());
statelessSchema.addOutputField(0, outputField.getName(), outputField.getType());
final OperatorDef stateless =
OperatorDefBuilder.newInstance("stateless", StatelessOperator.class)
.setExtendingSchema(statelessSchema)
.build();
final FlowDef flow =
new FlowDefBuilder()
.add(partitionedStateful)
.add(stateless)
.connect("partitionedStateful", "stateless")
.build();
final List<RegionDef> regions = new ArrayList<>();
regions.add(
new RegionDef(
0,
PARTITIONED_STATEFUL,
partitionedStateful.partitionFieldNames(),
singletonList(partitionedStateful)));
regions.add(new RegionDef(1, STATELESS, emptyList(), singletonList(stateless)));
flowOptimizer.mergeRegions(flow.getOperatorsMap(), flow.getConnections(), regions);
assertEquals(2, regions.size());
}
void verifyOrFail(final OperatorDef operatorDef, final SchedulingStrategy schedulingStrategy) {
if (schedulingStrategy instanceof ScheduleWhenAvailable) {
checkState(
operatorDef.inputPortCount() == 0,
"%s cannot be used by operator: %s with input port count: %s",
ScheduleWhenAvailable.class.getSimpleName(),
operatorDef.id(),
operatorDef.inputPortCount());
checkState(
version == 0,
"upstream context is closed for 0 input port operator: %s",
operatorDef.id());
} else if (schedulingStrategy instanceof ScheduleWhenTuplesAvailable) {
checkState(
operatorDef.inputPortCount() > 0,
"0 input port operator: %s cannot use %s",
operatorDef.id(),
ScheduleWhenTuplesAvailable.class.getSimpleName());
final ScheduleWhenTuplesAvailable s = (ScheduleWhenTuplesAvailable) schedulingStrategy;
if (s.getTupleAvailabilityByPort() == ANY_PORT) {
for (int i = 0; i < operatorDef.inputPortCount(); i++) {
if (s.getTupleCount(i) > 0 && getUpstreamConnectionStatus(i) == ACTIVE) {
return;
}
}
throw new IllegalStateException(
"SchedulingStrategy " + s + " is not invokable anymore since there is no open port");
} else if (s.getTupleAvailabilityByPort() == ALL_PORTS) {
for (int i = 0; i < operatorDef.inputPortCount(); i++) {
checkState(
getUpstreamConnectionStatus(i) == ACTIVE,
"SchedulingStrategy %s is not invokable anymore since there is closed port",
s);
}
} else {
throw new IllegalStateException(s.toString());
}
} else {
throw new IllegalStateException(
operatorDef.id() + " returns invalid initial scheduling strategy: " + schedulingStrategy);
}
}
boolean isInvokable(final OperatorDef operatorDef, final SchedulingStrategy schedulingStrategy) {
try {
verifyOrFail(operatorDef, schedulingStrategy);
return true;
} catch (IllegalStateException e) {
LOGGER.info(
"{} not invokable anymore. scheduling strategy: {} upstream context: {} error: {}",
operatorDef.id(),
schedulingStrategy,
this,
e.getMessage());
return false;
}
}
private void assertStatelessRegions(
final OperatorDef stateless1,
final OperatorDef stateless2,
final OperatorDef stateless3,
final Map<String, OperatorDef> optimizedOperators,
final Collection<Entry<Port, Port>> optimizedConnections,
final List<RegionDef> statelessRegions,
final boolean expectedNonMatchingStatelessRegion) {
boolean stateful1ConnectionExists = false, stateful2ConnectionExists = false;
for (RegionDef region : statelessRegions) {
if (region.getOperatorCount() != 3) {
assertTrue(expectedNonMatchingStatelessRegion);
continue;
}
final List<OperatorDef> operators = region.getOperators();
final OperatorDef optimizedStateless1 = operators.get(0);
final OperatorDef optimizedStateless2 = operators.get(1);
final OperatorDef optimizedStateless3 = operators.get(2);
assertEquals(StatelessOperator.class, optimizedStateless1.operatorClazz());
assertEquals(StatelessOperator.class, optimizedStateless2.operatorClazz());
assertEquals(StatelessOperator.class, optimizedStateless3.operatorClazz());
assertTrue(optimizedOperators.containsKey(optimizedStateless1.id()));
assertTrue(optimizedOperators.containsKey(optimizedStateless2.id()));
assertTrue(optimizedOperators.containsKey(optimizedStateless3.id()));
final Collection<Port> stateful1DownstreamConnections =
getDownstream(optimizedConnections, new Port("stateful1", 0));
final Collection<Port> stateful2DownstreamConnections =
getDownstream(optimizedConnections, new Port("stateful2", 0));
assertEquals(1, stateful1DownstreamConnections.size());
assertEquals(1, stateful2DownstreamConnections.size());
final Port stateful1DownstreamPort = stateful1DownstreamConnections.iterator().next();
if (stateful1DownstreamPort.operatorId.equals(optimizedStateless1.id())) {
stateful1ConnectionExists = true;
} else {
final Port stateful2DownstreamPort = stateful2DownstreamConnections.iterator().next();
if (stateful2DownstreamPort.operatorId.equals(optimizedStateless1.id())) {
stateful2ConnectionExists = true;
}
}
assertTrue(
optimizedConnections.contains(
new SimpleEntry<>(
new Port(optimizedStateless1.id(), 0), new Port(optimizedStateless2.id(), 0))));
assertTrue(
optimizedConnections.contains(
new SimpleEntry<>(
new Port(optimizedStateless2.id(), 0), new Port(optimizedStateless3.id(), 0))));
assertTrue(
optimizedConnections.contains(
new SimpleEntry<>(new Port(optimizedStateless3.id(), 0), new Port("stateful3", 0))));
assertTrue(
optimizedConnections.contains(
new SimpleEntry<>(new Port(optimizedStateless3.id(), 0), new Port("stateful4", 0))));
}
assertTrue(stateful1ConnectionExists);
assertTrue(stateful2ConnectionExists);
assertFalse(optimizedOperators.containsKey(stateless1.id()));
assertFalse(optimizedOperators.containsKey(stateless2.id()));
assertFalse(optimizedOperators.containsKey(stateless3.id()));
}
