@Activate
public void activate(ComponentContext context) {
cfgService.registerProperties(getClass());
modified(context);
local = clusterService.getLocalNode().id();
messageHandlingExecutor =
Executors.newFixedThreadPool(
messageHandlerThreadPoolSize,
groupedThreads("onos/store/statistic", "message-handlers", log));
clusterCommunicator.addSubscriber(
GET_CURRENT,
SERIALIZER::decode,
this::getCurrentStatisticInternal,
SERIALIZER::encode,
messageHandlingExecutor);
clusterCommunicator.addSubscriber(
GET_CURRENT,
SERIALIZER::decode,
this::getPreviousStatisticInternal,
SERIALIZER::encode,
messageHandlingExecutor);
log.info("Started");
}
@Before
public void setUp() throws Exception {
clusterCommunicator = createNiceMock(ClusterCommunicationService.class);
clusterCommunicator.addSubscriber(
anyObject(MessageSubject.class),
anyObject(ClusterMessageHandler.class),
anyObject(ExecutorService.class));
expectLastCall().anyTimes();
replay(clusterCommunicator);
ClusterService clusterService = new TestClusterService();
testGossipDeviceStore =
new TestGossipDeviceStore(deviceClockService, clusterService, clusterCommunicator);
testGossipDeviceStore.mastershipService = new TestMastershipService();
TestDatabaseManager testDatabaseManager = new TestDatabaseManager();
testDatabaseManager.init(clusterService, clusterCommunicator);
testGossipDeviceStore.storageService = testDatabaseManager;
testGossipDeviceStore.deviceClockService = deviceClockService;
gossipDeviceStore = testGossipDeviceStore;
gossipDeviceStore.activate();
deviceStore = gossipDeviceStore;
verify(clusterCommunicator);
reset(clusterCommunicator);
}
private void registerMessageHandlers(ExecutorService executor) {
clusterCommunicator.addSubscriber(APPLY_BATCH_FLOWS, new OnStoreBatch(), executor);
clusterCommunicator.<FlowRuleBatchEvent>addSubscriber(
REMOTE_APPLY_COMPLETED, SERIALIZER::decode, this::notifyDelegate, executor);
clusterCommunicator.addSubscriber(
GET_FLOW_ENTRY, SERIALIZER::decode, flowTable::getFlowEntry, SERIALIZER::encode, executor);
clusterCommunicator.addSubscriber(
GET_DEVICE_FLOW_ENTRIES,
SERIALIZER::decode,
flowTable::getFlowEntries,
SERIALIZER::encode,
executor);
clusterCommunicator.addSubscriber(
REMOVE_FLOW_ENTRY,
SERIALIZER::decode,
this::removeFlowRuleInternal,
SERIALIZER::encode,
executor);
clusterCommunicator.addSubscriber(
REMOVE_FLOW_ENTRY,
SERIALIZER::decode,
this::removeFlowRuleInternal,
SERIALIZER::encode,
executor);
clusterCommunicator.addSubscriber(
FLOW_TABLE_BACKUP,
SERIALIZER::decode,
flowTable::onBackupReceipt,
SERIALIZER::encode,
executor);
}
@Activate
public void activate() {
messageHandlingExecutor =
Executors.newFixedThreadPool(
messageHandlerThreadPoolSize,
groupedThreads("onos/store/packet", "message-handlers", log));
communicationService.<OutboundPacket>addSubscriber(
PACKET_OUT_SUBJECT,
SERIALIZER::decode,
packet -> notifyDelegate(new PacketEvent(Type.EMIT, packet)),
messageHandlingExecutor);
tracker = new PacketRequestTracker();
log.info("Started");
}
protected <K, V> DefaultAsyncConsistentMap<K, V> registerMap(
DefaultAsyncConsistentMap<K, V> map) {
DefaultAsyncConsistentMap<K, V> existing = maps.putIfAbsent(map.name(), map);
if (existing != null) {
// FIXME: We need to cleanly support different map instances with same name.
log.info("Map by name {} already exists", map.name());
return existing;
} else {
if (map.applicationId() != null) {
mapsByApplication.put(map.applicationId(), map);
}
}
clusterCommunicator.<MapEvent<K, V>>addSubscriber(
mapUpdatesSubject(map.name()),
map.serializer()::decode,
map::notifyLocalListeners,
eventDispatcher);
return map;
}
@Activate
public void activate() {
localNodeId = clusterService.getLocalNode().id();
// load database configuration
File databaseDefFile = new File(PARTITION_DEFINITION_FILE);
log.info("Loading database definition: {}", databaseDefFile.getAbsolutePath());
Map<String, Set<NodeInfo>> partitionMap;
try {
DatabaseDefinitionStore databaseDefStore = new DatabaseDefinitionStore(databaseDefFile);
if (!databaseDefFile.exists()) {
createDefaultDatabaseDefinition(databaseDefStore);
}
partitionMap = databaseDefStore.read().getPartitions();
} catch (IOException e) {
throw new IllegalStateException("Failed to load database config", e);
}
String[] activeNodeUris =
partitionMap
.values()
.stream()
.reduce((s1, s2) -> Sets.union(s1, s2))
.get()
.stream()
.map(this::nodeToUri)
.toArray(String[]::new);
String localNodeUri = nodeToUri(NodeInfo.of(clusterService.getLocalNode()));
Protocol protocol = new CopycatCommunicationProtocol(clusterService, clusterCommunicator);
ClusterConfig clusterConfig =
new ClusterConfig()
.withProtocol(protocol)
.withElectionTimeout(electionTimeoutMillis(activeNodeUris))
.withHeartbeatInterval(heartbeatTimeoutMillis(activeNodeUris))
.withMembers(activeNodeUris)
.withLocalMember(localNodeUri);
CopycatConfig copycatConfig =
new CopycatConfig()
.withName("onos")
.withClusterConfig(clusterConfig)
.withDefaultSerializer(new DatabaseSerializer())
.withDefaultExecutor(
Executors.newSingleThreadExecutor(
new NamedThreadFactory("copycat-coordinator-%d")));
coordinator = new DefaultClusterCoordinator(copycatConfig.resolve());
DatabaseConfig inMemoryDatabaseConfig =
newDatabaseConfig(BASE_PARTITION_NAME, newInMemoryLog(), activeNodeUris);
inMemoryDatabase =
coordinator.getResource(
inMemoryDatabaseConfig.getName(),
inMemoryDatabaseConfig
.resolve(clusterConfig)
.withSerializer(copycatConfig.getDefaultSerializer())
.withDefaultExecutor(copycatConfig.getDefaultExecutor()));
List<Database> partitions =
partitionMap
.entrySet()
.stream()
.map(
entry -> {
String[] replicas =
entry.getValue().stream().map(this::nodeToUri).toArray(String[]::new);
return newDatabaseConfig(entry.getKey(), newPersistentLog(), replicas);
})
.map(
config -> {
Database db =
coordinator.getResource(
config.getName(),
config
.resolve(clusterConfig)
.withSerializer(copycatConfig.getDefaultSerializer())
.withDefaultExecutor(copycatConfig.getDefaultExecutor()));
return db;
})
.collect(Collectors.toList());
partitionedDatabase = new PartitionedDatabase("onos-store", partitions);
CompletableFuture<Void> status =
coordinator
.open()
.thenCompose(
v ->
CompletableFuture.allOf(inMemoryDatabase.open(), partitionedDatabase.open())
.whenComplete(
(db, error) -> {
if (error != null) {
log.error("Failed to initialize database.", error);
} else {
log.info("Successfully initialized database.");
}
}));
Futures.getUnchecked(status);
transactionManager = new TransactionManager(partitionedDatabase, consistentMapBuilder());
partitionedDatabase.setTransactionManager(transactionManager);
eventDispatcher =
Executors.newSingleThreadExecutor(
groupedThreads("onos/store/manager", "map-event-dispatcher"));
queuePollExecutor =
Executors.newFixedThreadPool(4, groupedThreads("onos/store/manager", "queue-poll-handler"));
clusterCommunicator.<String>addSubscriber(
QUEUE_UPDATED_TOPIC,
data -> new String(data, Charsets.UTF_8),
name -> {
DefaultDistributedQueue q = queues.get(name);
if (q != null) {
q.tryPoll();
}
},
queuePollExecutor);
log.info("Started");
}
/**
* Creates a new eventually consistent map shared amongst multiple instances.
*
* <p>See {@link org.onosproject.store.service.EventuallyConsistentMapBuilder} for more
* description of the parameters expected by the map.
*
* @param mapName a String identifier for the map.
* @param clusterService the cluster service
* @param clusterCommunicator the cluster communications service
* @param ns a Kryo namespace that can serialize both K and V
* @param timestampProvider provider of timestamps for K and V
* @param peerUpdateFunction function that provides a set of nodes to immediately update to when
* there writes to the map
* @param eventExecutor executor to use for processing incoming events from peers
* @param communicationExecutor executor to use for sending events to peers
* @param backgroundExecutor executor to use for background anti-entropy tasks
* @param tombstonesDisabled true if this map should not maintain tombstones
* @param antiEntropyPeriod period that the anti-entropy task should run
* @param antiEntropyTimeUnit time unit for anti-entropy period
* @param convergeFaster make anti-entropy try to converge faster
* @param persistent persist data to disk
* @param persistenceService persistence service
*/
EventuallyConsistentMapImpl(
String mapName,
ClusterService clusterService,
ClusterCommunicationService clusterCommunicator,
KryoNamespace ns,
BiFunction<K, V, Timestamp> timestampProvider,
BiFunction<K, V, Collection<NodeId>> peerUpdateFunction,
ExecutorService eventExecutor,
ExecutorService communicationExecutor,
ScheduledExecutorService backgroundExecutor,
boolean tombstonesDisabled,
long antiEntropyPeriod,
TimeUnit antiEntropyTimeUnit,
boolean convergeFaster,
boolean persistent,
PersistenceService persistenceService) {
this.mapName = mapName;
this.serializer = createSerializer(ns);
this.persistenceService = persistenceService;
this.persistent = persistent;
if (persistent) {
items =
this.persistenceService
.<K, MapValue<V>>persistentMapBuilder()
.withName(PERSISTENT_LOCAL_MAP_NAME)
.withSerializer(this.serializer)
.build();
} else {
items = Maps.newConcurrentMap();
}
senderPending = Maps.newConcurrentMap();
destroyedMessage = mapName + ERROR_DESTROYED;
this.clusterService = clusterService;
this.clusterCommunicator = clusterCommunicator;
this.localNodeId = clusterService.getLocalNode().id();
this.timestampProvider = timestampProvider;
if (peerUpdateFunction != null) {
this.peerUpdateFunction = peerUpdateFunction;
} else {
this.peerUpdateFunction =
(key, value) ->
clusterService
.getNodes()
.stream()
.map(ControllerNode::id)
.filter(nodeId -> !nodeId.equals(localNodeId))
.collect(Collectors.toList());
}
if (eventExecutor != null) {
this.executor = eventExecutor;
} else {
// should be a normal executor; it's used for receiving messages
this.executor =
Executors.newFixedThreadPool(8, groupedThreads("onos/ecm", mapName + "-fg-%d", log));
}
if (communicationExecutor != null) {
this.communicationExecutor = communicationExecutor;
} else {
// sending executor; should be capped
// TODO this probably doesn't need to be bounded anymore
this.communicationExecutor =
newFixedThreadPool(8, groupedThreads("onos/ecm", mapName + "-publish-%d", log));
}
if (backgroundExecutor != null) {
this.backgroundExecutor = backgroundExecutor;
} else {
this.backgroundExecutor =
newSingleThreadScheduledExecutor(groupedThreads("onos/ecm", mapName + "-bg-%d", log));
}
// start anti-entropy thread
this.backgroundExecutor.scheduleAtFixedRate(
this::sendAdvertisement, initialDelaySec, antiEntropyPeriod, antiEntropyTimeUnit);
updateMessageSubject = new MessageSubject("ecm-" + mapName + "-update");
clusterCommunicator.addSubscriber(
updateMessageSubject, serializer::decode, this::processUpdates, this.executor);
antiEntropyAdvertisementSubject = new MessageSubject("ecm-" + mapName + "-anti-entropy");
clusterCommunicator.addSubscriber(
antiEntropyAdvertisementSubject,
serializer::decode,
this::handleAntiEntropyAdvertisement,
serializer::encode,
this.backgroundExecutor);
updateRequestSubject = new MessageSubject("ecm-" + mapName + "-update-request");
clusterCommunicator.addSubscriber(
updateRequestSubject,
serializer::decode,
this::handleUpdateRequests,
this.backgroundExecutor);
if (!tombstonesDisabled) {
previousTombstonePurgeTime = 0;
this.backgroundExecutor.scheduleWithFixedDelay(
this::purgeTombstones, initialDelaySec, antiEntropyPeriod, TimeUnit.SECONDS);
}
this.tombstonesDisabled = tombstonesDisabled;
this.lightweightAntiEntropy = !convergeFaster;
// Initiate first round of Gossip
this.bootstrap();
}
