private StoreSerializer createSerializer(KryoNamespace ns) {
return StoreSerializer.using(
KryoNamespace.newBuilder()
.register(ns)
// not so robust way to avoid collision with other
// user supplied registrations
.nextId(KryoNamespaces.BEGIN_USER_CUSTOM_ID + 100)
.register(KryoNamespaces.BASIC)
.register(LogicalTimestamp.class)
.register(WallClockTimestamp.class)
.register(AntiEntropyAdvertisement.class)
.register(AntiEntropyResponse.class)
.register(UpdateEntry.class)
.register(MapValue.class)
.register(MapValue.Digest.class)
.register(UpdateRequest.class)
.build(name() + "-ecmap"));
}
/** Distributed packet store implementation allowing packets to be sent to remote instances. */
@Component(immediate = true)
@Service
public class DistributedPacketStore extends AbstractStore<PacketEvent, PacketStoreDelegate>
implements PacketStore {
private final Logger log = getLogger(getClass());
private static final String FORMAT = "Setting: messageHandlerThreadPoolSize={}";
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected MastershipService mastershipService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ClusterService clusterService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ClusterCommunicationService communicationService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected StorageService storageService;
private PacketRequestTracker tracker;
private static final MessageSubject PACKET_OUT_SUBJECT = new MessageSubject("packet-out");
private static final StoreSerializer SERIALIZER = StoreSerializer.using(KryoNamespaces.API);
private ExecutorService messageHandlingExecutor;
private static final int DEFAULT_MESSAGE_HANDLER_THREAD_POOL_SIZE = 4;
@Property(
name = "messageHandlerThreadPoolSize",
intValue = DEFAULT_MESSAGE_HANDLER_THREAD_POOL_SIZE,
label = "Size of thread pool to assign message handler")
private static int messageHandlerThreadPoolSize = DEFAULT_MESSAGE_HANDLER_THREAD_POOL_SIZE;
private static final int MAX_BACKOFF = 50;
@Activate
public void activate() {
messageHandlingExecutor =
Executors.newFixedThreadPool(
messageHandlerThreadPoolSize,
groupedThreads("onos/store/packet", "message-handlers", log));
communicationService.<OutboundPacket>addSubscriber(
PACKET_OUT_SUBJECT,
SERIALIZER::decode,
packet -> notifyDelegate(new PacketEvent(Type.EMIT, packet)),
messageHandlingExecutor);
tracker = new PacketRequestTracker();
log.info("Started");
}
@Deactivate
public void deactivate() {
communicationService.removeSubscriber(PACKET_OUT_SUBJECT);
messageHandlingExecutor.shutdown();
tracker = null;
log.info("Stopped");
}
@Modified
public void modified(ComponentContext context) {
Dictionary<?, ?> properties = context != null ? context.getProperties() : new Properties();
int newMessageHandlerThreadPoolSize;
try {
String s = get(properties, "messageHandlerThreadPoolSize");
newMessageHandlerThreadPoolSize =
isNullOrEmpty(s) ? messageHandlerThreadPoolSize : Integer.parseInt(s.trim());
} catch (NumberFormatException e) {
log.warn(e.getMessage());
newMessageHandlerThreadPoolSize = messageHandlerThreadPoolSize;
}
// Any change in the following parameters implies thread pool restart
if (newMessageHandlerThreadPoolSize != messageHandlerThreadPoolSize) {
setMessageHandlerThreadPoolSize(newMessageHandlerThreadPoolSize);
restartMessageHandlerThreadPool();
}
log.info(FORMAT, messageHandlerThreadPoolSize);
}
@Override
public void emit(OutboundPacket packet) {
NodeId myId = clusterService.getLocalNode().id();
NodeId master = mastershipService.getMasterFor(packet.sendThrough());
if (master == null) {
return;
}
if (myId.equals(master)) {
notifyDelegate(new PacketEvent(Type.EMIT, packet));
return;
}
communicationService
.unicast(packet, PACKET_OUT_SUBJECT, SERIALIZER::encode, master)
.whenComplete(
(r, error) -> {
if (error != null) {
log.warn("Failed to send packet-out to {}", master, error);
}
});
}
@Override
public void requestPackets(PacketRequest request) {
tracker.add(request);
}
@Override
public void cancelPackets(PacketRequest request) {
tracker.remove(request);
}
@Override
public List<PacketRequest> existingRequests() {
return tracker.requests();
}
private final class PacketRequestTracker {
private ConsistentMap<TrafficSelector, Set<PacketRequest>> requests;
private PacketRequestTracker() {
requests =
storageService
.<TrafficSelector, Set<PacketRequest>>consistentMapBuilder()
.withName("onos-packet-requests")
.withPartitionsDisabled()
.withSerializer(Serializer.using(KryoNamespaces.API))
.build();
}
private void add(PacketRequest request) {
AtomicBoolean firstRequest =
retryable(
this::addInternal,
ConsistentMapException.ConcurrentModification.class,
Integer.MAX_VALUE,
MAX_BACKOFF)
.apply(request);
if (firstRequest.get() && delegate != null) {
// The instance that makes the first request will push to all devices
delegate.requestPackets(request);
}
}
private AtomicBoolean addInternal(PacketRequest request) {
AtomicBoolean firstRequest = new AtomicBoolean(false);
requests.compute(
request.selector(),
(s, existingRequests) -> {
if (existingRequests == null) {
firstRequest.set(true);
return ImmutableSet.of(request);
} else if (!existingRequests.contains(request)) {
return ImmutableSet.<PacketRequest>builder()
.addAll(existingRequests)
.add(request)
.build();
} else {
return existingRequests;
}
});
return firstRequest;
}
private void remove(PacketRequest request) {
AtomicBoolean removedLast =
retryable(
this::removeInternal,
ConsistentMapException.ConcurrentModification.class,
Integer.MAX_VALUE,
MAX_BACKOFF)
.apply(request);
if (removedLast.get() && delegate != null) {
// The instance that removes the last request will remove from all devices
delegate.cancelPackets(request);
}
}
private AtomicBoolean removeInternal(PacketRequest request) {
AtomicBoolean removedLast = new AtomicBoolean(false);
requests.computeIfPresent(
request.selector(),
(s, existingRequests) -> {
if (existingRequests.contains(request)) {
Set<PacketRequest> newRequests = Sets.newHashSet(existingRequests);
newRequests.remove(request);
if (newRequests.size() > 0) {
return ImmutableSet.copyOf(newRequests);
} else {
removedLast.set(true);
return null;
}
} else {
return existingRequests;
}
});
return removedLast;
}
private List<PacketRequest> requests() {
List<PacketRequest> list = Lists.newArrayList();
requests.values().forEach(v -> list.addAll(v.value()));
list.sort((o1, o2) -> o1.priority().priorityValue() - o2.priority().priorityValue());
return list;
}
}
/**
* Sets thread pool size of message handler.
*
* @param poolSize
*/
private void setMessageHandlerThreadPoolSize(int poolSize) {
checkArgument(poolSize >= 0, "Message handler pool size must be 0 or more");
messageHandlerThreadPoolSize = poolSize;
}
/** Restarts thread pool of message handler. */
private void restartMessageHandlerThreadPool() {
ExecutorService prevExecutor = messageHandlingExecutor;
messageHandlingExecutor = Executors.newFixedThreadPool(getMessageHandlerThreadPoolSize());
prevExecutor.shutdown();
}
/**
* Gets current thread pool size of message handler.
*
* @return messageHandlerThreadPoolSize
*/
private int getMessageHandlerThreadPoolSize() {
return messageHandlerThreadPoolSize;
}
}
/** Maintains flow statistics using RPC calls to collect stats from remote instances on demand. */
@Component(immediate = true)
@Service
public class DistributedFlowStatisticStore implements FlowStatisticStore {
private final Logger log = getLogger(getClass());
private static final String FORMAT = "Setting: messageHandlerThreadPoolSize={}";
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected MastershipService mastershipService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ClusterCommunicationService clusterCommunicator;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ClusterService clusterService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected ComponentConfigService cfgService;
private Map<ConnectPoint, Set<FlowEntry>> previous = new ConcurrentHashMap<>();
private Map<ConnectPoint, Set<FlowEntry>> current = new ConcurrentHashMap<>();
public static final MessageSubject GET_CURRENT = new MessageSubject("peer-return-current");
public static final MessageSubject GET_PREVIOUS = new MessageSubject("peer-return-previous");
protected static final StoreSerializer SERIALIZER = StoreSerializer.using(KryoNamespaces.API);
private NodeId local;
private ExecutorService messageHandlingExecutor;
private static final int DEFAULT_MESSAGE_HANDLER_THREAD_POOL_SIZE = 4;
@Property(
name = "messageHandlerThreadPoolSize",
intValue = DEFAULT_MESSAGE_HANDLER_THREAD_POOL_SIZE,
label = "Size of thread pool to assign message handler")
private static int messageHandlerThreadPoolSize = DEFAULT_MESSAGE_HANDLER_THREAD_POOL_SIZE;
private static final long STATISTIC_STORE_TIMEOUT_MILLIS = 3000;
@Activate
public void activate(ComponentContext context) {
cfgService.registerProperties(getClass());
modified(context);
local = clusterService.getLocalNode().id();
messageHandlingExecutor =
Executors.newFixedThreadPool(
messageHandlerThreadPoolSize,
groupedThreads("onos/store/statistic", "message-handlers", log));
clusterCommunicator.addSubscriber(
GET_CURRENT,
SERIALIZER::decode,
this::getCurrentStatisticInternal,
SERIALIZER::encode,
messageHandlingExecutor);
clusterCommunicator.addSubscriber(
GET_CURRENT,
SERIALIZER::decode,
this::getPreviousStatisticInternal,
SERIALIZER::encode,
messageHandlingExecutor);
log.info("Started");
}
@Deactivate
public void deactivate() {
cfgService.unregisterProperties(getClass(), false);
clusterCommunicator.removeSubscriber(GET_PREVIOUS);
clusterCommunicator.removeSubscriber(GET_CURRENT);
messageHandlingExecutor.shutdown();
log.info("Stopped");
}
@Modified
public void modified(ComponentContext context) {
Dictionary<?, ?> properties = context != null ? context.getProperties() : new Properties();
int newMessageHandlerThreadPoolSize;
try {
String s = get(properties, "messageHandlerThreadPoolSize");
newMessageHandlerThreadPoolSize =
isNullOrEmpty(s) ? messageHandlerThreadPoolSize : Integer.parseInt(s.trim());
} catch (NumberFormatException e) {
log.warn(e.getMessage());
newMessageHandlerThreadPoolSize = messageHandlerThreadPoolSize;
}
// Any change in the following parameters implies thread pool restart
if (newMessageHandlerThreadPoolSize != messageHandlerThreadPoolSize) {
setMessageHandlerThreadPoolSize(newMessageHandlerThreadPoolSize);
restartMessageHandlerThreadPool();
}
log.info(FORMAT, messageHandlerThreadPoolSize);
}
@Override
public synchronized void removeFlowStatistic(FlowRule rule) {
ConnectPoint cp = buildConnectPoint(rule);
if (cp == null) {
return;
}
// remove this rule if present from current map
current.computeIfPresent(
cp,
(c, e) -> {
e.remove(rule);
return e;
});
// remove this on if present from previous map
previous.computeIfPresent(
cp,
(c, e) -> {
e.remove(rule);
return e;
});
}
@Override
public synchronized void addFlowStatistic(FlowEntry rule) {
ConnectPoint cp = buildConnectPoint(rule);
if (cp == null) {
return;
}
// create one if absent and add this rule
current.putIfAbsent(cp, new HashSet<>());
current.computeIfPresent(
cp,
(c, e) -> {
e.add(rule);
return e;
});
// remove previous one if present
previous.computeIfPresent(
cp,
(c, e) -> {
e.remove(rule);
return e;
});
}
@Override
public synchronized void updateFlowStatistic(FlowEntry rule) {
ConnectPoint cp = buildConnectPoint(rule);
if (cp == null) {
return;
}
Set<FlowEntry> curr = current.get(cp);
if (curr == null) {
addFlowStatistic(rule);
} else {
Optional<FlowEntry> f = curr.stream().filter(c -> rule.equals(c)).findAny();
if (f.isPresent() && rule.bytes() < f.get().bytes()) {
log.debug(
"DistributedFlowStatisticStore:updateFlowStatistic():"
+ " Invalid Flow Update! Will be removed!!"
+ " curr flowId="
+ Long.toHexString(rule.id().value())
+ ", prev flowId="
+ Long.toHexString(f.get().id().value())
+ ", curr bytes="
+ rule.bytes()
+ ", prev bytes="
+ f.get().bytes()
+ ", curr life="
+ rule.life()
+ ", prev life="
+ f.get().life()
+ ", curr lastSeen="
+ rule.lastSeen()
+ ", prev lastSeen="
+ f.get().lastSeen());
// something is wrong! invalid flow entry, so delete it
removeFlowStatistic(rule);
return;
}
Set<FlowEntry> prev = previous.get(cp);
if (prev == null) {
prev = new HashSet<>();
previous.put(cp, prev);
}
// previous one is exist
if (f.isPresent()) {
// remove old one and add new one
prev.remove(rule);
if (!prev.add(f.get())) {
log.debug(
"DistributedFlowStatisticStore:updateFlowStatistic():"
+ " flowId={}, add failed into previous.",
Long.toHexString(rule.id().value()));
}
}
// remove old one and add new one
curr.remove(rule);
if (!curr.add(rule)) {
log.debug(
"DistributedFlowStatisticStore:updateFlowStatistic():"
+ " flowId={}, add failed into current.",
Long.toHexString(rule.id().value()));
}
}
}
@Override
public Set<FlowEntry> getCurrentFlowStatistic(ConnectPoint connectPoint) {
final DeviceId deviceId = connectPoint.deviceId();
NodeId master = mastershipService.getMasterFor(deviceId);
if (master == null) {
log.warn("No master for {}", deviceId);
return Collections.emptySet();
}
if (Objects.equal(local, master)) {
return getCurrentStatisticInternal(connectPoint);
} else {
return Tools.futureGetOrElse(
clusterCommunicator.sendAndReceive(
connectPoint, GET_CURRENT, SERIALIZER::encode, SERIALIZER::decode, master),
STATISTIC_STORE_TIMEOUT_MILLIS,
TimeUnit.MILLISECONDS,
Collections.emptySet());
}
}
private synchronized Set<FlowEntry> getCurrentStatisticInternal(ConnectPoint connectPoint) {
return current.get(connectPoint);
}
@Override
public Set<FlowEntry> getPreviousFlowStatistic(ConnectPoint connectPoint) {
final DeviceId deviceId = connectPoint.deviceId();
NodeId master = mastershipService.getMasterFor(deviceId);
if (master == null) {
log.warn("No master for {}", deviceId);
return Collections.emptySet();
}
if (Objects.equal(local, master)) {
return getPreviousStatisticInternal(connectPoint);
} else {
return Tools.futureGetOrElse(
clusterCommunicator.sendAndReceive(
connectPoint, GET_PREVIOUS, SERIALIZER::encode, SERIALIZER::decode, master),
STATISTIC_STORE_TIMEOUT_MILLIS,
TimeUnit.MILLISECONDS,
Collections.emptySet());
}
}
private synchronized Set<FlowEntry> getPreviousStatisticInternal(ConnectPoint connectPoint) {
return previous.get(connectPoint);
}
private ConnectPoint buildConnectPoint(FlowRule rule) {
PortNumber port = getOutput(rule);
if (port == null) {
return null;
}
ConnectPoint cp = new ConnectPoint(rule.deviceId(), port);
return cp;
}
private PortNumber getOutput(FlowRule rule) {
for (Instruction i : rule.treatment().allInstructions()) {
if (i.type() == Instruction.Type.OUTPUT) {
Instructions.OutputInstruction out = (Instructions.OutputInstruction) i;
return out.port();
}
}
return null;
}
/**
* Sets thread pool size of message handler.
*
* @param poolSize
*/
private void setMessageHandlerThreadPoolSize(int poolSize) {
checkArgument(poolSize >= 0, "Message handler pool size must be 0 or more");
messageHandlerThreadPoolSize = poolSize;
}
/** Restarts thread pool of message handler. */
private void restartMessageHandlerThreadPool() {
ExecutorService prevExecutor = messageHandlingExecutor;
messageHandlingExecutor =
newFixedThreadPool(
getMessageHandlerThreadPoolSize(),
groupedThreads("DistFlowStats", "messageHandling-%d", log));
prevExecutor.shutdown();
}
/**
* Gets current thread pool size of message handler.
*
* @return messageHandlerThreadPoolSize
*/
private int getMessageHandlerThreadPoolSize() {
return messageHandlerThreadPoolSize;
}
}