コード例 #1
0
 private StoreSerializer createSerializer(KryoNamespace ns) {
   return StoreSerializer.using(
       KryoNamespace.newBuilder()
           .register(ns)
           // not so robust way to avoid collision with other
           // user supplied registrations
           .nextId(KryoNamespaces.BEGIN_USER_CUSTOM_ID + 100)
           .register(KryoNamespaces.BASIC)
           .register(LogicalTimestamp.class)
           .register(WallClockTimestamp.class)
           .register(AntiEntropyAdvertisement.class)
           .register(AntiEntropyResponse.class)
           .register(UpdateEntry.class)
           .register(MapValue.class)
           .register(MapValue.Digest.class)
           .register(UpdateRequest.class)
           .build(name() + "-ecmap"));
 }
コード例 #2
0
/** Distributed packet store implementation allowing packets to be sent to remote instances. */
@Component(immediate = true)
@Service
public class DistributedPacketStore extends AbstractStore<PacketEvent, PacketStoreDelegate>
    implements PacketStore {

  private final Logger log = getLogger(getClass());

  private static final String FORMAT = "Setting: messageHandlerThreadPoolSize={}";

  @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
  protected MastershipService mastershipService;

  @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
  protected ClusterService clusterService;

  @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
  protected ClusterCommunicationService communicationService;

  @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
  protected StorageService storageService;

  private PacketRequestTracker tracker;

  private static final MessageSubject PACKET_OUT_SUBJECT = new MessageSubject("packet-out");

  private static final StoreSerializer SERIALIZER = StoreSerializer.using(KryoNamespaces.API);

  private ExecutorService messageHandlingExecutor;

  private static final int DEFAULT_MESSAGE_HANDLER_THREAD_POOL_SIZE = 4;

  @Property(
      name = "messageHandlerThreadPoolSize",
      intValue = DEFAULT_MESSAGE_HANDLER_THREAD_POOL_SIZE,
      label = "Size of thread pool to assign message handler")
  private static int messageHandlerThreadPoolSize = DEFAULT_MESSAGE_HANDLER_THREAD_POOL_SIZE;

  private static final int MAX_BACKOFF = 50;

  @Activate
  public void activate() {
    messageHandlingExecutor =
        Executors.newFixedThreadPool(
            messageHandlerThreadPoolSize,
            groupedThreads("onos/store/packet", "message-handlers", log));

    communicationService.<OutboundPacket>addSubscriber(
        PACKET_OUT_SUBJECT,
        SERIALIZER::decode,
        packet -> notifyDelegate(new PacketEvent(Type.EMIT, packet)),
        messageHandlingExecutor);

    tracker = new PacketRequestTracker();

    log.info("Started");
  }

  @Deactivate
  public void deactivate() {
    communicationService.removeSubscriber(PACKET_OUT_SUBJECT);
    messageHandlingExecutor.shutdown();
    tracker = null;
    log.info("Stopped");
  }

  @Modified
  public void modified(ComponentContext context) {
    Dictionary<?, ?> properties = context != null ? context.getProperties() : new Properties();

    int newMessageHandlerThreadPoolSize;

    try {
      String s = get(properties, "messageHandlerThreadPoolSize");

      newMessageHandlerThreadPoolSize =
          isNullOrEmpty(s) ? messageHandlerThreadPoolSize : Integer.parseInt(s.trim());

    } catch (NumberFormatException e) {
      log.warn(e.getMessage());
      newMessageHandlerThreadPoolSize = messageHandlerThreadPoolSize;
    }

    // Any change in the following parameters implies thread pool restart
    if (newMessageHandlerThreadPoolSize != messageHandlerThreadPoolSize) {
      setMessageHandlerThreadPoolSize(newMessageHandlerThreadPoolSize);
      restartMessageHandlerThreadPool();
    }

    log.info(FORMAT, messageHandlerThreadPoolSize);
  }

  @Override
  public void emit(OutboundPacket packet) {
    NodeId myId = clusterService.getLocalNode().id();
    NodeId master = mastershipService.getMasterFor(packet.sendThrough());

    if (master == null) {
      return;
    }

    if (myId.equals(master)) {
      notifyDelegate(new PacketEvent(Type.EMIT, packet));
      return;
    }

    communicationService
        .unicast(packet, PACKET_OUT_SUBJECT, SERIALIZER::encode, master)
        .whenComplete(
            (r, error) -> {
              if (error != null) {
                log.warn("Failed to send packet-out to {}", master, error);
              }
            });
  }

  @Override
  public void requestPackets(PacketRequest request) {
    tracker.add(request);
  }

  @Override
  public void cancelPackets(PacketRequest request) {
    tracker.remove(request);
  }

  @Override
  public List<PacketRequest> existingRequests() {
    return tracker.requests();
  }

  private final class PacketRequestTracker {

    private ConsistentMap<TrafficSelector, Set<PacketRequest>> requests;

    private PacketRequestTracker() {
      requests =
          storageService
              .<TrafficSelector, Set<PacketRequest>>consistentMapBuilder()
              .withName("onos-packet-requests")
              .withPartitionsDisabled()
              .withSerializer(Serializer.using(KryoNamespaces.API))
              .build();
    }

    private void add(PacketRequest request) {
      AtomicBoolean firstRequest =
          retryable(
                  this::addInternal,
                  ConsistentMapException.ConcurrentModification.class,
                  Integer.MAX_VALUE,
                  MAX_BACKOFF)
              .apply(request);
      if (firstRequest.get() && delegate != null) {
        // The instance that makes the first request will push to all devices
        delegate.requestPackets(request);
      }
    }

    private AtomicBoolean addInternal(PacketRequest request) {
      AtomicBoolean firstRequest = new AtomicBoolean(false);
      requests.compute(
          request.selector(),
          (s, existingRequests) -> {
            if (existingRequests == null) {
              firstRequest.set(true);
              return ImmutableSet.of(request);
            } else if (!existingRequests.contains(request)) {
              return ImmutableSet.<PacketRequest>builder()
                  .addAll(existingRequests)
                  .add(request)
                  .build();
            } else {
              return existingRequests;
            }
          });
      return firstRequest;
    }

    private void remove(PacketRequest request) {
      AtomicBoolean removedLast =
          retryable(
                  this::removeInternal,
                  ConsistentMapException.ConcurrentModification.class,
                  Integer.MAX_VALUE,
                  MAX_BACKOFF)
              .apply(request);
      if (removedLast.get() && delegate != null) {
        // The instance that removes the last request will remove from all devices
        delegate.cancelPackets(request);
      }
    }

    private AtomicBoolean removeInternal(PacketRequest request) {
      AtomicBoolean removedLast = new AtomicBoolean(false);
      requests.computeIfPresent(
          request.selector(),
          (s, existingRequests) -> {
            if (existingRequests.contains(request)) {
              Set<PacketRequest> newRequests = Sets.newHashSet(existingRequests);
              newRequests.remove(request);
              if (newRequests.size() > 0) {
                return ImmutableSet.copyOf(newRequests);
              } else {
                removedLast.set(true);
                return null;
              }
            } else {
              return existingRequests;
            }
          });
      return removedLast;
    }

    private List<PacketRequest> requests() {
      List<PacketRequest> list = Lists.newArrayList();
      requests.values().forEach(v -> list.addAll(v.value()));
      list.sort((o1, o2) -> o1.priority().priorityValue() - o2.priority().priorityValue());
      return list;
    }
  }

  /**
   * Sets thread pool size of message handler.
   *
   * @param poolSize
   */
  private void setMessageHandlerThreadPoolSize(int poolSize) {
    checkArgument(poolSize >= 0, "Message handler pool size must be 0 or more");
    messageHandlerThreadPoolSize = poolSize;
  }

  /** Restarts thread pool of message handler. */
  private void restartMessageHandlerThreadPool() {
    ExecutorService prevExecutor = messageHandlingExecutor;
    messageHandlingExecutor = Executors.newFixedThreadPool(getMessageHandlerThreadPoolSize());
    prevExecutor.shutdown();
  }

  /**
   * Gets current thread pool size of message handler.
   *
   * @return messageHandlerThreadPoolSize
   */
  private int getMessageHandlerThreadPoolSize() {
    return messageHandlerThreadPoolSize;
  }
}
コード例 #3
0
/** Maintains flow statistics using RPC calls to collect stats from remote instances on demand. */
@Component(immediate = true)
@Service
public class DistributedFlowStatisticStore implements FlowStatisticStore {
  private final Logger log = getLogger(getClass());

  private static final String FORMAT = "Setting: messageHandlerThreadPoolSize={}";

  @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
  protected MastershipService mastershipService;

  @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
  protected ClusterCommunicationService clusterCommunicator;

  @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
  protected ClusterService clusterService;

  @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
  protected ComponentConfigService cfgService;

  private Map<ConnectPoint, Set<FlowEntry>> previous = new ConcurrentHashMap<>();

  private Map<ConnectPoint, Set<FlowEntry>> current = new ConcurrentHashMap<>();

  public static final MessageSubject GET_CURRENT = new MessageSubject("peer-return-current");
  public static final MessageSubject GET_PREVIOUS = new MessageSubject("peer-return-previous");

  protected static final StoreSerializer SERIALIZER = StoreSerializer.using(KryoNamespaces.API);

  private NodeId local;
  private ExecutorService messageHandlingExecutor;

  private static final int DEFAULT_MESSAGE_HANDLER_THREAD_POOL_SIZE = 4;

  @Property(
      name = "messageHandlerThreadPoolSize",
      intValue = DEFAULT_MESSAGE_HANDLER_THREAD_POOL_SIZE,
      label = "Size of thread pool to assign message handler")
  private static int messageHandlerThreadPoolSize = DEFAULT_MESSAGE_HANDLER_THREAD_POOL_SIZE;

  private static final long STATISTIC_STORE_TIMEOUT_MILLIS = 3000;

  @Activate
  public void activate(ComponentContext context) {
    cfgService.registerProperties(getClass());

    modified(context);

    local = clusterService.getLocalNode().id();

    messageHandlingExecutor =
        Executors.newFixedThreadPool(
            messageHandlerThreadPoolSize,
            groupedThreads("onos/store/statistic", "message-handlers", log));

    clusterCommunicator.addSubscriber(
        GET_CURRENT,
        SERIALIZER::decode,
        this::getCurrentStatisticInternal,
        SERIALIZER::encode,
        messageHandlingExecutor);

    clusterCommunicator.addSubscriber(
        GET_CURRENT,
        SERIALIZER::decode,
        this::getPreviousStatisticInternal,
        SERIALIZER::encode,
        messageHandlingExecutor);

    log.info("Started");
  }

  @Deactivate
  public void deactivate() {
    cfgService.unregisterProperties(getClass(), false);
    clusterCommunicator.removeSubscriber(GET_PREVIOUS);
    clusterCommunicator.removeSubscriber(GET_CURRENT);
    messageHandlingExecutor.shutdown();
    log.info("Stopped");
  }

  @Modified
  public void modified(ComponentContext context) {
    Dictionary<?, ?> properties = context != null ? context.getProperties() : new Properties();

    int newMessageHandlerThreadPoolSize;

    try {
      String s = get(properties, "messageHandlerThreadPoolSize");

      newMessageHandlerThreadPoolSize =
          isNullOrEmpty(s) ? messageHandlerThreadPoolSize : Integer.parseInt(s.trim());

    } catch (NumberFormatException e) {
      log.warn(e.getMessage());
      newMessageHandlerThreadPoolSize = messageHandlerThreadPoolSize;
    }

    // Any change in the following parameters implies thread pool restart
    if (newMessageHandlerThreadPoolSize != messageHandlerThreadPoolSize) {
      setMessageHandlerThreadPoolSize(newMessageHandlerThreadPoolSize);
      restartMessageHandlerThreadPool();
    }

    log.info(FORMAT, messageHandlerThreadPoolSize);
  }

  @Override
  public synchronized void removeFlowStatistic(FlowRule rule) {
    ConnectPoint cp = buildConnectPoint(rule);
    if (cp == null) {
      return;
    }

    // remove this rule if present from current map
    current.computeIfPresent(
        cp,
        (c, e) -> {
          e.remove(rule);
          return e;
        });

    // remove this on if present from previous map
    previous.computeIfPresent(
        cp,
        (c, e) -> {
          e.remove(rule);
          return e;
        });
  }

  @Override
  public synchronized void addFlowStatistic(FlowEntry rule) {
    ConnectPoint cp = buildConnectPoint(rule);
    if (cp == null) {
      return;
    }

    // create one if absent and add this rule
    current.putIfAbsent(cp, new HashSet<>());
    current.computeIfPresent(
        cp,
        (c, e) -> {
          e.add(rule);
          return e;
        });

    // remove previous one if present
    previous.computeIfPresent(
        cp,
        (c, e) -> {
          e.remove(rule);
          return e;
        });
  }

  @Override
  public synchronized void updateFlowStatistic(FlowEntry rule) {
    ConnectPoint cp = buildConnectPoint(rule);
    if (cp == null) {
      return;
    }

    Set<FlowEntry> curr = current.get(cp);
    if (curr == null) {
      addFlowStatistic(rule);
    } else {
      Optional<FlowEntry> f = curr.stream().filter(c -> rule.equals(c)).findAny();
      if (f.isPresent() && rule.bytes() < f.get().bytes()) {
        log.debug(
            "DistributedFlowStatisticStore:updateFlowStatistic():"
                + " Invalid Flow Update! Will be removed!!"
                + " curr flowId="
                + Long.toHexString(rule.id().value())
                + ", prev flowId="
                + Long.toHexString(f.get().id().value())
                + ", curr bytes="
                + rule.bytes()
                + ", prev bytes="
                + f.get().bytes()
                + ", curr life="
                + rule.life()
                + ", prev life="
                + f.get().life()
                + ", curr lastSeen="
                + rule.lastSeen()
                + ", prev lastSeen="
                + f.get().lastSeen());
        // something is wrong! invalid flow entry, so delete it
        removeFlowStatistic(rule);
        return;
      }
      Set<FlowEntry> prev = previous.get(cp);
      if (prev == null) {
        prev = new HashSet<>();
        previous.put(cp, prev);
      }

      // previous one is exist
      if (f.isPresent()) {
        // remove old one and add new one
        prev.remove(rule);
        if (!prev.add(f.get())) {
          log.debug(
              "DistributedFlowStatisticStore:updateFlowStatistic():"
                  + " flowId={}, add failed into previous.",
              Long.toHexString(rule.id().value()));
        }
      }

      // remove old one and add new one
      curr.remove(rule);
      if (!curr.add(rule)) {
        log.debug(
            "DistributedFlowStatisticStore:updateFlowStatistic():"
                + " flowId={}, add failed into current.",
            Long.toHexString(rule.id().value()));
      }
    }
  }

  @Override
  public Set<FlowEntry> getCurrentFlowStatistic(ConnectPoint connectPoint) {
    final DeviceId deviceId = connectPoint.deviceId();

    NodeId master = mastershipService.getMasterFor(deviceId);
    if (master == null) {
      log.warn("No master for {}", deviceId);
      return Collections.emptySet();
    }

    if (Objects.equal(local, master)) {
      return getCurrentStatisticInternal(connectPoint);
    } else {
      return Tools.futureGetOrElse(
          clusterCommunicator.sendAndReceive(
              connectPoint, GET_CURRENT, SERIALIZER::encode, SERIALIZER::decode, master),
          STATISTIC_STORE_TIMEOUT_MILLIS,
          TimeUnit.MILLISECONDS,
          Collections.emptySet());
    }
  }

  private synchronized Set<FlowEntry> getCurrentStatisticInternal(ConnectPoint connectPoint) {
    return current.get(connectPoint);
  }

  @Override
  public Set<FlowEntry> getPreviousFlowStatistic(ConnectPoint connectPoint) {
    final DeviceId deviceId = connectPoint.deviceId();

    NodeId master = mastershipService.getMasterFor(deviceId);
    if (master == null) {
      log.warn("No master for {}", deviceId);
      return Collections.emptySet();
    }

    if (Objects.equal(local, master)) {
      return getPreviousStatisticInternal(connectPoint);
    } else {
      return Tools.futureGetOrElse(
          clusterCommunicator.sendAndReceive(
              connectPoint, GET_PREVIOUS, SERIALIZER::encode, SERIALIZER::decode, master),
          STATISTIC_STORE_TIMEOUT_MILLIS,
          TimeUnit.MILLISECONDS,
          Collections.emptySet());
    }
  }

  private synchronized Set<FlowEntry> getPreviousStatisticInternal(ConnectPoint connectPoint) {
    return previous.get(connectPoint);
  }

  private ConnectPoint buildConnectPoint(FlowRule rule) {
    PortNumber port = getOutput(rule);

    if (port == null) {
      return null;
    }
    ConnectPoint cp = new ConnectPoint(rule.deviceId(), port);
    return cp;
  }

  private PortNumber getOutput(FlowRule rule) {
    for (Instruction i : rule.treatment().allInstructions()) {
      if (i.type() == Instruction.Type.OUTPUT) {
        Instructions.OutputInstruction out = (Instructions.OutputInstruction) i;
        return out.port();
      }
    }
    return null;
  }

  /**
   * Sets thread pool size of message handler.
   *
   * @param poolSize
   */
  private void setMessageHandlerThreadPoolSize(int poolSize) {
    checkArgument(poolSize >= 0, "Message handler pool size must be 0 or more");
    messageHandlerThreadPoolSize = poolSize;
  }

  /** Restarts thread pool of message handler. */
  private void restartMessageHandlerThreadPool() {
    ExecutorService prevExecutor = messageHandlingExecutor;
    messageHandlingExecutor =
        newFixedThreadPool(
            getMessageHandlerThreadPoolSize(),
            groupedThreads("DistFlowStats", "messageHandling-%d", log));
    prevExecutor.shutdown();
  }

  /**
   * Gets current thread pool size of message handler.
   *
   * @return messageHandlerThreadPoolSize
   */
  private int getMessageHandlerThreadPoolSize() {
    return messageHandlerThreadPoolSize;
  }
}