/** * Class is a transport layer for WebRTC data channels. It consists of SCTP connection running on * top of ICE/DTLS layer. Manages WebRTC data channels. See * http://tools.ietf.org/html/draft-ietf-rtcweb-data-channel-08 for more info on WebRTC data * channels. * * <p>Control protocol: http://tools.ietf.org/html/draft-ietf-rtcweb-data-protocol-03 FIXME handle * closing of data channels(SCTP stream reset) * * @author Pawel Domas * @author Lyubomir Marinov * @author Boris Grozev */ public class SctpConnection extends Channel implements SctpDataCallback, SctpSocket.NotificationListener { /** Generator used to track debug IDs. */ private static int debugIdGen = -1; /** DTLS transport buffer size. Note: randomly chosen. */ private static final int DTLS_BUFFER_SIZE = 2048; /** Switch used for debugging SCTP traffic purposes. FIXME to be removed */ private static final boolean LOG_SCTP_PACKETS = false; /** The logger */ private static final Logger logger = Logger.getLogger(SctpConnection.class); /** * Message type used to acknowledge WebRTC data channel allocation on SCTP stream ID on which * <tt>MSG_OPEN_CHANNEL</tt> message arrives. */ private static final int MSG_CHANNEL_ACK = 0x2; private static final byte[] MSG_CHANNEL_ACK_BYTES = new byte[] {MSG_CHANNEL_ACK}; /** * Message with this type sent over control PPID in order to open new WebRTC data channel on SCTP * stream ID that this message is sent. */ private static final int MSG_OPEN_CHANNEL = 0x3; /** SCTP transport buffer size. */ private static final int SCTP_BUFFER_SIZE = DTLS_BUFFER_SIZE - 13; /** The pool of <tt>Thread</tt>s which run <tt>SctpConnection</tt>s. */ private static final ExecutorService threadPool = ExecutorUtils.newCachedThreadPool(true, SctpConnection.class.getName()); /** Payload protocol id that identifies binary data in WebRTC data channel. */ static final int WEB_RTC_PPID_BIN = 53; /** Payload protocol id for control data. Used for <tt>WebRtcDataStream</tt> allocation. */ static final int WEB_RTC_PPID_CTRL = 50; /** Payload protocol id that identifies text data UTF8 encoded in WebRTC data channels. */ static final int WEB_RTC_PPID_STRING = 51; /** * The <tt>String</tt> value of the <tt>Protocol</tt> field of the <tt>DATA_CHANNEL_OPEN</tt> * message. */ private static final String WEBRTC_DATA_CHANNEL_PROTOCOL = "http://jitsi.org/protocols/colibri"; private static synchronized int generateDebugId() { debugIdGen += 2; return debugIdGen; } /** * Indicates whether the STCP association is ready and has not been ended by a subsequent state * change. */ private boolean assocIsUp; /** Indicates if we have accepted incoming connection. */ private boolean acceptedIncomingConnection; /** Data channels mapped by SCTP stream identified(sid). */ private final Map<Integer, WebRtcDataStream> channels = new HashMap<Integer, WebRtcDataStream>(); /** Debug ID used to distinguish SCTP sockets in packet logs. */ private final int debugId; /** * The <tt>AsyncExecutor</tt> which is to asynchronously dispatch the events fired by this * instance in order to prevent possible listeners from blocking this <tt>SctpConnection</tt> in * general and {@link #sctpSocket} in particular for too long. The timeout of <tt>15</tt> is * chosen to be in accord with the time it takes to expire a <tt>Channel</tt>. */ private final AsyncExecutor<Runnable> eventDispatcher = new AsyncExecutor<Runnable>(15, TimeUnit.MILLISECONDS); /** Datagram socket for ICE/UDP layer. */ private IceSocketWrapper iceSocket; /** * List of <tt>WebRtcDataStreamListener</tt>s that will be notified whenever new WebRTC data * channel is opened. */ private final List<WebRtcDataStreamListener> listeners = new ArrayList<WebRtcDataStreamListener>(); /** Remote SCTP port. */ private final int remoteSctpPort; /** <tt>SctpSocket</tt> used for SCTP transport. */ private SctpSocket sctpSocket; /** * Flag prevents from starting this connection multiple times from {@link #maybeStartStream()}. */ private boolean started; /** * Initializes a new <tt>SctpConnection</tt> instance. * * @param id the string identifier of this connection instance * @param content the <tt>Content</tt> which is initializing the new instance * @param endpoint the <tt>Endpoint</tt> of newly created instance * @param remoteSctpPort the SCTP port used by remote peer * @param channelBundleId the ID of the channel-bundle this <tt>SctpConnection</tt> is to be a * part of (or <tt>null</tt> if no it is not to be a part of a channel-bundle). * @throws Exception if an error occurs while initializing the new instance */ public SctpConnection( String id, Content content, Endpoint endpoint, int remoteSctpPort, String channelBundleId) throws Exception { super(content, id, channelBundleId); setEndpoint(endpoint.getID()); this.remoteSctpPort = remoteSctpPort; this.debugId = generateDebugId(); } /** * Adds <tt>WebRtcDataStreamListener</tt> to the list of listeners. * * @param listener the <tt>WebRtcDataStreamListener</tt> to be added to the listeners list. */ public void addChannelListener(WebRtcDataStreamListener listener) { if (listener == null) { throw new NullPointerException("listener"); } else { synchronized (listeners) { if (!listeners.contains(listener)) { listeners.add(listener); } } } } /** {@inheritDoc} */ @Override protected void closeStream() throws IOException { try { synchronized (this) { assocIsUp = false; acceptedIncomingConnection = false; if (sctpSocket != null) { sctpSocket.close(); sctpSocket = null; } } } finally { if (iceSocket != null) { // It is now the responsibility of the transport manager to // close the socket. // iceUdpSocket.close(); } } } /** {@inheritDoc} */ @Override public void expire() { try { eventDispatcher.shutdown(); } finally { super.expire(); } } /** * Gets the <tt>WebRtcDataStreamListener</tt>s added to this instance. * * @return the <tt>WebRtcDataStreamListener</tt>s added to this instance or <tt>null</tt> if there * are no <tt>WebRtcDataStreamListener</tt>s added to this instance */ private WebRtcDataStreamListener[] getChannelListeners() { WebRtcDataStreamListener[] ls; synchronized (listeners) { if (listeners.isEmpty()) { ls = null; } else { ls = listeners.toArray(new WebRtcDataStreamListener[listeners.size()]); } } return ls; } /** * Returns default <tt>WebRtcDataStream</tt> if it's ready or <tt>null</tt> otherwise. * * @return <tt>WebRtcDataStream</tt> if it's ready or <tt>null</tt> otherwise. * @throws IOException */ public WebRtcDataStream getDefaultDataStream() throws IOException { WebRtcDataStream def; synchronized (this) { if (sctpSocket == null) { def = null; } else { // Channel that runs on sid 0 def = channels.get(0); if (def == null) { def = openChannel(0, 0, 0, 0, "default"); } // Pawel Domas: Must be acknowledged before use /* * XXX Lyubomir Marinov: We're always sending ordered. According * to "WebRTC Data Channel Establishment Protocol", we can start * sending messages containing user data after the * DATA_CHANNEL_OPEN message has been sent without waiting for * the reception of the corresponding DATA_CHANNEL_ACK message. */ // if (!def.isAcknowledged()) // def = null; } } return def; } /** * Returns <tt>true</tt> if this <tt>SctpConnection</tt> is connected to the remote peer and * operational. * * @return <tt>true</tt> if this <tt>SctpConnection</tt> is connected to the remote peer and * operational */ public boolean isReady() { return assocIsUp && acceptedIncomingConnection; } /** {@inheritDoc} */ @Override protected void maybeStartStream() throws IOException { // connector final StreamConnector connector = getStreamConnector(); if (connector == null) return; synchronized (this) { if (started) return; threadPool.execute( new Runnable() { @Override public void run() { try { Sctp.init(); runOnDtlsTransport(connector); } catch (IOException e) { logger.error(e, e); } finally { try { Sctp.finish(); } catch (IOException e) { logger.error("Failed to shutdown SCTP stack", e); } } } }); started = true; } } /** * Submits {@link #notifyChannelOpenedInEventDispatcher(WebRtcDataStream)} to {@link * #eventDispatcher} for asynchronous execution. * * @param dataChannel */ private void notifyChannelOpened(final WebRtcDataStream dataChannel) { if (!isExpired()) { eventDispatcher.execute( new Runnable() { @Override public void run() { notifyChannelOpenedInEventDispatcher(dataChannel); } }); } } private void notifyChannelOpenedInEventDispatcher(WebRtcDataStream dataChannel) { /* * When executing asynchronously in eventDispatcher, it is technically * possible that this SctpConnection may have expired by now. */ if (!isExpired()) { WebRtcDataStreamListener[] ls = getChannelListeners(); if (ls != null) { for (WebRtcDataStreamListener l : ls) { l.onChannelOpened(this, dataChannel); } } } } /** * Submits {@link #notifySctpConnectionReadyInEventDispatcher()} to {@link #eventDispatcher} for * asynchronous execution. */ private void notifySctpConnectionReady() { if (!isExpired()) { eventDispatcher.execute( new Runnable() { @Override public void run() { notifySctpConnectionReadyInEventDispatcher(); } }); } } /** * Notifies the <tt>WebRtcDataStreamListener</tt>s added to this instance that this * <tt>SctpConnection</tt> is ready i.e. it is connected to the remote peer and operational. */ private void notifySctpConnectionReadyInEventDispatcher() { /* * When executing asynchronously in eventDispatcher, it is technically * possible that this SctpConnection may have expired by now. */ if (!isExpired() && isReady()) { WebRtcDataStreamListener[] ls = getChannelListeners(); if (ls != null) { for (WebRtcDataStreamListener l : ls) { l.onSctpConnectionReady(this); } } } } /** * Handles control packet. * * @param data raw packet data that arrived on control PPID. * @param sid SCTP stream id on which the data has arrived. */ private synchronized void onCtrlPacket(byte[] data, int sid) throws IOException { ByteBuffer buffer = ByteBuffer.wrap(data); int messageType = /* 1 byte unsigned integer */ 0xFF & buffer.get(); if (messageType == MSG_CHANNEL_ACK) { if (logger.isDebugEnabled()) { logger.debug(getEndpoint().getID() + " ACK received SID: " + sid); } // Open channel ACK WebRtcDataStream channel = channels.get(sid); if (channel != null) { // Ack check prevents from firing multiple notifications // if we get more than one ACKs (by mistake/bug). if (!channel.isAcknowledged()) { channel.ackReceived(); notifyChannelOpened(channel); } else { logger.warn("Redundant ACK received for SID: " + sid); } } else { logger.error("No channel exists on sid: " + sid); } } else if (messageType == MSG_OPEN_CHANNEL) { int channelType = /* 1 byte unsigned integer */ 0xFF & buffer.get(); int priority = /* 2 bytes unsigned integer */ 0xFFFF & buffer.getShort(); long reliability = /* 4 bytes unsigned integer */ 0xFFFFFFFFL & buffer.getInt(); int labelLength = /* 2 bytes unsigned integer */ 0xFFFF & buffer.getShort(); int protocolLength = /* 2 bytes unsigned integer */ 0xFFFF & buffer.getShort(); String label; String protocol; if (labelLength == 0) { label = ""; } else { byte[] labelBytes = new byte[labelLength]; buffer.get(labelBytes); label = new String(labelBytes, "UTF-8"); } if (protocolLength == 0) { protocol = ""; } else { byte[] protocolBytes = new byte[protocolLength]; buffer.get(protocolBytes); protocol = new String(protocolBytes, "UTF-8"); } if (logger.isDebugEnabled()) { logger.debug( "!!! " + getEndpoint().getID() + " data channel open request on SID: " + sid + " type: " + channelType + " prio: " + priority + " reliab: " + reliability + " label: " + label + " proto: " + protocol); } if (channels.containsKey(sid)) { logger.error("Channel on sid: " + sid + " already exists"); } WebRtcDataStream newChannel = new WebRtcDataStream(sctpSocket, sid, label, true); channels.put(sid, newChannel); sendOpenChannelAck(sid); notifyChannelOpened(newChannel); } else { logger.error("Unexpected ctrl msg type: " + messageType); } } /** {@inheritDoc} */ @Override protected void onEndpointChanged(Endpoint oldValue, Endpoint newValue) { if (oldValue != null) oldValue.setSctpConnection(null); if (newValue != null) newValue.setSctpConnection(this); } /** Implements notification in order to track socket state. */ @Override public synchronized void onSctpNotification(SctpSocket socket, SctpNotification notification) { if (logger.isDebugEnabled()) { logger.debug("socket=" + socket + "; notification=" + notification); } switch (notification.sn_type) { case SctpNotification.SCTP_ASSOC_CHANGE: SctpNotification.AssociationChange assocChange = (SctpNotification.AssociationChange) notification; switch (assocChange.state) { case SctpNotification.AssociationChange.SCTP_COMM_UP: if (!assocIsUp) { boolean wasReady = isReady(); assocIsUp = true; if (isReady() && !wasReady) notifySctpConnectionReady(); } break; case SctpNotification.AssociationChange.SCTP_COMM_LOST: case SctpNotification.AssociationChange.SCTP_SHUTDOWN_COMP: case SctpNotification.AssociationChange.SCTP_CANT_STR_ASSOC: try { closeStream(); } catch (IOException e) { logger.error("Error closing SCTP socket", e); } break; } break; } } /** * {@inheritDoc} * * <p>SCTP input data callback. */ @Override public void onSctpPacket( byte[] data, int sid, int ssn, int tsn, long ppid, int context, int flags) { if (ppid == WEB_RTC_PPID_CTRL) { // Channel control PPID try { onCtrlPacket(data, sid); } catch (IOException e) { logger.error("IOException when processing ctrl packet", e); } } else if (ppid == WEB_RTC_PPID_STRING || ppid == WEB_RTC_PPID_BIN) { WebRtcDataStream channel; synchronized (this) { channel = channels.get(sid); } if (channel == null) { logger.error("No channel found for sid: " + sid); return; } if (ppid == WEB_RTC_PPID_STRING) { // WebRTC String String str; String charsetName = "UTF-8"; try { str = new String(data, charsetName); } catch (UnsupportedEncodingException uee) { logger.error("Unsupported charset encoding/name " + charsetName, uee); str = null; } channel.onStringMsg(str); } else { // WebRTC Binary channel.onBinaryMsg(data); } } else { logger.warn("Got message on unsupported PPID: " + ppid); } } /** * Opens new WebRTC data channel using specified parameters. * * @param type channel type as defined in control protocol description. Use 0 for "reliable". * @param prio channel priority. The higher the number, the lower the priority. * @param reliab Reliability Parameter<br> * This field is ignored if a reliable channel is used. If a partial reliable channel with * limited number of retransmissions is used, this field specifies the number of * retransmissions. If a partial reliable channel with limited lifetime is used, this field * specifies the maximum lifetime in milliseconds. The following table summarizes this:<br> * </br> * <p>+------------------------------------------------+------------------+ | Channel Type | * Reliability | | | Parameter | * +------------------------------------------------+------------------+ | * DATA_CHANNEL_RELIABLE | Ignored | | DATA_CHANNEL_RELIABLE_UNORDERED | Ignored | | * DATA_CHANNEL_PARTIAL_RELIABLE_REXMIT | Number of RTX | | * DATA_CHANNEL_PARTIAL_RELIABLE_REXMIT_UNORDERED | Number of RTX | | * DATA_CHANNEL_PARTIAL_RELIABLE_TIMED | Lifetime in ms | | * DATA_CHANNEL_PARTIAL_RELIABLE_TIMED_UNORDERED | Lifetime in ms | * +------------------------------------------------+------------------+ * @param sid SCTP stream id that will be used by new channel (it must not be already used). * @param label text label for the channel. * @return new instance of <tt>WebRtcDataStream</tt> that represents opened WebRTC data channel. * @throws IOException if IO error occurs. */ public synchronized WebRtcDataStream openChannel( int type, int prio, long reliab, int sid, String label) throws IOException { if (channels.containsKey(sid)) { throw new IOException("Channel on sid: " + sid + " already exists"); } // Label Length & Label byte[] labelBytes; int labelByteLength; if (label == null) { labelBytes = null; labelByteLength = 0; } else { labelBytes = label.getBytes("UTF-8"); labelByteLength = labelBytes.length; if (labelByteLength > 0xFFFF) labelByteLength = 0xFFFF; } // Protocol Length & Protocol String protocol = WEBRTC_DATA_CHANNEL_PROTOCOL; byte[] protocolBytes; int protocolByteLength; if (protocol == null) { protocolBytes = null; protocolByteLength = 0; } else { protocolBytes = protocol.getBytes("UTF-8"); protocolByteLength = protocolBytes.length; if (protocolByteLength > 0xFFFF) protocolByteLength = 0xFFFF; } ByteBuffer packet = ByteBuffer.allocate(12 + labelByteLength + protocolByteLength); // Message open new channel on current sid // Message Type packet.put((byte) MSG_OPEN_CHANNEL); // Channel Type packet.put((byte) type); // Priority packet.putShort((short) prio); // Reliability Parameter packet.putInt((int) reliab); // Label Length packet.putShort((short) labelByteLength); // Protocol Length packet.putShort((short) protocolByteLength); // Label if (labelByteLength != 0) { packet.put(labelBytes, 0, labelByteLength); } // Protocol if (protocolByteLength != 0) { packet.put(protocolBytes, 0, protocolByteLength); } int sentCount = sctpSocket.send(packet.array(), true, sid, WEB_RTC_PPID_CTRL); if (sentCount != packet.capacity()) { throw new IOException("Failed to open new chanel on sid: " + sid); } WebRtcDataStream channel = new WebRtcDataStream(sctpSocket, sid, label, false); channels.put(sid, channel); return channel; } /** * Removes <tt>WebRtcDataStreamListener</tt> from the list of listeners. * * @param listener the <tt>WebRtcDataStreamListener</tt> to be removed from the listeners list. */ public void removeChannelListener(WebRtcDataStreamListener listener) { if (listener != null) { synchronized (listeners) { listeners.remove(listener); } } } private void runOnDtlsTransport(StreamConnector connector) throws IOException { DtlsControlImpl dtlsControl = (DtlsControlImpl) getTransportManager().getDtlsControl(this); DtlsTransformEngine engine = dtlsControl.getTransformEngine(); final DtlsPacketTransformer transformer = (DtlsPacketTransformer) engine.getRTPTransformer(); byte[] receiveBuffer = new byte[SCTP_BUFFER_SIZE]; if (LOG_SCTP_PACKETS) { System.setProperty( ConfigurationService.PNAME_SC_HOME_DIR_LOCATION, System.getProperty("java.io.tmpdir")); System.setProperty( ConfigurationService.PNAME_SC_HOME_DIR_NAME, SctpConnection.class.getName()); } synchronized (this) { // FIXME local SCTP port is hardcoded in bridge offer SDP (Jitsi // Meet) sctpSocket = Sctp.createSocket(5000); assocIsUp = false; acceptedIncomingConnection = false; } // Implement output network link for SCTP stack on DTLS transport sctpSocket.setLink( new NetworkLink() { @Override public void onConnOut(SctpSocket s, byte[] packet) throws IOException { if (LOG_SCTP_PACKETS) { LibJitsi.getPacketLoggingService() .logPacket( PacketLoggingService.ProtocolName.ICE4J, new byte[] {0, 0, 0, (byte) debugId}, 5000, new byte[] {0, 0, 0, (byte) (debugId + 1)}, remoteSctpPort, PacketLoggingService.TransportName.UDP, true, packet); } // Send through DTLS transport transformer.sendApplicationData(packet, 0, packet.length); } }); if (logger.isDebugEnabled()) { logger.debug("Connecting SCTP to port: " + remoteSctpPort + " to " + getEndpoint().getID()); } sctpSocket.setNotificationListener(this); sctpSocket.listen(); // FIXME manage threads threadPool.execute( new Runnable() { @Override public void run() { SctpSocket sctpSocket = null; try { // sctpSocket is set to null on close sctpSocket = SctpConnection.this.sctpSocket; while (sctpSocket != null) { if (sctpSocket.accept()) { acceptedIncomingConnection = true; break; } Thread.sleep(100); sctpSocket = SctpConnection.this.sctpSocket; } if (isReady()) { notifySctpConnectionReady(); } } catch (Exception e) { logger.error("Error accepting SCTP connection", e); } if (sctpSocket == null && logger.isInfoEnabled()) { logger.info( "SctpConnection " + getID() + " closed" + " before SctpSocket accept()-ed."); } } }); // Notify that from now on SCTP connection is considered functional sctpSocket.setDataCallback(this); // Setup iceSocket DatagramSocket datagramSocket = connector.getDataSocket(); if (datagramSocket != null) { this.iceSocket = new IceUdpSocketWrapper(datagramSocket); } else { this.iceSocket = new IceTcpSocketWrapper(connector.getDataTCPSocket()); } DatagramPacket rcvPacket = new DatagramPacket(receiveBuffer, 0, receiveBuffer.length); // Receive loop, breaks when SCTP socket is closed try { do { iceSocket.receive(rcvPacket); RawPacket raw = new RawPacket(rcvPacket.getData(), rcvPacket.getOffset(), rcvPacket.getLength()); raw = transformer.reverseTransform(raw); // Check for app data if (raw == null) continue; if (LOG_SCTP_PACKETS) { LibJitsi.getPacketLoggingService() .logPacket( PacketLoggingService.ProtocolName.ICE4J, new byte[] {0, 0, 0, (byte) (debugId + 1)}, remoteSctpPort, new byte[] {0, 0, 0, (byte) debugId}, 5000, PacketLoggingService.TransportName.UDP, false, raw.getBuffer(), raw.getOffset(), raw.getLength()); } // Pass network packet to SCTP stack sctpSocket.onConnIn(raw.getBuffer(), raw.getOffset(), raw.getLength()); } while (true); } finally { // Eventually, close the socket although it should happen from // expire(). synchronized (this) { assocIsUp = false; acceptedIncomingConnection = false; if (sctpSocket != null) { sctpSocket.close(); sctpSocket = null; } } } } /** * Sends acknowledgment for open channel request on given SCTP stream ID. * * @param sid SCTP stream identifier to be used for sending ack. */ private void sendOpenChannelAck(int sid) throws IOException { // Send ACK byte[] ack = MSG_CHANNEL_ACK_BYTES; int sendAck = sctpSocket.send(ack, true, sid, WEB_RTC_PPID_CTRL); if (sendAck != ack.length) { logger.error("Failed to send open channel confirmation"); } } /** * {@inheritDoc} * * <p>Creates a <tt>TransportManager</tt> instance suitable for an <tt>SctpConnection</tt> (e.g. * with 1 component only). */ protected TransportManager createTransportManager(String xmlNamespace) throws IOException { if (IceUdpTransportPacketExtension.NAMESPACE.equals(xmlNamespace)) { Content content = getContent(); return new IceUdpTransportManager( content.getConference(), isInitiator(), 1 /* num components */, content.getName()); } else if (RawUdpTransportPacketExtension.NAMESPACE.equals(xmlNamespace)) { // TODO: support RawUdp once RawUdpTransportManager is updated // return new RawUdpTransportManager(this); throw new IllegalArgumentException("Unsupported Jingle transport " + xmlNamespace); } else { throw new IllegalArgumentException("Unsupported Jingle transport " + xmlNamespace); } } }
/** * The <tt>SimulcastReceiver</tt> of a <tt>SimulcastEngine</tt> receives the simulcast streams from * a simulcast enabled participant and manages 1 or more <tt>SimulcastLayer</tt>s. It fires a * property change event whenever the simulcast layers that it manages change. * * <p>This class is thread safe. * * @author George Politis * @author Lyubomir Marinov */ public class SimulcastReceiver extends PropertyChangeNotifier { /** * The <tt>Logger</tt> used by the <tt>ReceivingLayers</tt> class and its instances to print debug * information. */ private static final Logger logger = Logger.getLogger(SimulcastReceiver.class); /** * The name of the property that gets fired when there's a change in the simulcast layers that * this receiver manages. */ public static final String SIMULCAST_LAYERS_PNAME = SimulcastReceiver.class.getName() + ".simulcastLayers"; /** * The number of (video) frames which defines the interval of time (indirectly) during which a * {@code SimulcastLayer} needs to receive data from its remote peer or it will be declared * paused/stopped/not streaming by its {@code SimulcastReceiver}. */ static final int TIMEOUT_ON_FRAME_COUNT = 5; /** The pool of threads utilized by this class. */ private static final ExecutorService executorService = ExecutorUtils.newCachedThreadPool(true, SimulcastReceiver.class.getName()); /** Helper object that <tt>SwitchingSimulcastSender</tt> instances use to build JSON messages. */ private static final SimulcastMessagesMapper mapper = new SimulcastMessagesMapper(); /** The <tt>SimulcastEngine</tt> that owns this receiver. */ private final SimulcastEngine simulcastEngine; /** The simulcast layers of this <tt>VideoChannel</tt>. */ private SimulcastLayer[] simulcastLayers; /** * Indicates whether we're receiving native or non-native simulcast from the associated endpoint. * It determines whether the bridge should send messages over the data channels to manage the * non-native simulcast. In the case of native simulcast, there's nothing to do for the bridge. * * <p>NOTE that at the time of this writing we only support native simulcast. Last time we tried * non-native simulcast there was no way to limit the bitrate of lower layer streams and thus * there was no point in implementing non-native simulcast. * * <p>NOTE^2 This has changed recently with the webrtc stack automatically limiting the stream * bitrate based on its resolution (see commit 1c7d48d431e098ba42fa6bd9f1cfe69a703edee5 in the * webrtc git repository). So it might be something that we will want to implement in the future * for browsers that don't support native simulcast (Temasys). */ private boolean nativeSimulcast = true; /** * The history of the order/sequence of receipt of (video) frames by {@link #simulcastLayers}. * Used in an attempt to speed up the detection of paused/stopped {@code SimulcastLayer}s by * counting (video) frames. */ private final List<SimulcastLayer> simulcastLayerFrameHistory = new LinkedList<SimulcastLayer>(); /** * Ctor. * * @param simulcastEngine the <tt>SimulcastEngine</tt> that owns this receiver. */ public SimulcastReceiver(SimulcastEngine simulcastEngine) { this.simulcastEngine = simulcastEngine; } /** * Gets the <tt>SimulcastEngine</tt> that owns this receiver. * * @return the <tt>SimulcastEngine</tt> that owns this receiver. */ public SimulcastEngine getSimulcastEngine() { return this.simulcastEngine; } /** * Returns true if the endpoint has signaled two or more simulcast layers. * * @return true if the endpoint has signaled two or more simulcast layers, false otherwise. */ public boolean hasLayers() { SimulcastLayer[] sl = simulcastLayers; return sl != null && sl.length != 0; } /** * Returns a <tt>SimulcastLayer</tt> that is the closest match to the target order, or null if * simulcast hasn't been configured for this receiver. * * @param targetOrder the simulcast layer target order. * @return a <tt>SimulcastLayer</tt> that is the closest match to the target order, or null. */ public SimulcastLayer getSimulcastLayer(int targetOrder) { SimulcastLayer[] layers = getSimulcastLayers(); if (layers == null || layers.length == 0) { return null; } // Iterate through the simulcast layers that we own and return the one // that matches best the targetOrder parameter. SimulcastLayer next = layers[0]; for (int i = 1; i < Math.min(targetOrder + 1, layers.length); i++) { if (!layers[i].isStreaming()) { break; } next = layers[i]; } return next; } /** * Gets the simulcast layers of this simulcast manager in a new <tt>SortedSet</tt> so that the * caller won't have to worry about the structure changing by some other thread. * * @return the simulcast layers of this receiver in a new sorted set if simulcast is signaled, or * null. */ public SimulcastLayer[] getSimulcastLayers() { return simulcastLayers; } /** * Sets the simulcast layers for this receiver and fires an event about it. * * @param simulcastLayers the simulcast layers for this receiver. */ public void setSimulcastLayers(SimulcastLayer[] simulcastLayers) { this.simulcastLayers = simulcastLayers; if (logger.isInfoEnabled()) { if (simulcastLayers == null) { logInfo("Simulcast disabled."); } else { for (SimulcastLayer l : simulcastLayers) { logInfo(l.getOrder() + ": " + l.getPrimarySSRC()); } } } executorService.execute( new Runnable() { public void run() { firePropertyChange(SIMULCAST_LAYERS_PNAME, null, null); } }); // TODO If simulcastLayers has changed, then simulcastLayerFrameHistory // has very likely become irrelevant. In other words, clear // simulcastLayerFrameHistory. } /** * Notifies this instance that a <tt>DatagramPacket</tt> packet received on the data * <tt>DatagramSocket</tt> of this <tt>Channel</tt> has been accepted for further processing * within Jitsi Videobridge. * * @param pkt the accepted <tt>RawPacket</tt>. */ public void accepted(RawPacket pkt) { // With native simulcast we don't have a notification when a stream // has started/stopped. The simulcast manager implements a timeout // for the high quality stream and it needs to be notified when // the channel has accepted a datagram packet for the timeout to // function correctly. if (!hasLayers() || pkt == null) { return; } // Find the layer that corresponds to this packet. int acceptedSSRC = pkt.getSSRC(); SimulcastLayer[] layers = getSimulcastLayers(); SimulcastLayer acceptedLayer = null; for (SimulcastLayer layer : layers) { // We only care about the primary SSRC and not the RTX ssrc (or // future FEC ssrc). if ((int) layer.getPrimarySSRC() == acceptedSSRC) { acceptedLayer = layer; break; } } // If this is not an RTP packet or if we can't find an accepted // layer, log and return as it makes no sense to continue in this // situation. if (acceptedLayer == null) { return; } // There are sequences of packets with increasing timestamps but without // the marker bit set. Supposedly, they are probes to detect whether the // bandwidth may increase. We think that they should cause neither the // start nor the stop of any SimulcastLayer. // XXX There's RawPacket#getPayloadLength() but the implementation // includes pkt.paddingSize at the time of this writing and we do not // know whether that's going to stay that way. int pktPayloadLength = pkt.getLength() - pkt.getHeaderLength(); int pktPaddingSize = pkt.getPaddingSize(); if (pktPayloadLength <= pktPaddingSize) { if (logger.isTraceEnabled()) { logger.trace( "pkt.payloadLength= " + pktPayloadLength + " <= pkt.paddingSize= " + pktPaddingSize + "(" + pkt.getSequenceNumber() + ")"); } return; } // NOTE(gp) we expect the base layer to be always on, so we never touch // it or starve it. // XXX Refer to the implementation of // SimulcastLayer#touch(boolean, RawPacket) for an explanation of why we // chose to use a return value. boolean frameStarted = acceptedLayer.touch(pkt); if (frameStarted) simulcastLayerFrameStarted(acceptedLayer, pkt, layers); } /** * Maybe send a data channel command to the associated <tt>Endpoint</tt> to make it start * streaming its hq stream, if it's being watched by some receiver. */ public void maybeSendStartHighQualityStreamCommand() { if (nativeSimulcast || !hasLayers()) { // In native simulcast the client adjusts its layers autonomously so // we don't need (nor we can) to control it with data channel // messages. return; } Endpoint newEndpoint = getSimulcastEngine().getVideoChannel().getEndpoint(); SimulcastLayer[] newSimulcastLayers = getSimulcastLayers(); SctpConnection sctpConnection; if (newSimulcastLayers == null || newSimulcastLayers.length <= 1 /* newEndpoint != null is implied */ || (sctpConnection = newEndpoint.getSctpConnection()) == null || !sctpConnection.isReady() || sctpConnection.isExpired()) { return; } // we have a new endpoint and it has an SCTP connection that is // ready and not expired. if somebody else is watching the new // endpoint, start its hq stream. boolean startHighQualityStream = false; for (Endpoint e : getSimulcastEngine().getVideoChannel().getContent().getConference().getEndpoints()) { // TODO(gp) need some synchronization here. What if the // selected endpoint changes while we're in the loop? if (e == newEndpoint) continue; Endpoint eSelectedEndpoint = e.getEffectivelySelectedEndpoint(); if (newEndpoint == eSelectedEndpoint) { // somebody is watching the new endpoint or somebody has not // yet signaled its selected endpoint to the bridge, start // the hq stream. if (logger.isDebugEnabled()) { Map<String, Object> map = new HashMap<String, Object>(3); map.put("e", e); map.put("newEndpoint", newEndpoint); map.put("maybe", eSelectedEndpoint == null ? "(maybe) " : ""); StringCompiler sc = new StringCompiler(map).c("{e.id} is {maybe} watching {newEndpoint.id}."); logDebug(sc.toString().replaceAll("\\s+", " ")); } startHighQualityStream = true; break; } } if (startHighQualityStream) { // TODO(gp) this assumes only a single hq stream. logDebug( getSimulcastEngine().getVideoChannel().getEndpoint().getID() + " notifies " + newEndpoint.getID() + " to start its HQ stream."); SimulcastLayer hqLayer = newSimulcastLayers[newSimulcastLayers.length - 1]; ; StartSimulcastLayerCommand command = new StartSimulcastLayerCommand(hqLayer); String json = mapper.toJson(command); try { newEndpoint.sendMessageOnDataChannel(json); } catch (IOException e) { logError(newEndpoint.getID() + " failed to send message on data channel.", e); } } } /** * Maybe send a data channel command to he associated simulcast sender to make it stop streaming * its hq stream, if it's not being watched by any participant. */ public void maybeSendStopHighQualityStreamCommand() { if (nativeSimulcast || !hasLayers()) { // In native simulcast the client adjusts its layers autonomously so // we don't need (nor we can) to control it with data channel // messages. return; } Endpoint oldEndpoint = getSimulcastEngine().getVideoChannel().getEndpoint(); SimulcastLayer[] oldSimulcastLayers = getSimulcastLayers(); SctpConnection sctpConnection; if (oldSimulcastLayers != null && oldSimulcastLayers.length > 1 /* oldEndpoint != null is implied*/ && (sctpConnection = oldEndpoint.getSctpConnection()) != null && sctpConnection.isReady() && !sctpConnection.isExpired()) { // we have an old endpoint and it has an SCTP connection that is // ready and not expired. if nobody else is watching the old // endpoint, stop its hq stream. boolean stopHighQualityStream = true; for (Endpoint e : getSimulcastEngine().getVideoChannel().getContent().getConference().getEndpoints()) { // TODO(gp) need some synchronization here. What if the selected // endpoint changes while we're in the loop? if (oldEndpoint != e && (oldEndpoint == e.getEffectivelySelectedEndpoint()) || e.getEffectivelySelectedEndpoint() == null) { // somebody is watching the old endpoint or somebody has not // yet signaled its selected endpoint to the bridge, don't // stop the hq stream. stopHighQualityStream = false; break; } } if (stopHighQualityStream) { // TODO(gp) this assumes only a single hq stream. logDebug( getSimulcastEngine().getVideoChannel().getEndpoint().getID() + " notifies " + oldEndpoint.getID() + " to stop " + "its HQ stream."); SimulcastLayer hqLayer = oldSimulcastLayers[oldSimulcastLayers.length - 1]; StopSimulcastLayerCommand command = new StopSimulcastLayerCommand(hqLayer); String json = mapper.toJson(command); try { oldEndpoint.sendMessageOnDataChannel(json); } catch (IOException e1) { logError(oldEndpoint.getID() + " failed to send " + "message on data channel.", e1); } } } } private void logDebug(String msg) { if (logger.isDebugEnabled()) { msg = getSimulcastEngine().getVideoChannel().getEndpoint().getID() + ": " + msg; logger.debug(msg); } } private void logWarn(String msg) { if (logger.isWarnEnabled()) { msg = getSimulcastEngine().getVideoChannel().getEndpoint().getID() + ": " + msg; logger.warn(msg); } } private void logError(String msg, Throwable e) { msg = getSimulcastEngine().getVideoChannel().getEndpoint().getID() + ": " + msg; logger.error(msg, e); } private void logInfo(String msg) { if (logger.isInfoEnabled()) { msg = getSimulcastEngine().getVideoChannel().getEndpoint().getID() + ": " + msg; logger.info(msg); } } /** * Notifies this {@code SimulcastReceiver} that a specific {@code SimulcastReceiver} has detected * the start of a new video frame in the RTP stream that it represents. Determines whether any of * {@link #simulcastLayers} other than {@code source} have been paused/stopped by the remote peer. * The determination is based on counting (video) frames. * * @param source the {@code SimulcastLayer} which is the source of the event i.e. which has * detected the start of a new video frame in the RTP stream that it represents * @param pkt the {@code RawPacket} which was received by {@code source} and possibly influenced * the decision that a new view frame was started in the RTP stream represented by {@code * source} * @param layers the set of {@code SimulcastLayer}s managed by this {@code SimulcastReceiver}. * Explicitly provided to the method in order to avoid invocations of {@link * #getSimulcastLayers()} because the latter makes a copy at the time of this writing. */ private void simulcastLayerFrameStarted( SimulcastLayer source, RawPacket pkt, SimulcastLayer[] layers) { // Allow the value of the constant TIMEOUT_ON_FRAME_COUNT to disable (at // compile time) the frame-based approach to the detection of layer // drops. if (TIMEOUT_ON_FRAME_COUNT <= 1) return; // Timeouts in layers caused by source may occur only based on the span // (of time or received frames) during which source has received // TIMEOUT_ON_FRAME_COUNT number of frames. The current method // invocation signals the receipt of 1 frame by source. int indexOfLastSourceOccurrenceInHistory = -1; int sourceFrameCount = 0; int ix = 0; for (Iterator<SimulcastLayer> it = simulcastLayerFrameHistory.iterator(); it.hasNext(); ++ix) { if (it.next() == source) { if (indexOfLastSourceOccurrenceInHistory != -1) { // Prune simulcastLayerFrameHistory so that it does not // become unnecessarily long. it.remove(); } else if (++sourceFrameCount >= TIMEOUT_ON_FRAME_COUNT - 1) { // The span of TIMEOUT_ON_FRAME_COUNT number of frames // received by source only is to be examined for the // purposes of timeouts. The current method invocations // signals the receipt of 1 frame by source so // TIMEOUT_ON_FRAME_COUNT - 1 occurrences of source in // simulcastLayerFrameHistory is enough. indexOfLastSourceOccurrenceInHistory = ix; } } } if (indexOfLastSourceOccurrenceInHistory != -1) { // Presumably, if a SimulcastLayer is active, all SimulcastLayers // before it (according to SimulcastLayer's order) are active as // well. Consequently, timeouts may occur in SimulcastLayers which // are after source. boolean maybeTimeout = false; for (SimulcastLayer layer : layers) { if (maybeTimeout) { // There's no point in timing layer out if it's timed out // already. if (layer.isStreaming()) { maybeTimeout(source, pkt, layer, indexOfLastSourceOccurrenceInHistory); } } else if (layer == source) { maybeTimeout = true; } } } // As previously stated, the current method invocation signals the // receipt of 1 frame by source. simulcastLayerFrameHistory.add(0, source); // TODO Prune simulcastLayerFrameHistory by forgetting so that it does // not become too long. } /** * Determines whether {@code effect} has been paused/stopped by the remote peer. The determination * is based on counting frames and is triggered by the receipt of (a piece of) a new (video) frame * by {@code cause}. * * @param cause the {@code SimulcastLayer} which has received (a piece of) a new (video) frame and * has thus triggered a check on {@code effect} * @param pkt the {@code RawPacket} which was received by {@code cause} and possibly influenced * the decision to trigger a check on {@code effect} * @param effect the {@code SimulcastLayer} which is to be checked whether it looks like it has * been paused/stopped by the remote peer * @param endIndexInSimulcastLayerFrameHistory */ private void maybeTimeout( SimulcastLayer cause, RawPacket pkt, SimulcastLayer effect, int endIndexInSimulcastLayerFrameHistory) { Iterator<SimulcastLayer> it = simulcastLayerFrameHistory.iterator(); boolean timeout = true; for (int ix = 0; it.hasNext() && ix < endIndexInSimulcastLayerFrameHistory; ++ix) { if (it.next() == effect) { timeout = false; break; } } if (timeout) { effect.maybeTimeout(pkt); if (!effect.isStreaming()) { // Since effect has been determined to have been paused/stopped // by the remote peer, its possible presence in // simulcastLayerFrameHistory is irrelevant now. In other words, // remove effect from simulcastLayerFrameHistory. while (it.hasNext()) { if (it.next() == effect) it.remove(); } } } } }