/** * Sets the simulcast layers for this receiver and fires an event about it. * * @param simulcastLayers the simulcast layers for this receiver. */ public void setSimulcastLayers(SimulcastLayer[] simulcastLayers) { this.simulcastLayers = simulcastLayers; if (logger.isInfoEnabled()) { if (simulcastLayers == null) { logInfo("Simulcast disabled."); } else { for (SimulcastLayer l : simulcastLayers) { logInfo(l.getOrder() + ": " + l.getPrimarySSRC()); } } } executorService.execute( new Runnable() { public void run() { firePropertyChange(SIMULCAST_LAYERS_PNAME, null, null); } }); // TODO If simulcastLayers has changed, then simulcastLayerFrameHistory // has very likely become irrelevant. In other words, clear // simulcastLayerFrameHistory. }
private void logInfo(String msg) { if (logger.isInfoEnabled()) { msg = getSimulcastEngine().getVideoChannel().getEndpoint().getID() + ": " + msg; logger.info(msg); } }
private void logError(String msg, Throwable e) { msg = getSimulcastEngine().getVideoChannel().getEndpoint().getID() + ": " + msg; logger.error(msg, e); }
private void logDebug(String msg) { if (logger.isDebugEnabled()) { msg = getSimulcastEngine().getVideoChannel().getEndpoint().getID() + ": " + msg; logger.debug(msg); } }
/** * The <tt>SimulcastReceiver</tt> of a <tt>SimulcastEngine</tt> receives the simulcast streams from * a simulcast enabled participant and manages 1 or more <tt>SimulcastLayer</tt>s. It fires a * property change event whenever the simulcast layers that it manages change. * * <p>This class is thread safe. * * @author George Politis * @author Lyubomir Marinov */ public class SimulcastReceiver extends PropertyChangeNotifier { /** * The <tt>Logger</tt> used by the <tt>ReceivingLayers</tt> class and its instances to print debug * information. */ private static final Logger logger = Logger.getLogger(SimulcastReceiver.class); /** * The name of the property that gets fired when there's a change in the simulcast layers that * this receiver manages. */ public static final String SIMULCAST_LAYERS_PNAME = SimulcastReceiver.class.getName() + ".simulcastLayers"; /** * The number of (video) frames which defines the interval of time (indirectly) during which a * {@code SimulcastLayer} needs to receive data from its remote peer or it will be declared * paused/stopped/not streaming by its {@code SimulcastReceiver}. */ static final int TIMEOUT_ON_FRAME_COUNT = 5; /** The pool of threads utilized by this class. */ private static final ExecutorService executorService = ExecutorUtils.newCachedThreadPool(true, SimulcastReceiver.class.getName()); /** Helper object that <tt>SwitchingSimulcastSender</tt> instances use to build JSON messages. */ private static final SimulcastMessagesMapper mapper = new SimulcastMessagesMapper(); /** The <tt>SimulcastEngine</tt> that owns this receiver. */ private final SimulcastEngine simulcastEngine; /** The simulcast layers of this <tt>VideoChannel</tt>. */ private SimulcastLayer[] simulcastLayers; /** * Indicates whether we're receiving native or non-native simulcast from the associated endpoint. * It determines whether the bridge should send messages over the data channels to manage the * non-native simulcast. In the case of native simulcast, there's nothing to do for the bridge. * * <p>NOTE that at the time of this writing we only support native simulcast. Last time we tried * non-native simulcast there was no way to limit the bitrate of lower layer streams and thus * there was no point in implementing non-native simulcast. * * <p>NOTE^2 This has changed recently with the webrtc stack automatically limiting the stream * bitrate based on its resolution (see commit 1c7d48d431e098ba42fa6bd9f1cfe69a703edee5 in the * webrtc git repository). So it might be something that we will want to implement in the future * for browsers that don't support native simulcast (Temasys). */ private boolean nativeSimulcast = true; /** * The history of the order/sequence of receipt of (video) frames by {@link #simulcastLayers}. * Used in an attempt to speed up the detection of paused/stopped {@code SimulcastLayer}s by * counting (video) frames. */ private final List<SimulcastLayer> simulcastLayerFrameHistory = new LinkedList<SimulcastLayer>(); /** * Ctor. * * @param simulcastEngine the <tt>SimulcastEngine</tt> that owns this receiver. */ public SimulcastReceiver(SimulcastEngine simulcastEngine) { this.simulcastEngine = simulcastEngine; } /** * Gets the <tt>SimulcastEngine</tt> that owns this receiver. * * @return the <tt>SimulcastEngine</tt> that owns this receiver. */ public SimulcastEngine getSimulcastEngine() { return this.simulcastEngine; } /** * Returns true if the endpoint has signaled two or more simulcast layers. * * @return true if the endpoint has signaled two or more simulcast layers, false otherwise. */ public boolean hasLayers() { SimulcastLayer[] sl = simulcastLayers; return sl != null && sl.length != 0; } /** * Returns a <tt>SimulcastLayer</tt> that is the closest match to the target order, or null if * simulcast hasn't been configured for this receiver. * * @param targetOrder the simulcast layer target order. * @return a <tt>SimulcastLayer</tt> that is the closest match to the target order, or null. */ public SimulcastLayer getSimulcastLayer(int targetOrder) { SimulcastLayer[] layers = getSimulcastLayers(); if (layers == null || layers.length == 0) { return null; } // Iterate through the simulcast layers that we own and return the one // that matches best the targetOrder parameter. SimulcastLayer next = layers[0]; for (int i = 1; i < Math.min(targetOrder + 1, layers.length); i++) { if (!layers[i].isStreaming()) { break; } next = layers[i]; } return next; } /** * Gets the simulcast layers of this simulcast manager in a new <tt>SortedSet</tt> so that the * caller won't have to worry about the structure changing by some other thread. * * @return the simulcast layers of this receiver in a new sorted set if simulcast is signaled, or * null. */ public SimulcastLayer[] getSimulcastLayers() { return simulcastLayers; } /** * Sets the simulcast layers for this receiver and fires an event about it. * * @param simulcastLayers the simulcast layers for this receiver. */ public void setSimulcastLayers(SimulcastLayer[] simulcastLayers) { this.simulcastLayers = simulcastLayers; if (logger.isInfoEnabled()) { if (simulcastLayers == null) { logInfo("Simulcast disabled."); } else { for (SimulcastLayer l : simulcastLayers) { logInfo(l.getOrder() + ": " + l.getPrimarySSRC()); } } } executorService.execute( new Runnable() { public void run() { firePropertyChange(SIMULCAST_LAYERS_PNAME, null, null); } }); // TODO If simulcastLayers has changed, then simulcastLayerFrameHistory // has very likely become irrelevant. In other words, clear // simulcastLayerFrameHistory. } /** * Notifies this instance that a <tt>DatagramPacket</tt> packet received on the data * <tt>DatagramSocket</tt> of this <tt>Channel</tt> has been accepted for further processing * within Jitsi Videobridge. * * @param pkt the accepted <tt>RawPacket</tt>. */ public void accepted(RawPacket pkt) { // With native simulcast we don't have a notification when a stream // has started/stopped. The simulcast manager implements a timeout // for the high quality stream and it needs to be notified when // the channel has accepted a datagram packet for the timeout to // function correctly. if (!hasLayers() || pkt == null) { return; } // Find the layer that corresponds to this packet. int acceptedSSRC = pkt.getSSRC(); SimulcastLayer[] layers = getSimulcastLayers(); SimulcastLayer acceptedLayer = null; for (SimulcastLayer layer : layers) { // We only care about the primary SSRC and not the RTX ssrc (or // future FEC ssrc). if ((int) layer.getPrimarySSRC() == acceptedSSRC) { acceptedLayer = layer; break; } } // If this is not an RTP packet or if we can't find an accepted // layer, log and return as it makes no sense to continue in this // situation. if (acceptedLayer == null) { return; } // There are sequences of packets with increasing timestamps but without // the marker bit set. Supposedly, they are probes to detect whether the // bandwidth may increase. We think that they should cause neither the // start nor the stop of any SimulcastLayer. // XXX There's RawPacket#getPayloadLength() but the implementation // includes pkt.paddingSize at the time of this writing and we do not // know whether that's going to stay that way. int pktPayloadLength = pkt.getLength() - pkt.getHeaderLength(); int pktPaddingSize = pkt.getPaddingSize(); if (pktPayloadLength <= pktPaddingSize) { if (logger.isTraceEnabled()) { logger.trace( "pkt.payloadLength= " + pktPayloadLength + " <= pkt.paddingSize= " + pktPaddingSize + "(" + pkt.getSequenceNumber() + ")"); } return; } // NOTE(gp) we expect the base layer to be always on, so we never touch // it or starve it. // XXX Refer to the implementation of // SimulcastLayer#touch(boolean, RawPacket) for an explanation of why we // chose to use a return value. boolean frameStarted = acceptedLayer.touch(pkt); if (frameStarted) simulcastLayerFrameStarted(acceptedLayer, pkt, layers); } /** * Maybe send a data channel command to the associated <tt>Endpoint</tt> to make it start * streaming its hq stream, if it's being watched by some receiver. */ public void maybeSendStartHighQualityStreamCommand() { if (nativeSimulcast || !hasLayers()) { // In native simulcast the client adjusts its layers autonomously so // we don't need (nor we can) to control it with data channel // messages. return; } Endpoint newEndpoint = getSimulcastEngine().getVideoChannel().getEndpoint(); SimulcastLayer[] newSimulcastLayers = getSimulcastLayers(); SctpConnection sctpConnection; if (newSimulcastLayers == null || newSimulcastLayers.length <= 1 /* newEndpoint != null is implied */ || (sctpConnection = newEndpoint.getSctpConnection()) == null || !sctpConnection.isReady() || sctpConnection.isExpired()) { return; } // we have a new endpoint and it has an SCTP connection that is // ready and not expired. if somebody else is watching the new // endpoint, start its hq stream. boolean startHighQualityStream = false; for (Endpoint e : getSimulcastEngine().getVideoChannel().getContent().getConference().getEndpoints()) { // TODO(gp) need some synchronization here. What if the // selected endpoint changes while we're in the loop? if (e == newEndpoint) continue; Endpoint eSelectedEndpoint = e.getEffectivelySelectedEndpoint(); if (newEndpoint == eSelectedEndpoint) { // somebody is watching the new endpoint or somebody has not // yet signaled its selected endpoint to the bridge, start // the hq stream. if (logger.isDebugEnabled()) { Map<String, Object> map = new HashMap<String, Object>(3); map.put("e", e); map.put("newEndpoint", newEndpoint); map.put("maybe", eSelectedEndpoint == null ? "(maybe) " : ""); StringCompiler sc = new StringCompiler(map).c("{e.id} is {maybe} watching {newEndpoint.id}."); logDebug(sc.toString().replaceAll("\\s+", " ")); } startHighQualityStream = true; break; } } if (startHighQualityStream) { // TODO(gp) this assumes only a single hq stream. logDebug( getSimulcastEngine().getVideoChannel().getEndpoint().getID() + " notifies " + newEndpoint.getID() + " to start its HQ stream."); SimulcastLayer hqLayer = newSimulcastLayers[newSimulcastLayers.length - 1]; ; StartSimulcastLayerCommand command = new StartSimulcastLayerCommand(hqLayer); String json = mapper.toJson(command); try { newEndpoint.sendMessageOnDataChannel(json); } catch (IOException e) { logError(newEndpoint.getID() + " failed to send message on data channel.", e); } } } /** * Maybe send a data channel command to he associated simulcast sender to make it stop streaming * its hq stream, if it's not being watched by any participant. */ public void maybeSendStopHighQualityStreamCommand() { if (nativeSimulcast || !hasLayers()) { // In native simulcast the client adjusts its layers autonomously so // we don't need (nor we can) to control it with data channel // messages. return; } Endpoint oldEndpoint = getSimulcastEngine().getVideoChannel().getEndpoint(); SimulcastLayer[] oldSimulcastLayers = getSimulcastLayers(); SctpConnection sctpConnection; if (oldSimulcastLayers != null && oldSimulcastLayers.length > 1 /* oldEndpoint != null is implied*/ && (sctpConnection = oldEndpoint.getSctpConnection()) != null && sctpConnection.isReady() && !sctpConnection.isExpired()) { // we have an old endpoint and it has an SCTP connection that is // ready and not expired. if nobody else is watching the old // endpoint, stop its hq stream. boolean stopHighQualityStream = true; for (Endpoint e : getSimulcastEngine().getVideoChannel().getContent().getConference().getEndpoints()) { // TODO(gp) need some synchronization here. What if the selected // endpoint changes while we're in the loop? if (oldEndpoint != e && (oldEndpoint == e.getEffectivelySelectedEndpoint()) || e.getEffectivelySelectedEndpoint() == null) { // somebody is watching the old endpoint or somebody has not // yet signaled its selected endpoint to the bridge, don't // stop the hq stream. stopHighQualityStream = false; break; } } if (stopHighQualityStream) { // TODO(gp) this assumes only a single hq stream. logDebug( getSimulcastEngine().getVideoChannel().getEndpoint().getID() + " notifies " + oldEndpoint.getID() + " to stop " + "its HQ stream."); SimulcastLayer hqLayer = oldSimulcastLayers[oldSimulcastLayers.length - 1]; StopSimulcastLayerCommand command = new StopSimulcastLayerCommand(hqLayer); String json = mapper.toJson(command); try { oldEndpoint.sendMessageOnDataChannel(json); } catch (IOException e1) { logError(oldEndpoint.getID() + " failed to send " + "message on data channel.", e1); } } } } private void logDebug(String msg) { if (logger.isDebugEnabled()) { msg = getSimulcastEngine().getVideoChannel().getEndpoint().getID() + ": " + msg; logger.debug(msg); } } private void logWarn(String msg) { if (logger.isWarnEnabled()) { msg = getSimulcastEngine().getVideoChannel().getEndpoint().getID() + ": " + msg; logger.warn(msg); } } private void logError(String msg, Throwable e) { msg = getSimulcastEngine().getVideoChannel().getEndpoint().getID() + ": " + msg; logger.error(msg, e); } private void logInfo(String msg) { if (logger.isInfoEnabled()) { msg = getSimulcastEngine().getVideoChannel().getEndpoint().getID() + ": " + msg; logger.info(msg); } } /** * Notifies this {@code SimulcastReceiver} that a specific {@code SimulcastReceiver} has detected * the start of a new video frame in the RTP stream that it represents. Determines whether any of * {@link #simulcastLayers} other than {@code source} have been paused/stopped by the remote peer. * The determination is based on counting (video) frames. * * @param source the {@code SimulcastLayer} which is the source of the event i.e. which has * detected the start of a new video frame in the RTP stream that it represents * @param pkt the {@code RawPacket} which was received by {@code source} and possibly influenced * the decision that a new view frame was started in the RTP stream represented by {@code * source} * @param layers the set of {@code SimulcastLayer}s managed by this {@code SimulcastReceiver}. * Explicitly provided to the method in order to avoid invocations of {@link * #getSimulcastLayers()} because the latter makes a copy at the time of this writing. */ private void simulcastLayerFrameStarted( SimulcastLayer source, RawPacket pkt, SimulcastLayer[] layers) { // Allow the value of the constant TIMEOUT_ON_FRAME_COUNT to disable (at // compile time) the frame-based approach to the detection of layer // drops. if (TIMEOUT_ON_FRAME_COUNT <= 1) return; // Timeouts in layers caused by source may occur only based on the span // (of time or received frames) during which source has received // TIMEOUT_ON_FRAME_COUNT number of frames. The current method // invocation signals the receipt of 1 frame by source. int indexOfLastSourceOccurrenceInHistory = -1; int sourceFrameCount = 0; int ix = 0; for (Iterator<SimulcastLayer> it = simulcastLayerFrameHistory.iterator(); it.hasNext(); ++ix) { if (it.next() == source) { if (indexOfLastSourceOccurrenceInHistory != -1) { // Prune simulcastLayerFrameHistory so that it does not // become unnecessarily long. it.remove(); } else if (++sourceFrameCount >= TIMEOUT_ON_FRAME_COUNT - 1) { // The span of TIMEOUT_ON_FRAME_COUNT number of frames // received by source only is to be examined for the // purposes of timeouts. The current method invocations // signals the receipt of 1 frame by source so // TIMEOUT_ON_FRAME_COUNT - 1 occurrences of source in // simulcastLayerFrameHistory is enough. indexOfLastSourceOccurrenceInHistory = ix; } } } if (indexOfLastSourceOccurrenceInHistory != -1) { // Presumably, if a SimulcastLayer is active, all SimulcastLayers // before it (according to SimulcastLayer's order) are active as // well. Consequently, timeouts may occur in SimulcastLayers which // are after source. boolean maybeTimeout = false; for (SimulcastLayer layer : layers) { if (maybeTimeout) { // There's no point in timing layer out if it's timed out // already. if (layer.isStreaming()) { maybeTimeout(source, pkt, layer, indexOfLastSourceOccurrenceInHistory); } } else if (layer == source) { maybeTimeout = true; } } } // As previously stated, the current method invocation signals the // receipt of 1 frame by source. simulcastLayerFrameHistory.add(0, source); // TODO Prune simulcastLayerFrameHistory by forgetting so that it does // not become too long. } /** * Determines whether {@code effect} has been paused/stopped by the remote peer. The determination * is based on counting frames and is triggered by the receipt of (a piece of) a new (video) frame * by {@code cause}. * * @param cause the {@code SimulcastLayer} which has received (a piece of) a new (video) frame and * has thus triggered a check on {@code effect} * @param pkt the {@code RawPacket} which was received by {@code cause} and possibly influenced * the decision to trigger a check on {@code effect} * @param effect the {@code SimulcastLayer} which is to be checked whether it looks like it has * been paused/stopped by the remote peer * @param endIndexInSimulcastLayerFrameHistory */ private void maybeTimeout( SimulcastLayer cause, RawPacket pkt, SimulcastLayer effect, int endIndexInSimulcastLayerFrameHistory) { Iterator<SimulcastLayer> it = simulcastLayerFrameHistory.iterator(); boolean timeout = true; for (int ix = 0; it.hasNext() && ix < endIndexInSimulcastLayerFrameHistory; ++ix) { if (it.next() == effect) { timeout = false; break; } } if (timeout) { effect.maybeTimeout(pkt); if (!effect.isStreaming()) { // Since effect has been determined to have been paused/stopped // by the remote peer, its possible presence in // simulcastLayerFrameHistory is irrelevant now. In other words, // remove effect from simulcastLayerFrameHistory. while (it.hasNext()) { if (it.next() == effect) it.remove(); } } } } }
/** * Maybe send a data channel command to the associated <tt>Endpoint</tt> to make it start * streaming its hq stream, if it's being watched by some receiver. */ public void maybeSendStartHighQualityStreamCommand() { if (nativeSimulcast || !hasLayers()) { // In native simulcast the client adjusts its layers autonomously so // we don't need (nor we can) to control it with data channel // messages. return; } Endpoint newEndpoint = getSimulcastEngine().getVideoChannel().getEndpoint(); SimulcastLayer[] newSimulcastLayers = getSimulcastLayers(); SctpConnection sctpConnection; if (newSimulcastLayers == null || newSimulcastLayers.length <= 1 /* newEndpoint != null is implied */ || (sctpConnection = newEndpoint.getSctpConnection()) == null || !sctpConnection.isReady() || sctpConnection.isExpired()) { return; } // we have a new endpoint and it has an SCTP connection that is // ready and not expired. if somebody else is watching the new // endpoint, start its hq stream. boolean startHighQualityStream = false; for (Endpoint e : getSimulcastEngine().getVideoChannel().getContent().getConference().getEndpoints()) { // TODO(gp) need some synchronization here. What if the // selected endpoint changes while we're in the loop? if (e == newEndpoint) continue; Endpoint eSelectedEndpoint = e.getEffectivelySelectedEndpoint(); if (newEndpoint == eSelectedEndpoint) { // somebody is watching the new endpoint or somebody has not // yet signaled its selected endpoint to the bridge, start // the hq stream. if (logger.isDebugEnabled()) { Map<String, Object> map = new HashMap<String, Object>(3); map.put("e", e); map.put("newEndpoint", newEndpoint); map.put("maybe", eSelectedEndpoint == null ? "(maybe) " : ""); StringCompiler sc = new StringCompiler(map).c("{e.id} is {maybe} watching {newEndpoint.id}."); logDebug(sc.toString().replaceAll("\\s+", " ")); } startHighQualityStream = true; break; } } if (startHighQualityStream) { // TODO(gp) this assumes only a single hq stream. logDebug( getSimulcastEngine().getVideoChannel().getEndpoint().getID() + " notifies " + newEndpoint.getID() + " to start its HQ stream."); SimulcastLayer hqLayer = newSimulcastLayers[newSimulcastLayers.length - 1]; ; StartSimulcastLayerCommand command = new StartSimulcastLayerCommand(hqLayer); String json = mapper.toJson(command); try { newEndpoint.sendMessageOnDataChannel(json); } catch (IOException e) { logError(newEndpoint.getID() + " failed to send message on data channel.", e); } } }
/** * Notifies this instance that a <tt>DatagramPacket</tt> packet received on the data * <tt>DatagramSocket</tt> of this <tt>Channel</tt> has been accepted for further processing * within Jitsi Videobridge. * * @param pkt the accepted <tt>RawPacket</tt>. */ public void accepted(RawPacket pkt) { // With native simulcast we don't have a notification when a stream // has started/stopped. The simulcast manager implements a timeout // for the high quality stream and it needs to be notified when // the channel has accepted a datagram packet for the timeout to // function correctly. if (!hasLayers() || pkt == null) { return; } // Find the layer that corresponds to this packet. int acceptedSSRC = pkt.getSSRC(); SimulcastLayer[] layers = getSimulcastLayers(); SimulcastLayer acceptedLayer = null; for (SimulcastLayer layer : layers) { // We only care about the primary SSRC and not the RTX ssrc (or // future FEC ssrc). if ((int) layer.getPrimarySSRC() == acceptedSSRC) { acceptedLayer = layer; break; } } // If this is not an RTP packet or if we can't find an accepted // layer, log and return as it makes no sense to continue in this // situation. if (acceptedLayer == null) { return; } // There are sequences of packets with increasing timestamps but without // the marker bit set. Supposedly, they are probes to detect whether the // bandwidth may increase. We think that they should cause neither the // start nor the stop of any SimulcastLayer. // XXX There's RawPacket#getPayloadLength() but the implementation // includes pkt.paddingSize at the time of this writing and we do not // know whether that's going to stay that way. int pktPayloadLength = pkt.getLength() - pkt.getHeaderLength(); int pktPaddingSize = pkt.getPaddingSize(); if (pktPayloadLength <= pktPaddingSize) { if (logger.isTraceEnabled()) { logger.trace( "pkt.payloadLength= " + pktPayloadLength + " <= pkt.paddingSize= " + pktPaddingSize + "(" + pkt.getSequenceNumber() + ")"); } return; } // NOTE(gp) we expect the base layer to be always on, so we never touch // it or starve it. // XXX Refer to the implementation of // SimulcastLayer#touch(boolean, RawPacket) for an explanation of why we // chose to use a return value. boolean frameStarted = acceptedLayer.touch(pkt); if (frameStarted) simulcastLayerFrameStarted(acceptedLayer, pkt, layers); }